From pypy.commits at gmail.com  Sun Jan  1 05:16:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 01 Jan 2017 02:16:44 -0800 (PST)
Subject: [pypy-commit] pypy sandbox-lib: expand the interface,
 still only theoretical
Message-ID: <5868d70c.0b561c0a.9ea67.0263@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sandbox-lib
Changeset: r89283:be4412e6ecf2
Date: 2016-12-29 19:02 +0100
http://bitbucket.org/pypy/pypy/changeset/be4412e6ecf2/

Log:	expand the interface, still only theoretical

diff --git a/rpython/translator/rsandbox/src/part.h b/rpython/translator/rsandbox/src/part.h
--- a/rpython/translator/rsandbox/src/part.h
+++ b/rpython/translator/rsandbox/src/part.h
@@ -1,40 +1,46 @@
-/*** rpython/translator/rsandbox/src/part.h ***/
-
 #ifndef _RSANDBOX_H_
 #define _RSANDBOX_H_
 
 #ifndef RPY_SANDBOX_EXPORTED
-/* Common definitions when including this file from an external C project */
-
-#include <stdlib.h>
-#include <sys/utsname.h>
-
-#define RPY_SANDBOX_EXPORTED  extern
-
-typedef long Signed;
-typedef unsigned long Unsigned;
-
+#  define RPY_SANDBOX_EXPORTED  extern
 #endif
 
 
 /* ***********************************************************
 
-   WARNING: Python is not meant to be a safe language.  For example,
-   think about making a custom code object with a random byte string and
-   trying to interpret that.  A sandboxed PyPy contains extra safety
-   checks that can detect such invalid operations before they cause
-   problems.  When such a case is detected, THE WHOLE PROCESS IS
+   A direct interface for safely embedding Python inside a larger
+   application written in C (or any other language which can access C
+   libraries).
+
+   For now, there is little support for more complex cases.  Notably,
+   any call to functions like open() or any attempt to do 'import' of
+   any non-builtin module will fail.  This interface is not meant to
+   "drop in" a large amount of existing Python code.  If you are looking
+   for this and are not concerned about security, look at CFFI
+   embedding: http://cffi.readthedocs.org/en/latest/embedding.html .
+   Instead, this interface is meant to run small amounts of untrusted
+   Python code from third-party sources.  (It is possible to rebuild a
+   module system on top of this interface, by writing a custom
+   __import__ hook in Python.  Similarly, you cannot return arbitrary
+   Python objects to C code, but you can make a Python-side data
+   structure like a list or a dict, and pass integer indices to C.)
+
+   WARNING: Python is originally not meant to be a safe language.  For
+   example, think about making a custom code object with a random byte
+   string and trying to interpret that.  A sandboxed PyPy contains extra
+   safety checks that can detect such invalid operations before they
+   cause problems.  When such a case is detected, THE WHOLE PROCESS IS
    ABORTED right now.  In the future, there should be a setjmp/longjmp
    alternative to this, but the details need a bit of care (e.g. it
    would still create memory leaks).
 
-   For now, you have to accept that the process can be aborted if
-   given malicious code.  Also, running several Python sources from
-   different sources in the same process is not recommended---there is
-   only one global state: malicious code can easily mangle the state
-   of the Python interpreter, influencing subsequent runs.  Unless you
-   are fine with both issues, you MUST run Python from subprocesses,
-   not from your main program.
+   For now, you have to accept that the process can be aborted if given
+   malicious code.  Also, running several Python codes from different
+   untrusted sources in the same process is not recommended---there is
+   only one global state: malicious code can easily mangle the state of
+   the PyPy interpreter, influencing subsequent runs.  Unless you are
+   fine with both issues, you MUST run Python from subprocesses, not
+   from your main program.
 
    Multi-threading issues: DO NOT USE FROM SEVERAL THREADS AT THE SAME
    TIME!  You need a lock.  If you use subprocesses, they will likely
@@ -150,6 +156,14 @@
 */
 RPY_SANDBOX_EXPORTED void rsandbox_result_bytes(char *buf, size_t bufsize);
 
+/* If the called function returns a tuple of values, then the above
+   'result' functions work on individual items in the tuple, initially
+   the 0th one.  This function changes the current item to
+   'current_item' if that is within bounds.  Returns the total length of
+   the tuple, or -1 if not a tuple.
+*/
+RPY_SANDBOX_EXPORTED int rsandbox_result_tuple_item(int current_item);
+
 /* When an exception occurred in rsandbox_open() or rsandbox_call(),
    return more information as a 'char *' string.  Same rules as
    rsandbox_result_bytes().  (Careful, you MUST NOT assume that the
@@ -163,14 +177,38 @@
 RPY_SANDBOX_EXPORTED void rsandbox_last_exception(char *buf, size_t bufsize,
                                                   int traceback_limit);
 
+/* Installs a callback inside the module 'mod' under the name 'fnname'.
+   The Python code then sees a function 'fnname()' which invokes back
+   the C function given as the 'callback' parameter.  The 'callback' is
+   called with 'data' as sole argument (use NULL if you don't need
+   this).
+
+   When the Python 'fnname()' is called, the 'callback' is executed.  At
+   this point it can read the tuple of provided arguments using
+   rsandbox_result_...() and rsandbox_result_tuple_item().  Before
+   returning, it can use rsandbox_push_...() to push a number of
+   answers.  If more than one answer is pushed, the Python-side will get
+   them inside a tuple.  The callback cannot raise a Python-level
+   exception; if you need this, write a Python wrapper around the
+   callback, and have the callback return an error code.  (Or, of course,
+   just write the error message to stderr and call abort(), like many
+   other operations do.)
+
+   As usual, be ready to handle any broken argument combination in
+   rsandbox_result_...().
+
+   It is ok to use rsandbox_call() recursively from a callback.  Likely,
+   if you do, malicious code could in theory cause infinite recursion,
+   but any infinite recursion including this one should be caught by the
+   general detection logic and cause a Python-level
+   RuntimeError/RecursionError exception.
+*/
+RPY_SANDBOX_EXPORTED void rsandbox_install_callback(rsandbox_module_t *mod,
+                                                    const char *fnname,
+                                                    void (*callback)(void *),
+                                                    void *data);
+
 
 /************************************************************/
 
-
-/* The list of 'rsandbox_fnptr_*' function pointers is automatically
-   generated.  Most of these function pointers are initialized to
-   point to a function that aborts the sandboxed execution.  The
-   sandboxed program cannot, by default, use any of them.  A few
-   exceptions are provided, where the default implementation returns a
-   safe default (for example rsandbox_fnptr_getenv()).
-*/
+#endif

From pypy.commits at gmail.com  Sun Jan  1 05:16:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 01 Jan 2017 02:16:42 -0800 (PST)
Subject: [pypy-commit] pypy sandbox-lib: string => bytes
Message-ID: <5868d70a.61c9c20a.d7016.e827@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sandbox-lib
Changeset: r89282:3d02cf9459c7
Date: 2016-12-28 17:59 +0100
http://bitbucket.org/pypy/pypy/changeset/3d02cf9459c7/

Log:	string => bytes

diff --git a/rpython/translator/rsandbox/src/part.h b/rpython/translator/rsandbox/src/part.h
--- a/rpython/translator/rsandbox/src/part.h
+++ b/rpython/translator/rsandbox/src/part.h
@@ -20,7 +20,7 @@
 /* ***********************************************************
 
    WARNING: Python is not meant to be a safe language.  For example,
-   think about making a custom code object with a random string and
+   think about making a custom code object with a random byte string and
    trying to interpret that.  A sandboxed PyPy contains extra safety
    checks that can detect such invalid operations before they cause
    problems.  When such a case is detected, THE WHOLE PROCESS IS
@@ -72,7 +72,7 @@
 
    rsandbox_module_t *compile_expression(const char *expression)
    {
-       rsandbox_push_string(expression);   // 'expression' is untrusted
+       rsandbox_push_bytes(expression);   // 'expression' is untrusted
        return rsandbox_open(
            "code = compile(args[0], '<untrusted>', 'eval')\n"
            "def evaluate(n):\n"
@@ -102,8 +102,8 @@
 */
 RPY_SANDBOX_EXPORTED void rsandbox_push_long(long);
 RPY_SANDBOX_EXPORTED void rsandbox_push_double(double);
-RPY_SANDBOX_EXPORTED void rsandbox_push_string(const char *);
-RPY_SANDBOX_EXPORTED void rsandbox_push_string_and_size(const char *, size_t);
+RPY_SANDBOX_EXPORTED void rsandbox_push_bytes(const char *);
+RPY_SANDBOX_EXPORTED void rsandbox_push_bytes_and_size(const char *, size_t);
 RPY_SANDBOX_EXPORTED void rsandbox_push_none(void);
 RPY_SANDBOX_EXPORTED void rsandbox_push_rw_buffer(char *, size_t);
 
@@ -122,24 +122,25 @@
    malicious code returning results like inf, nan, or 1e-323.) */
 RPY_SANDBOX_EXPORTED double rsandbox_result_double(void);
 
-/* Returns the length of the string returned in the previous
-   rsandbox_call().  If it was not a string, returns 0. */
-RPY_SANDBOX_EXPORTED size_t rsandbox_result_string_length(void);
+/* Returns the length of the byte string returned in the previous
+   rsandbox_call().  If it was not a byte string, returns 0. */
+RPY_SANDBOX_EXPORTED size_t rsandbox_result_bytes_length(void);
 
-/* Returns the data in the string.  This function always writes an
-   additional '\0'.  If the string is longer than 'bufsize-1', it is
+/* Returns the data in the byte string.  This function always writes an
+   additional '\0'.  If the byte string is longer than 'bufsize-1', it is
    truncated to 'bufsize-1' characters.
 
    For small human-readable strings you can call
-   rsandbox_result_string() with some fixed maximum size.  You get a
+   rsandbox_result_bytes() with some fixed maximum size.  You get a
    regular null-terminated 'char *' string.  (If it contains embedded
    '\0', it will appear truncated; if the Python function did not
-   return a string at all, it will be completely empty; but anyway
+   return a byte string at all, it will be completely empty; but anyway
    you MUST be ready to handle any malformed string at all.)
 
    For strings of larger sizes or strings that can meaningfully
-   contain embedded '\0', you should allocate a 'buf' of size
-   'rsandbox_result_string_length() + 1'.
+   contain embedded '\0', you should compute 'bufsize =
+   rsandbox_result_bytes_length() + 1' and allocate a buffer of this
+   length.
 
    To repeat: Be careful when reading strings from Python!  They can
    contain any character, so be sure to escape them correctly (or
@@ -147,17 +148,20 @@
    further.  Malicious code can return any string.  Your code must be
    ready for anything.  Err on the side of caution.
 */
-RPY_SANDBOX_EXPORTED void rsandbox_result_string(char *buf, size_t bufsize);
+RPY_SANDBOX_EXPORTED void rsandbox_result_bytes(char *buf, size_t bufsize);
 
 /* When an exception occurred in rsandbox_open() or rsandbox_call(),
-   return more information as a string.  Same rules as
-   rsandbox_result_string().  (Careful, you MUST NOT assume that the
+   return more information as a 'char *' string.  Same rules as
+   rsandbox_result_bytes().  (Careful, you MUST NOT assume that the
    string is well-formed: malicious code can make it contain anything.
    If you are copying it to a web page, for example, then a good idea
    is to replace any character not in a whitelist with '?'.)
+
+   If 'traceback_limit' is greater than zero, the output is a multiline
+   traceback like in standard Python, with up to 'traceback_limit' levels.
 */
 RPY_SANDBOX_EXPORTED void rsandbox_last_exception(char *buf, size_t bufsize,
-                                                  int include_traceback);
+                                                  int traceback_limit);
 
 
 /************************************************************/

From pypy.commits at gmail.com  Sun Jan  1 06:09:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 01 Jan 2017 03:09:26 -0800 (PST)
Subject: [pypy-commit] pypy default: Allow --gc=boehm with the cpyext module.
Message-ID: <5868e366.43e61c0a.f871f.1d46@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89285:257848776fca
Date: 2017-01-01 11:30 +0100
http://bitbucket.org/pypy/pypy/changeset/257848776fca/

Log:	Allow --gc=boehm with the cpyext module.

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -305,9 +305,9 @@
             config.objspace.lonepycfiles = False
 
         if config.objspace.usemodules.cpyext:
-            if config.translation.gc != 'incminimark':
+            if config.translation.gc not in ('incminimark', 'boehm'):
                 raise Exception("The 'cpyext' module requires the 'incminimark'"
-                                " GC.  You need either 'targetpypystandalone.py"
+                                " 'boehm' GC.  You need either 'targetpypystandalone.py"
                                 " --withoutmod-cpyext' or '--gc=incminimark'")
 
         config.translating = True

From pypy.commits at gmail.com  Sun Jan  1 06:09:28 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 01 Jan 2017 03:09:28 -0800 (PST)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <5868e368.cf3fc20a.aa7e8.8225@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89286:999ff3b3f9a4
Date: 2017-01-01 11:31 +0100
http://bitbucket.org/pypy/pypy/changeset/999ff3b3f9a4/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -76,3 +76,8 @@
 PyMemoryViewObject with a PyBuffer attached so that the call to 
 ``PyMemoryView_GET_BUFFER`` does not leak a PyBuffer-sized piece of memory.
 Properly call ``bf_releasebuffer`` when not ``NULL``.
+
+.. branch: boehm-rawrefcount
+
+Support translations of cpyext with the Boehm GC (for special cases like
+revdb).

From pypy.commits at gmail.com  Sun Jan  1 06:09:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 01 Jan 2017 03:09:24 -0800 (PST)
Subject: [pypy-commit] pypy default: hg merge boehm-rawrefcount
Message-ID: <5868e364.0f341c0a.add22.0ae5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89284:a3aedbe6023d
Date: 2017-01-01 11:27 +0100
http://bitbucket.org/pypy/pypy/changeset/a3aedbe6023d/

Log:	hg merge boehm-rawrefcount

	A branch to add minimal support for rawrefcount in Boehm
	translations. This is needed by revdb.

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -1,7 +1,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.executioncontext import AsyncAction
+from pypy.interpreter import executioncontext
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
@@ -14,8 +14,9 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
-        pyobj_dealloc_action = PyObjDeallocAction(space)
-        self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+        if space.config.translation.gc != "boehm":
+            pyobj_dealloc_action = PyObjDeallocAction(space)
+            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -67,6 +68,11 @@
             state.api_lib = str(api.build_bridge(self.space))
         else:
             api.setup_library(self.space)
+            #
+            if self.space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(self.space)
+                self.space.actionflag.register_periodic_action(action,
+                    use_bytecode_counter=True)
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -84,8 +90,10 @@
         from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
-            rawrefcount.init(llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
-                                      self.dealloc_trigger))
+            if space.config.translation.gc != "boehm":
+                rawrefcount.init(
+                    llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
+                    self.dealloc_trigger))
             init_static_data_translated(space)
 
         setup_new_method_def(space)
@@ -143,15 +151,23 @@
         self.extensions[path] = w_copy
 
 
-class PyObjDeallocAction(AsyncAction):
+def _rawrefcount_perform(space):
+    from pypy.module.cpyext.pyobject import PyObject, decref
+    while True:
+        py_obj = rawrefcount.next_dead(PyObject)
+        if not py_obj:
+            break
+        decref(space, py_obj)
+
+class PyObjDeallocAction(executioncontext.AsyncAction):
     """An action that invokes _Py_Dealloc() on the dying PyObjects.
     """
+    def perform(self, executioncontext, frame):
+        _rawrefcount_perform(self.space)
 
+class BoehmPyObjDeallocAction(executioncontext.PeriodicAsyncAction):
+    # This variant is used with Boehm, which doesn't have the explicit
+    # callback.  Instead we must periodically check ourselves.
     def perform(self, executioncontext, frame):
-        from pypy.module.cpyext.pyobject import PyObject, decref
-
-        while True:
-            py_obj = rawrefcount.next_dead(PyObject)
-            if not py_obj:
-                break
-            decref(self.space, py_obj)
+        if we_are_translated():
+            _rawrefcount_perform(self.space)
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -4,10 +4,11 @@
 #  This is meant for pypy's cpyext module, but is a generally
 #  useful interface over our GC.  XXX "pypy" should be removed here
 #
-import sys, weakref
-from rpython.rtyper.lltypesystem import lltype, llmemory
+import sys, weakref, py
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.objectmodel import we_are_translated, specialize, not_rpython
 from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib import rgc
 
 
@@ -245,6 +246,11 @@
         v_p, v_ob = hop.inputargs(*hop.args_r)
         hop.exception_cannot_occur()
         hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)])
+        #
+        if hop.rtyper.annotator.translator.config.translation.gc == "boehm":
+            c_func = hop.inputconst(lltype.typeOf(func_boehm_eci),
+                                    func_boehm_eci)
+            hop.genop('direct_call', [c_func])
 
 
 class Entry(ExtRegistryEntry):
@@ -297,3 +303,10 @@
         v_ob = hop.genop('gc_rawrefcount_next_dead', [],
                          resulttype = llmemory.Address)
         return _spec_ob(hop, v_ob)
+
+src_dir = py.path.local(__file__).dirpath() / 'src'
+boehm_eci = ExternalCompilationInfo(
+    post_include_bits     = [(src_dir / 'boehm-rawrefcount.h').read()],
+    separate_module_files = [(src_dir / 'boehm-rawrefcount.c')],
+)
+func_boehm_eci = rffi.llexternal_use_eci(boehm_eci)
diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -0,0 +1,282 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <assert.h>
+#include <limits.h>
+#include <gc/gc.h>
+#include <gc/gc_mark.h>
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+#  define RPY_EXTERN  /* nothing */
+#else
+#  include "common_header.h"
+#endif
+
+
+#define REFCNT_FROM_PYPY  (LONG_MAX / 4 + 1)
+
+typedef struct pypy_header0 gcobj_t;    /* opaque here */
+
+#ifndef _WIN32
+typedef intptr_t Py_ssize_t;
+#else
+typedef long Py_ssize_t;
+#endif
+
+/* this is the first two words of the PyObject structure used in
+   pypy/module/cpyext */
+typedef struct {
+    Py_ssize_t ob_refcnt;
+    Py_ssize_t ob_pypy_link;
+} pyobj_t;
+
+struct link_s {
+    pyobj_t *pyobj;    /* NULL if entry unused */
+    uintptr_t gcenc;
+    struct link_s *next_in_bucket;
+};
+
+#define MARKER_LIST_START  ((pyobj_t *)-1)
+
+static struct link_s **hash_buckets, *hash_list, *hash_free_list;
+static uintptr_t hash_mask_bucket;
+static intptr_t hash_list_walk_next = -1;
+
+static uintptr_t hash_get_hash(gcobj_t *gcobj)
+{
+    assert(gcobj != NULL);
+    uintptr_t h = (uintptr_t)gcobj;
+    assert((h & 1) == 0);
+    h -= (h >> 6);
+    return h & hash_mask_bucket;
+}
+
+static gcobj_t *decode_gcenc(uintptr_t gcenc)
+{
+    if (gcenc & 1)
+        gcenc = ~gcenc;
+    return (gcobj_t *)gcenc;
+}
+
+static void hash_link(struct link_s *lnk)
+{
+    uintptr_t h = hash_get_hash(decode_gcenc(lnk->gcenc));
+    lnk->next_in_bucket = hash_buckets[h];
+    hash_buckets[h] = lnk;
+}
+
+static void boehm_is_about_to_collect(void);
+
+static void hash_grow_table(void)
+{
+    static int rec = 0;
+    assert(!rec);   /* recursive hash_grow_table() */
+    rec = 1;
+
+    if (hash_buckets == NULL)
+        GC_set_start_callback(boehm_is_about_to_collect);
+
+    uintptr_t i, num_buckets = (hash_mask_bucket + 1) * 2;
+    if (num_buckets < 16) num_buckets = 16;
+    assert((num_buckets & (num_buckets - 1)) == 0);  /* power of two */
+
+    /* The new hash_buckets: an array of pointers to struct link_s, of
+       length a power of two, used as a dictionary hash table.  It is
+       not allocated with Boehm because there is no point in Boehm looking
+       in it.
+     */
+    struct link_s **new_buckets = calloc(num_buckets, sizeof(struct link_s *));
+    assert(new_buckets);
+
+    /* The new hash_list: the array of all struct link_s.  Their order
+       is irrelevant.  There is a GC_register_finalizer() on the 'gcenc'
+       field, so we don't move the array; instead we allocate a new array
+       to use in addition to the old one.  There are a total of 2 to 4
+       times as many 'struct link_s' as the length of 'buckets'.
+     */
+    uintptr_t num_list = num_buckets * 2;
+    struct link_s *new_list = GC_MALLOC(num_list * sizeof(struct link_s));
+    for (i = num_list; i-- > 1; ) {
+        new_list[i].next_in_bucket = hash_free_list;
+        hash_free_list = &new_list[i];
+    }
+    /* list[0] is abused to store a pointer to the previous list and
+       the length of the current list */
+    struct link_s *old_list = hash_list;
+    new_list[0].next_in_bucket = old_list;
+    new_list[0].gcenc = num_list;
+    new_list[0].pyobj = MARKER_LIST_START;
+
+    hash_list = new_list;
+    free(hash_buckets);
+    hash_buckets = new_buckets;
+    hash_mask_bucket = num_buckets - 1;
+    hash_list_walk_next = hash_mask_bucket;
+
+    /* re-add all old 'struct link_s' to the hash_buckets */
+    struct link_s *plist = old_list;
+    while (plist != NULL) {
+        uintptr_t count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc != 0)
+                hash_link(&plist[i]);
+        }
+        plist = plist[0].next_in_bucket;
+    }
+    GC_reachable_here(old_list);
+
+    rec = 0;
+}
+
+static void hash_add_entry(gcobj_t *gcobj, pyobj_t *pyobj)
+{
+    if (hash_free_list == NULL) {
+        hash_grow_table();
+    }
+    assert(pyobj->ob_pypy_link == 0);
+
+    struct link_s *lnk = hash_free_list;
+    hash_free_list = lnk->next_in_bucket;
+    lnk->pyobj = pyobj;
+    lnk->gcenc = (uintptr_t)gcobj;
+    pyobj->ob_pypy_link = (Py_ssize_t)lnk;
+
+    hash_link(lnk);
+
+    if (GC_base(gcobj) == NULL) {
+        /* 'gcobj' is probably a prebuilt object - it makes no */
+        /* sense to register it then, and it crashes Boehm in */
+        /* quite obscure ways */
+    }
+    else {
+        int j = GC_general_register_disappearing_link(
+                                    (void **)&lnk->gcenc, gcobj);
+        assert(j == GC_SUCCESS);
+    }
+}
+
+static pyobj_t *hash_get_entry(gcobj_t *gcobj)
+{
+    if (hash_buckets == NULL)
+        return NULL;
+    uintptr_t h = hash_get_hash(gcobj);
+    struct link_s *lnk = hash_buckets[h];
+    while (lnk != NULL) {
+        assert(lnk->pyobj != NULL);
+        if (decode_gcenc(lnk->gcenc) == gcobj)
+            return lnk->pyobj;
+        lnk = lnk->next_in_bucket;
+    }
+    return NULL;
+}
+
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_next_dead(void)
+{
+    while (hash_list_walk_next >= 0) {
+        struct link_s *p, **pp = &hash_buckets[hash_list_walk_next];
+        while (1) {
+            p = *pp;
+            if (p == NULL)
+                break;
+            assert(p->pyobj != NULL);
+            if (p->gcenc == 0) {
+                /* quadratic time on the number of links from the same
+                   bucket chain, but it should be small with very high
+                   probability */
+                pyobj_t *result = p->pyobj;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+                printf("next_dead: %p\n", result);
+#endif
+                assert(result->ob_refcnt == REFCNT_FROM_PYPY);
+                result->ob_refcnt = 1;
+                p->pyobj = NULL;
+                *pp = p->next_in_bucket;
+                p->next_in_bucket = hash_free_list;
+                hash_free_list = p;
+                return result;
+            }
+            else {
+                assert(p->gcenc != ~(uintptr_t)0);
+                pp = &p->next_in_bucket;
+            }
+        }
+        hash_list_walk_next--;
+    }
+    return NULL;
+}
+
+RPY_EXTERN
+void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                     /*pyobj_t*/void *pyobj)
+{
+    gcobj_t *gcobj1 = (gcobj_t *)gcobj;
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    assert(pyobj1->ob_pypy_link == 0);
+    /*assert(pyobj1->ob_refcnt >= REFCNT_FROM_PYPY);*/
+    /*^^^ could also be fixed just after the call to create_link_pypy()*/
+
+    hash_add_entry(gcobj1, pyobj1);
+}
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj)
+{
+    return hash_get_entry((gcobj_t *)gcobj);
+}
+
+RPY_EXTERN
+/*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj)
+{
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    if (pyobj1->ob_pypy_link == 0)
+        return NULL;
+
+    struct link_s *lnk = (struct link_s *)pyobj1->ob_pypy_link;
+    assert(lnk->pyobj == pyobj1);
+    
+    gcobj_t *g = decode_gcenc(lnk->gcenc);
+    assert(g != NULL);
+    return g;
+}
+
+static void boehm_is_about_to_collect(void)
+{
+    struct link_s *plist = hash_list;
+    uintptr_t gcenc_union = 0;
+    while (plist != NULL) {
+        uintptr_t i, count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc == 0)
+                continue;
+
+            pyobj_t *p = plist[i].pyobj;
+            assert(p != NULL);
+            assert(p->ob_refcnt >= REFCNT_FROM_PYPY);
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+            printf("plist[%d].gcenc: %p ", (int)i, plist[i].gcenc);
+#endif
+
+            if ((plist[i].gcenc & 1) ^ (p->ob_refcnt == REFCNT_FROM_PYPY)) {
+                /* ob_refcnt > FROM_PYPY: non-zero regular refcnt, 
+                   the gc obj must stay alive.  decode gcenc.
+                   ---OR---
+                   ob_refcnt == FROM_PYPY: no refs from C code, the
+                   gc obj must not (necessarily) stay alive.  encode gcenc.
+                */
+                plist[i].gcenc = ~plist[i].gcenc;
+            }
+            gcenc_union |= plist[i].gcenc;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+            printf("-> %p\n", plist[i].gcenc);
+#endif
+    }
+        plist = plist[0].next_in_bucket;
+    }
+    if (gcenc_union & 1)   /* if there is at least one item potentially dead */
+        hash_list_walk_next = hash_mask_bucket;
+}
diff --git a/rpython/rlib/src/boehm-rawrefcount.h b/rpython/rlib/src/boehm-rawrefcount.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.h
@@ -0,0 +1,24 @@
+
+/* Missing:
+   OP_GC_RAWREFCOUNT_INIT(callback, r): the callback is not supported here
+   OP_GC_RAWREFCOUNT_CREATE_LINK_PYOBJ(): not implemented, maybe not needed
+*/
+
+#define OP_GC_RAWREFCOUNT_CREATE_LINK_PYPY(gcobj, pyobj, r)   \
+    gc_rawrefcount_create_link_pypy(gcobj, pyobj)
+
+#define OP_GC_RAWREFCOUNT_FROM_OBJ(gcobj, r)   \
+    r = gc_rawrefcount_from_obj(gcobj)
+
+#define OP_GC_RAWREFCOUNT_TO_OBJ(pyobj, r)   \
+    r = gc_rawrefcount_to_obj(pyobj)
+
+#define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
+    r = gc_rawrefcount_next_dead()
+
+
+RPY_EXTERN void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                                /*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj);
+RPY_EXTERN /*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_next_dead(void);
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -264,6 +264,9 @@
             if rawrefcount.next_dead(PyObject) != ob:
                 print "NEXT_DEAD != OB"
                 return 1
+            if ob.c_ob_refcnt != 1:
+                print "next_dead().ob_refcnt != 1"
+                return 1
             if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
                 print "NEXT_DEAD second time != NULL"
                 return 1
@@ -283,3 +286,55 @@
         t, cbuilder = self.compile(entry_point)
         data = cbuilder.cmdexec('hi there')
         assert data.startswith('OK!\n')
+
+
+class TestBoehmTranslated(StandaloneTests):
+
+    def test_full_translation(self):
+
+        def make_ob():
+            p = W_Root(42)
+            ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+            rawrefcount.create_link_pypy(p, ob)
+            ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            assert rawrefcount.from_obj(PyObject, p) == ob
+            assert rawrefcount.to_obj(W_Root, ob) == p
+            return ob
+
+        prebuilt_p = W_Root(-42)
+        prebuilt_ob = lltype.malloc(PyObjectS, flavor='raw', zero=True,
+                                    immortal=True)
+
+        def entry_point(argv):
+            rawrefcount.create_link_pypy(prebuilt_p, prebuilt_ob)
+            prebuilt_ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            oblist = [make_ob() for i in range(50)]
+            rgc.collect()
+            deadlist = []
+            while True:
+                ob = rawrefcount.next_dead(PyObject)
+                if not ob: break
+                if ob.c_ob_refcnt != 1:
+                    print "next_dead().ob_refcnt != 1"
+                    return 1
+                deadlist.append(ob)
+            if len(deadlist) == 0:
+                print "no dead object"
+                return 1
+            if len(deadlist) < 30:
+                print "not enough dead objects"
+                return 1
+            for ob in deadlist:
+                if ob not in oblist:
+                    print "unexpected value for dead pointer"
+                    return 1
+                oblist.remove(ob)
+            print "OK!"
+            lltype.free(ob, flavor='raw')
+            return 0
+
+        self.config = get_combined_translation_config(translating=True)
+        self.config.translation.gc = "boehm"
+        t, cbuilder = self.compile(entry_point)
+        data = cbuilder.cmdexec('hi there')
+        assert data.startswith('OK!\n')
diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -0,0 +1,230 @@
+import itertools, os, subprocess
+from hypothesis import given, strategies
+from rpython.tool.udir import udir
+
+
+TEST_CODE = r"""
+#define TEST_BOEHM_RAWREFCOUNT
+#include "boehm-rawrefcount.c"
+
+static gcobj_t *alloc_gcobj(void)   /* for tests */
+{
+    gcobj_t *g = GC_MALLOC(1000);
+    printf("gc obj: %p\n", g);
+    return g;
+}
+
+static pyobj_t *alloc_pyobj(void)   /* for tests */
+{
+    pyobj_t *p = malloc(1000);
+    p->ob_refcnt = 1;
+    p->ob_pypy_link = 0;
+    printf("py obj: %p\n", p);
+    return p;
+}
+
+static void decref(pyobj_t *p)      /* for tests */
+{
+    p->ob_refcnt--;
+    if (p->ob_refcnt == 0) {
+        printf("decref to zero: %p\n", p);
+        free(p);
+    }
+    assert(p->ob_refcnt >= REFCNT_FROM_PYPY ||
+           p->ob_refcnt < REFCNT_FROM_PYPY * 0.99);
+}
+
+void run_test(void);     /* forward declaration, produced by the test */
+
+int main(void)
+{
+    run_test();
+    while (gc_rawrefcount_next_dead() != NULL)
+        ;
+    return 0;
+}
+"""
+
+
+operations = strategies.sampled_from([
+    'new_pyobj',
+    'new_gcobj',
+    'create_link',
+    'from_obj',
+    'to_obj',
+    'forget_pyobj',
+    'forget_gcobj',
+    'collect',
+    'dead',
+    ])
+
+
+ at strategies.composite
+def make_code(draw):
+    code = []
+    pyobjs = []
+    gcobjs = []
+    num_gcobj = itertools.count()
+    num_pyobj = itertools.count()
+    links_g2p = {}
+    links_p2g = {}
+
+    def new_gcobj():
+        varname = 'g%d' % next(num_gcobj)
+        code.append('gcobj_t *volatile %s = alloc_gcobj();' % varname)
+        gcobjs.append(varname)
+        return varname
+
+    def new_pyobj():
+        varname = 'p%d' % next(num_pyobj)
+        code.append('pyobj_t *%s = alloc_pyobj();' % varname)
+        pyobjs.append(varname)
+        return varname
+
+    for op in draw(strategies.lists(operations, average_size=250)):
+        if op == 'new_gcobj':
+            new_gcobj()
+        elif op == 'new_pyobj':
+            new_pyobj()
+        elif op == 'create_link':
+            gvars = [varname for varname in gcobjs if varname not in links_g2p]
+            if gvars == []:
+                gvars.append(new_gcobj())
+            pvars = [varname for varname in pyobjs if varname not in links_p2g]
+            if pvars == []:
+                pvars.append(new_pyobj())
+            gvar = draw(strategies.sampled_from(gvars))
+            pvar = draw(strategies.sampled_from(pvars))
+            code.append(r'printf("create_link %%p-%%p\n", %s, %s); '
+                            % (gvar, pvar) +
+                        "%s->ob_refcnt += REFCNT_FROM_PYPY; " % pvar +
+                        "gc_rawrefcount_create_link_pypy(%s, %s);"
+                            % (gvar, pvar))
+            links_g2p[gvar] = pvar
+            links_p2g[pvar] = gvar
+        elif op == 'from_obj':
+            if gcobjs:
+                prnt = False
+                gvar = draw(strategies.sampled_from(gcobjs))
+                if gvar not in links_g2p:
+                    check = "== NULL"
+                elif links_g2p[gvar] in pyobjs:
+                    check = "== %s" % (links_g2p[gvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_from_obj(%s) %s);"
+                            % (gvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", %s, '
+                        'gc_rawrefcount_from_obj(%s));' % (gvar, gvar))
+        elif op == 'to_obj':
+            if pyobjs:
+                prnt = False
+                pvar = draw(strategies.sampled_from(pyobjs))
+                if pvar not in links_p2g:
+                    check = "== NULL"
+                elif links_p2g[pvar] in gcobjs:
+                    check = "== %s" % (links_p2g[pvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_to_obj(%s) %s);"
+                            % (pvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", '
+                        'gc_rawrefcount_to_obj(%s), %s);' % (pvar, pvar))
+        elif op == 'forget_pyobj':
+            if pyobjs:
+                index = draw(strategies.sampled_from(range(len(pyobjs))))
+                pvar = pyobjs.pop(index)
+                code.append(r'printf("-p%%p\n", %s); ' % pvar +
+                            "decref(%s); %s = NULL;" % (pvar, pvar))
+        elif op == 'forget_gcobj':
+            if gcobjs:
+                index = draw(strategies.sampled_from(range(len(gcobjs))))
+                gvar = gcobjs.pop(index)
+                code.append(r'printf("-g%%p\n", %s); ' % gvar +
+                            "%s = NULL;" % (gvar,))
+        elif op == 'collect':
+            code.append("GC_gcollect();")
+        elif op == 'dead':
+            code.append('gc_rawrefcount_next_dead();')
+        else:
+            assert False, op
+
+    return '\n'.join(code)
+
+
+ at given(make_code())
+def test_random(code):
+    filename = str(udir.join("test-rawrefcount-boehm.c"))
+    with open(filename, "w") as f:
+        print >> f, TEST_CODE
+        print >> f, 'void run_test(void) {'
+        print >> f, code
+        print >> f, '}'
+
+    srcdir = os.path.dirname(os.path.dirname(
+        os.path.abspath(os.path.join(__file__))))
+    srcdir = os.path.join(srcdir, 'src')
+
+    err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o test-rawrefcount-boehm"
+                    " test-rawrefcount-boehm.c" % (udir, srcdir))
+    assert err == 0
+    p = subprocess.Popen("./test-rawrefcount-boehm", stdout=subprocess.PIPE,
+                         cwd=str(udir))
+    stdout, _ = p.communicate()
+    assert p.wait() == 0
+
+    gcobjs = {}
+    pyobjs = {}
+    links_p2g = {}
+    links_g2p = {}
+    for line in stdout.splitlines():
+        if line.startswith('py obj: '):
+            p = line[8:]
+            assert not pyobjs.get(p)
+            pyobjs[p] = True
+            assert p not in links_p2g
+        elif line.startswith('gc obj: '):
+            g = line[8:]
+            assert not gcobjs.get(g)
+            gcobjs[g] = True
+            if g in links_g2p: del links_g2p[g]
+        elif line.startswith('-p'):
+            p = line[2:]
+            assert pyobjs[p] == True
+            pyobjs[p] = False
+        elif line.startswith('-g'):
+            g = line[2:]
+            assert gcobjs[g] == True
+            gcobjs[g] = False
+        elif line.startswith('decref to zero: '):
+            p = line[16:]
+            assert pyobjs[p] == False
+            assert p not in links_p2g
+            del pyobjs[p]
+        elif line.startswith('create_link '):
+            g, p = line[12:].split('-')
+            assert g in gcobjs
+            assert p in pyobjs
+            assert g not in links_g2p
+            assert p not in links_p2g
+            links_g2p[g] = p
+            links_p2g[p] = g
+        elif line.startswith('link '):
+            g, p = line[5:].split('-')
+            assert g in gcobjs
+            assert p in pyobjs
+            assert links_g2p[g] == p
+            assert links_p2g[p] == g
+        elif line.startswith('plist['):
+            pass
+        elif line.startswith('next_dead: '):
+            p = line[11:]
+            assert pyobjs[p] == False
+            del pyobjs[p]
+            del links_p2g[p]
+        else:
+            assert False, repr(line)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -494,6 +494,7 @@
     'gc_rawrefcount_mark_deallocating': LLOp(),
     'gc_rawrefcount_from_obj':          LLOp(sideeffects=False),
     'gc_rawrefcount_to_obj':            LLOp(sideeffects=False),
+    'gc_rawrefcount_next_dead':         LLOp(),
 
     # ------- JIT & GC interaction, only for some GCs ----------
 

From pypy.commits at gmail.com  Mon Jan  2 08:57:55 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 02 Jan 2017 05:57:55 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Define PyComplex_AsCComplex and
 PyComplex_FromCComplex in a .c instead of a .h
Message-ID: <586a5c63.0b561c0a.9ea67.f850@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89290:babb6b221e5e
Date: 2017-01-02 13:56 +0000
http://bitbucket.org/pypy/pypy/changeset/babb6b221e5e/

Log:	Define PyComplex_AsCComplex and PyComplex_FromCComplex in a .c
	instead of a .h

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1325,6 +1325,7 @@
                          source_dir / "pythonrun.c",
                          source_dir / "sysmodule.c",
                          source_dir / "bufferobject.c",
+                         source_dir / "complexobject.c",
                          source_dir / "cobject.c",
                          source_dir / "structseq.c",
                          source_dir / "capsule.c",
diff --git a/pypy/module/cpyext/include/complexobject.h b/pypy/module/cpyext/include/complexobject.h
--- a/pypy/module/cpyext/include/complexobject.h
+++ b/pypy/module/cpyext/include/complexobject.h
@@ -16,23 +16,8 @@
     Py_complex cval;
 } PyComplexObject;
 
-/* generated function */
-PyAPI_FUNC(int) _PyComplex_AsCComplex(PyObject *, Py_complex *);
-PyAPI_FUNC(PyObject *) _PyComplex_FromCComplex(Py_complex *);
-
-Py_LOCAL_INLINE(Py_complex) PyComplex_AsCComplex(PyObject *obj)
-{
-    Py_complex result;
-    _PyComplex_AsCComplex(obj, &result);
-    return result;
-}
-
-// shmuller 2013/07/30: Make a function, since macro will fail in C++ due to 
-//                      const correctness if called with "const Py_complex"
-//#define PyComplex_FromCComplex(c) _PyComplex_FromCComplex(&c)
-Py_LOCAL_INLINE(PyObject *) PyComplex_FromCComplex(Py_complex c) {
-    return _PyComplex_FromCComplex(&c);
-}
+PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *obj);
+PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex c);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/src/complexobject.c b/pypy/module/cpyext/src/complexobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/complexobject.c
@@ -0,0 +1,17 @@
+
+#include "Python.h"
+
+Py_complex
+PyComplex_AsCComplex(PyObject *obj)
+{
+    printf("asdfgg");
+    Py_complex result;
+    _PyComplex_AsCComplex(obj, &result);
+    return result;
+}
+
+PyObject *
+PyComplex_FromCComplex(Py_complex c)
+{
+    return _PyComplex_FromCComplex(&c);
+}

From pypy.commits at gmail.com  Mon Jan  2 09:29:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 06:29:29 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-skip: does not work (tested on tannit)
Message-ID: <586a63c9.031f1c0a.bd093.185d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cppyy-skip
Changeset: r89291:270b5ea6f31a
Date: 2017-01-02 15:28 +0100
http://bitbucket.org/pypy/pypy/changeset/270b5ea6f31a/

Log:	does not work (tested on tannit)


From pypy.commits at gmail.com  Mon Jan  2 09:30:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 06:30:25 -0800 (PST)
Subject: [pypy-commit] pypy default: Really skip the cppyy tests (tested on
 tannit where gcc is old)
Message-ID: <586a6401.43e61c0a.f871f.1244@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89292:1717aa219d47
Date: 2017-01-02 15:29 +0100
http://bitbucket.org/pypy/pypy/changeset/1717aa219d47/

Log:	Really skip the cppyy tests (tested on tannit where gcc is old)

diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -2,12 +2,6 @@
 
 @py.test.mark.tryfirst
 def pytest_runtest_setup(item):
-    if 'linux' in sys.platform:
-        # tests require minimally std=c++11
-        cc_info = py.process.cmdexec('gcc -v --help')
-        if not '-std=c++11' in cc_info:
-            py.test.skip('skipping tests because gcc does not support C++11')
-
     if py.path.local.sysfind('genreflex') is None:
         import pypy.module.cppyy.capi.loadable_capi as lcapi
         if 'dummy' in lcapi.reflection_library:
diff --git a/pypy/module/cppyy/test/support.py b/pypy/module/cppyy/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cppyy/test/support.py
@@ -0,0 +1,16 @@
+import py, sys, subprocess
+
+currpath = py.path.local(__file__).dirpath()
+
+
+def setup_make(targetname):
+    if sys.platform == 'win32':
+        py.test.skip("win32 not supported so far")
+    import pypy.module.cppyy.capi.loadable_capi as lcapi
+    popen = subprocess.Popen(["make", targetname], cwd=str(currpath),
+                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    stdout, _ = popen.communicate()
+    if popen.returncode:
+        if '-std=c++11' in stdout:
+            py.test.skip("gcc does not seem to support -std=c++11")
+        raise OSError("'make' failed:\n%s" % (stdout,))
diff --git a/pypy/module/cppyy/test/test_cppyy.py b/pypy/module/cppyy/test/test_cppyy.py
--- a/pypy/module/cppyy/test/test_cppyy.py
+++ b/pypy/module/cppyy/test/test_cppyy.py
@@ -1,18 +1,15 @@
 import py, os, sys
+import subprocess
 
 from pypy.module.cppyy import interp_cppyy, executor
+from .support import setup_make
 
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict.so"))
 
 def setup_module(mod):
-    if sys.platform == 'win32':
-        py.test.skip("win32 not supported so far")
-    import pypy.module.cppyy.capi.loadable_capi as lcapi
-    err = os.system("cd '%s' && make example01Dict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    setup_make("example01Dict.so")
 
 class TestCPPYYImplementation:
     def test01_class_query(self, space):
diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py
--- a/pypy/module/cppyy/test/test_datatypes.py
+++ b/pypy/module/cppyy/test/test_datatypes.py
@@ -1,15 +1,12 @@
 import py, os, sys
+from .support import setup_make
 
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("datatypesDict.so"))
 
 def setup_module(mod):
-    if sys.platform == 'win32':
-        py.test.skip("win32 not supported so far")
-    err = os.system("cd '%s' && make datatypesDict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    setup_make("datatypesDict.so")
 
 class AppTestDATATYPES:
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py
--- a/pypy/module/cppyy/test/test_pythonify.py
+++ b/pypy/module/cppyy/test/test_pythonify.py
@@ -1,17 +1,14 @@
 import py, os, sys
 
 from pypy.module.cppyy import interp_cppyy, executor
+from .support import setup_make
 
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict.so"))
 
 def setup_module(mod):
-    if sys.platform == 'win32':
-        py.test.skip("win32 not supported so far")
-    err = os.system("cd '%s' && make example01Dict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    setup_make("example01Dict.so")
 
 class AppTestPYTHONIFY:
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])

From pypy.commits at gmail.com  Mon Jan  2 09:31:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 06:31:48 -0800 (PST)
Subject: [pypy-commit] pypy default: Update the license year
Message-ID: <586a6454.8d071c0a.32208.226a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89293:a193393f9691
Date: 2017-01-02 15:31 +0100
http://bitbucket.org/pypy/pypy/changeset/a193393f9691/

Log:	Update the license year

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at

From pypy.commits at gmail.com  Mon Jan  2 09:32:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 06:32:26 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix test_lloperation
Message-ID: <586a647a.4dd41c0a.c02f0.fc56@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89294:232e4fe233a9
Date: 2017-01-02 15:31 +0100
http://bitbucket.org/pypy/pypy/changeset/232e4fe233a9/

Log:	Fix test_lloperation

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -966,6 +966,9 @@
     def op_gc_rawrefcount_mark_deallocating(self, *args):
         raise NotImplementedError("gc_rawrefcount_mark_deallocating")
 
+    def op_gc_rawrefcount_next_dead(self, *args):
+        raise NotImplementedError("gc_rawrefcount_next_dead")
+
     def op_do_malloc_fixedsize(self):
         raise NotImplementedError("do_malloc_fixedsize")
     def op_do_malloc_fixedsize_clear(self):

From pypy.commits at gmail.com  Mon Jan  2 09:35:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 06:35:50 -0800 (PST)
Subject: [pypy-commit] pypy default: fix
Message-ID: <586a6546.0e0a1c0a.5228b.0f36@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89295:8def4d18f54d
Date: 2017-01-02 14:46 +0000
http://bitbucket.org/pypy/pypy/changeset/8def4d18f54d/

Log:	fix

diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
--- a/rpython/rlib/src/boehm-rawrefcount.c
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -258,7 +258,7 @@
             assert(p->ob_refcnt >= REFCNT_FROM_PYPY);
 
 #ifdef TEST_BOEHM_RAWREFCOUNT
-            printf("plist[%d].gcenc: %p ", (int)i, plist[i].gcenc);
+            printf("plist[%d].gcenc: %p ", (int)i, (void *)plist[i].gcenc);
 #endif
 
             if ((plist[i].gcenc & 1) ^ (p->ob_refcnt == REFCNT_FROM_PYPY)) {
@@ -272,7 +272,7 @@
             }
             gcenc_union |= plist[i].gcenc;
 #ifdef TEST_BOEHM_RAWREFCOUNT
-            printf("-> %p\n", plist[i].gcenc);
+            printf("-> %p\n", (void *)plist[i].gcenc);
 #endif
     }
         plist = plist[0].next_in_bucket;

From pypy.commits at gmail.com  Mon Jan  2 09:43:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 06:43:46 -0800 (PST)
Subject: [pypy-commit] pypy default: Skip this test if Boehm is too old
Message-ID: <586a6722.86cbc20a.960e5.d8d0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89296:7a93d726ec2a
Date: 2017-01-02 14:54 +0000
http://bitbucket.org/pypy/pypy/changeset/7a93d726ec2a/

Log:	Skip this test if Boehm is too old

diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
--- a/rpython/rlib/test/test_rawrefcount_boehm.py
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -1,8 +1,23 @@
-import itertools, os, subprocess
+import itertools, os, subprocess, py
 from hypothesis import given, strategies
 from rpython.tool.udir import udir
 
 
+def setup_module():
+    filename = str(udir.join("test-rawrefcount-boehm-check.c"))
+    with open(filename, "w") as f:
+        print >> f, '#include "gc/gc_mark.h"'
+        print >> f, 'void *testing(void) {'
+        print >> f, '    return &GC_set_start_callback;'
+        print >> f, '}'
+
+    err = os.system("cd '%s' && gcc -c test-rawrefcount-boehm-check.c"
+                    % (udir,))
+    if err != 0:
+        py.test.skip("Boehm GC not installed or too old version")
+
+
+
 TEST_CODE = r"""
 #define TEST_BOEHM_RAWREFCOUNT
 #include "boehm-rawrefcount.c"

From pypy.commits at gmail.com  Mon Jan  2 09:55:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 06:55:26 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix the exception raised by
 socket.getservbyport(-1)
Message-ID: <586a69de.8675c20a.dfef3.887b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89297:6ed7e32b3c10
Date: 2017-01-02 15:54 +0100
http://bitbucket.org/pypy/pypy/changeset/6ed7e32b3c10/

Log:	Fix the exception raised by socket.getservbyport(-1)

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -98,7 +98,8 @@
         proto = space.str_w(w_proto)
 
     if port < 0 or port > 0xffff:
-        raise oefmt(space.w_ValueError, "getservbyport: port must be 0-65535.")
+        raise oefmt(space.w_OverflowError,
+                    "getservbyport: port must be 0-65535.")
 
     try:
         service = rsocket.getservbyport(port, proto)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -83,11 +83,6 @@
                          "(_socket, port): return _socket.getservbyport(port)")
     assert space.unwrap(name) == "smtp"
 
-    from pypy.interpreter.error import OperationError
-    exc = raises(OperationError, space.appexec,
-           [w_socket], "(_socket): return _socket.getservbyport(-1)")
-    assert exc.value.match(space, space.w_ValueError)
-
 def test_getprotobyname():
     name = "tcp"
     w_n = space.appexec([w_socket, space.wrap(name)],
@@ -325,6 +320,10 @@
         assert _socket.socket.__name__ == 'socket'
         assert _socket.socket.__module__ == '_socket'
 
+    def test_getservbyport(self):
+        import _socket
+        raises(OverflowError, _socket.getservbyport, -1)
+
     def test_ntoa_exception(self):
         import _socket
         raises(_socket.error, _socket.inet_ntoa, b"ab")

From pypy.commits at gmail.com  Mon Jan  2 10:02:45 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 07:02:45 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: be sure w_obj's best_base is
 attached before attching w_obj
Message-ID: <586a6b95.43e61c0a.f871f.1dc4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89299:41382dd5d75f
Date: 2017-01-02 16:53 +0200
http://bitbucket.org/pypy/pypy/changeset/41382dd5d75f/

Log:	be sure w_obj's best_base is attached before attching w_obj

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -41,6 +41,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 DEBUG_WRAPPER = True
 
@@ -1164,6 +1165,34 @@
     setup_init_functions(eci, translating=False)
     return modulename.new(ext='')
 
+def attach_recusively(space, static_pyobjs, static_objs_w, attached_objs, i):
+    # Start at i but make sure all the base classes are already attached
+    from pypy.module.cpyext.pyobject import get_typedescr, make_ref
+    if i in attached_objs:
+        return
+    py_obj = static_pyobjs[i]
+    w_obj = static_objs_w[i]
+    w_base = None
+    # w_obj can be NotImplemented, which is not a W_TypeObject
+    if isinstance(w_obj, W_TypeObject):
+        bases_w = w_obj.bases_w
+        if bases_w:
+            w_base = find_best_base(bases_w)
+        if w_base:
+            try:
+                j = static_objs_w.index(w_base)
+            except ValueError:
+                j = -1
+            if j >=0 and j not in attached_objs:
+                attach_recusively(space, static_pyobjs, static_objs_w,
+                                                 attached_objs, j)
+    w_type = space.type(w_obj)
+    typedescr = get_typedescr(w_type.layout.typedef)
+    py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
+                                 make_ref(space, w_type))
+    typedescr.attach(space, py_obj, w_obj)
+    attached_objs.append(i)
+
 
 class StaticObjectBuilder:
     def __init__(self, space):
@@ -1185,7 +1214,6 @@
 
     def attach_all(self):
         # this is RPython, called once in pypy-c when it imports cpyext
-        from pypy.module.cpyext.pyobject import get_typedescr, make_ref
         from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
         from pypy.module.cpyext.pyobject import track_reference
         #
@@ -1196,14 +1224,9 @@
             track_reference(space, static_pyobjs[i], static_objs_w[i])
         #
         self.cpyext_type_init = []
+        attached_objs = []
         for i in range(len(static_objs_w)):
-            py_obj = static_pyobjs[i]
-            w_obj = static_objs_w[i]
-            w_type = space.type(w_obj)
-            typedescr = get_typedescr(w_type.layout.typedef)
-            py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
-                                         make_ref(space, w_type))
-            typedescr.attach(space, py_obj, w_obj)
+            attach_recusively(space, static_pyobjs, static_objs_w, attached_objs, i)
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:

From pypy.commits at gmail.com  Mon Jan  2 10:02:43 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 07:02:43 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: lookup slot functions from
 tp_base if not in search_dict_w
Message-ID: <586a6b93.43e61c0a.f871f.1dc0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89298:1563ed4d99ac
Date: 2017-01-02 16:52 +0200
http://bitbucket.org/pypy/pypy/changeset/1563ed4d99ac/

Log:	lookup slot functions from tp_base if not in search_dict_w

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -275,25 +275,38 @@
         search_dict_w = None
 
     for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots:
+        slot_func_helper = None
         if search_dict_w is None:
             # built-in types: expose as many slots as possible, even
             # if it happens to come from some parent class
             slot_apifunc = None # use get_slot_tp_function 
         else:
-            # use the slot_apifunc (userslots) to lookup at runtime
-            pass
+            # For heaptypes, w_type.layout.typedef will be object's typedef, and
+            # get_slot_tp_function will fail
+            w_descr = search_dict_w.get(method_name, None)
+            if w_descr: 
+                # use the slot_apifunc (userslots) to lookup at runtime
+                pass
+            elif len(slot_names) ==1:
+                # 'inherit' from tp_base
+                slot_func_helper = getattr(pto.c_tp_base, slot_names[0])
+            else:
+                struct = getattr(pto.c_tp_base, slot_names[0])
+                if struct:
+                    slot_func_helper = getattr(struct, slot_names[1])
 
-        if typedef is not None:
-            if slot_apifunc is None:
-                slot_apifunc = get_slot_tp_function(space, typedef, slot_name)
-        if not slot_apifunc:
-            if not we_are_translated():
-                if slot_name not in missing_slots:
-                    missing_slots[slot_name] = w_type.getname(space)
-                    print "missing slot %r/%r, discovered on %r" % (
-                        method_name, slot_name, w_type.getname(space))
-            continue
-        slot_func_helper = slot_apifunc.get_llhelper(space)
+        if not slot_func_helper:
+            if typedef is not None:
+                if slot_apifunc is None:
+                    slot_apifunc = get_slot_tp_function(space, typedef, slot_name)
+            if not slot_apifunc:
+                if not we_are_translated():
+                    if slot_name not in missing_slots:
+                        missing_slots[slot_name] = w_type.getname(space)
+                        print "missing slot %r/%r, discovered on %r" % (
+                            method_name, slot_name, w_type.getname(space))
+                continue
+            slot_func_helper = slot_apifunc.get_llhelper(space)
 
         # XXX special case wrapper-functions and use a "specific" slot func
 
@@ -526,18 +539,18 @@
     # w_obj is an instance of w_A or one of its subclasses. So climb up the
     # inheritance chain until base.c_tp_dealloc is exactly this_func, and then
     # continue on up until they differ.
-    print 'subtype_dealloc, start from', rffi.charp2str(base.c_tp_name)
+    #print 'subtype_dealloc, start from', rffi.charp2str(base.c_tp_name)
     while base.c_tp_dealloc != this_func_ptr:
         base = base.c_tp_base
         assert base
-        print '                 ne move to', rffi.charp2str(base.c_tp_name)
+        #print '                 ne move to', rffi.charp2str(base.c_tp_name)
         w_obj = from_ref(space, rffi.cast(PyObject, base))
     while base.c_tp_dealloc == this_func_ptr:
         base = base.c_tp_base
         assert base
-        print '                 eq move to', rffi.charp2str(base.c_tp_name)
+        #print '                 eq move to', rffi.charp2str(base.c_tp_name)
         w_obj = from_ref(space, rffi.cast(PyObject, base))
-    print '                   end with', rffi.charp2str(base.c_tp_name)
+    #print '                   end with', rffi.charp2str(base.c_tp_name)
     dealloc = base.c_tp_dealloc
     # XXX call tp_del if necessary
     generic_cpy_call(space, dealloc, obj)
@@ -718,13 +731,6 @@
 
     typedescr = get_typedescr(w_type.layout.typedef)
 
-    # dealloc
-    if space.gettypeobject(w_type.layout.typedef) is w_type:
-        # only for the exact type, like 'space.w_tuple' or 'space.w_list'
-        pto.c_tp_dealloc = typedescr.get_dealloc().get_llhelper(space)
-    else:
-        # for all subtypes, use subtype_dealloc()
-        pto.c_tp_dealloc = llslot(space, subtype_dealloc)
     if space.is_w(w_type, space.w_str):
         pto.c_tp_itemsize = 1
     elif space.is_w(w_type, space.w_tuple):
@@ -755,6 +761,17 @@
     w_base = best_base(space, w_type.bases_w)
     pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base))
 
+    # dealloc
+    if space.gettypeobject(w_type.layout.typedef) is w_type:
+        # only for the exact type, like 'space.w_tuple' or 'space.w_list'
+        pto.c_tp_dealloc = typedescr.get_dealloc().get_llhelper(space)
+    else:
+        # for all subtypes, use base's dealloc (requires sorting in attach_all)
+        pto.c_tp_dealloc = pto.c_tp_base.c_tp_dealloc
+        if not pto.c_tp_dealloc:
+            # strange, but happens (ABCMeta)
+            pto.c_tp_dealloc = llslot(space, subtype_dealloc)
+
     if builder.cpyext_type_init is not None:
         builder.cpyext_type_init.append((pto, w_type))
     else:

From pypy.commits at gmail.com  Mon Jan  2 10:20:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 07:20:55 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix the out-of-range cases of
 socket.ntohl() & co., as an
Message-ID: <586a6fd7.88711c0a.281e6.16f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89300:f934a882b7ba
Date: 2017-01-02 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/f934a882b7ba/

Log:	Fix the out-of-range cases of socket.ntohl() & co., as an attempt to
	pass the stricter tests of py3.5

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -1,6 +1,6 @@
 from rpython.rlib import rsocket
 from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, r_longlong, r_uint32
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -164,40 +164,58 @@
         space.wrap(W_Socket(space, sock2))
     ])
 
-# The following 4 functions refuse all negative numbers, like CPython 2.6.
-# They could also check that the argument is not too large, but CPython 2.6
-# is not doing that consistently.
- at unwrap_spec(x="c_uint")
+# The following 4 functions refuse all negative numbers.
+# They also check that the argument is not too large, but note that
+# CPython 2.7 is not doing that consistently (CPython 3.x does).
+LONGLONG_UINT32_MAX = r_longlong(2**32-1)
+
+ at unwrap_spec(x="c_int")
 def ntohs(space, x):
     """ntohs(integer) -> integer
 
     Convert a 16-bit integer from network to host byte order.
     """
+    if x < 0:
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
     return space.wrap(rsocket.ntohs(intmask(x)))
 
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x=r_longlong)
 def ntohl(space, x):
     """ntohl(integer) -> integer
 
     Convert a 32-bit integer from network to host byte order.
     """
-    return space.wrap(rsocket.ntohl(x))
+    if x < r_longlong(0):
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
+    if x > LONGLONG_UINT32_MAX:
+        raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
+    return space.wrap(rsocket.ntohl(r_uint32(x)))
 
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x="c_int")
 def htons(space, x):
     """htons(integer) -> integer
 
     Convert a 16-bit integer from host to network byte order.
     """
-    return space.wrap(rsocket.htons(intmask(x)))
+    if x < 0:
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
+    return space.wrap(rsocket.htons(x))
 
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x=r_longlong)
 def htonl(space, x):
     """htonl(integer) -> integer
 
     Convert a 32-bit integer from host to network byte order.
     """
-    return space.wrap(rsocket.htonl(x))
+    if x < r_longlong(0):
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
+    if x > LONGLONG_UINT32_MAX:
+        raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
+    return space.wrap(rsocket.htonl(r_uint32(x)))
 
 @unwrap_spec(ip=str)
 def inet_aton(space, ip):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -320,9 +320,10 @@
         assert _socket.socket.__name__ == 'socket'
         assert _socket.socket.__module__ == '_socket'
 
-    def test_getservbyport(self):
+    def test_overflow_errors(self):
         import _socket
         raises(OverflowError, _socket.getservbyport, -1)
+        raises(OverflowError, _socket.getservbyport, 65536)
 
     def test_ntoa_exception(self):
         import _socket
@@ -518,44 +519,25 @@
     def test_NtoH(self):
         import sys
         import _socket as socket
-        # This just checks that htons etc. are their own inverse,
-        # when looking at the lower 16 or 32 bits.
+        # This checks that htons etc. are their own inverse,
+        # when looking at the lower 16 or 32 bits.  It also
+        # checks that we get OverflowErrors when calling with -1,
+        # or (for XtoXl()) with too large values.  For XtoXs()
+        # large values are silently truncated instead, like CPython.
         sizes = {socket.htonl: 32, socket.ntohl: 32,
                  socket.htons: 16, socket.ntohs: 16}
         for func, size in sizes.items():
             mask = (1 << size) - 1
-            for i in (0, 1, 0xffff, ~0xffff, 2, 0x01234567, 0x76543210):
+            for i in (0, 1, 0xffff, 0xffff0000, 2, 0x01234567, 0x76543210):
                 assert i & mask == func(func(i&mask)) & mask
 
             swapped = func(mask)
             assert swapped & mask == mask
-            try:
-                func(-1)
-            except (OverflowError, ValueError):
-                pass
-            else:
-                assert False
-            try:
-                func(sys.maxint*2+2)
-            except OverflowError:
-                pass
-            else:
-                assert False
-
-    def test_NtoH_overflow(self):
-        skip("we are not checking for overflowing values yet")
-        import _socket as socket
-        # Checks that we cannot give too large values to htons etc.
-        # Skipped for now; CPython 2.6 is also not consistent.
-        sizes = {socket.htonl: 32, socket.ntohl: 32,
-                 socket.htons: 16, socket.ntohs: 16}
-        for func, size in sizes.items():
-            try:
-                func(1 << size)
-            except OverflowError:
-                pass
-            else:
-                assert False
+            raises(OverflowError, func, -1)
+            raises(OverflowError, func, -1L)
+            if size > 16:    # else, values too large are ignored
+                raises(OverflowError, func, 2 ** size)
+                raises(OverflowError, func, 2L ** size)
 
     def test_newsocket(self):
         import socket

From pypy.commits at gmail.com  Mon Jan  2 10:49:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 07:49:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: gethostbyname(): support for the 'idna'
 encoding
Message-ID: <586a769f.cb911c0a.66b4.e157@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89301:4fbab31451c0
Date: 2017-01-02 16:49 +0100
http://bitbucket.org/pypy/pypy/changeset/4fbab31451c0/

Log:	gethostbyname(): support for the 'idna' encoding

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -21,12 +21,16 @@
         raise converted_error(space, e)
     return space.fsdecode(space.newbytes(res))
 
- at unwrap_spec(host=str)
-def gethostbyname(space, host):
+def encode_idna(space, w_host):
+    return space.bytes_w(space.call_method(space.w_unicode, 'encode',
+                                           w_host, space.wrap('idna')))
+
+def gethostbyname(space, w_host):
     """gethostbyname(host) -> address
 
     Return the IP address (a string of the form '255.255.255.255') for a host.
     """
+    host = encode_idna(space, w_host)
     try:
         addr = rsocket.gethostbyname(host)
         ip = addr.get_host()
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import sys, os
 import pytest
 from pypy.tool.pytest.objspace import gettestobjspace
@@ -681,6 +682,11 @@
         s1.close()
         s2.close()
 
+    def test_gethostbyname_unicode(self):
+        import _socket
+        domain = u"&#1080;&#1089;&#1087;&#1099;&#1090;&#1072;&#1085;&#1080;&#1077;.pythontest.net"
+        _socket.gethostbyname(domain)
+
 
 class AppTestNetlink:
     def setup_class(cls):

From pypy.commits at gmail.com  Mon Jan  2 10:51:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 07:51:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: gethostbyname_ex(): support for the
 'idna' encoding
Message-ID: <586a7711.52301c0a.a3729.25f3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89302:724153585911
Date: 2017-01-02 16:51 +0100
http://bitbucket.org/pypy/pypy/changeset/724153585911/

Log:	gethostbyname_ex(): support for the 'idna' encoding

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -45,13 +45,13 @@
                            space.newlist(aliases),
                            space.newlist(address_list)])
 
- at unwrap_spec(host=str)
-def gethostbyname_ex(space, host):
+def gethostbyname_ex(space, w_host):
     """gethostbyname_ex(host) -> (name, aliaslist, addresslist)
 
     Return the true host name, a list of aliases, and a list of IP addresses,
     for a host.  The host argument is a string giving a host name or IP number.
     """
+    host = encode_idna(space, w_host)
     try:
         res = rsocket.gethostbyname_ex(host)
     except SocketError as e:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -686,6 +686,7 @@
         import _socket
         domain = u"&#1080;&#1089;&#1087;&#1099;&#1090;&#1072;&#1085;&#1080;&#1077;.pythontest.net"
         _socket.gethostbyname(domain)
+        _socket.gethostbyname_ex(domain)
 
 
 class AppTestNetlink:

From pypy.commits at gmail.com  Mon Jan  2 10:59:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 07:59:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: getaddrinfo() already supports the idna
 encoding, although it was not tested
Message-ID: <586a78f5.05371c0a.31ce.2691@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89303:7e9cf53b2273
Date: 2017-01-02 16:59 +0100
http://bitbucket.org/pypy/pypy/changeset/7e9cf53b2273/

Log:	getaddrinfo() already supports the idna encoding, although it was
	not tested

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -687,6 +687,7 @@
         domain = u"&#1080;&#1089;&#1087;&#1099;&#1090;&#1072;&#1085;&#1080;&#1077;.pythontest.net"
         _socket.gethostbyname(domain)
         _socket.gethostbyname_ex(domain)
+        _socket.getaddrinfo(domain, 0, _socket.AF_UNSPEC, _socket.SOCK_STREAM)
 
 
 class AppTestNetlink:

From pypy.commits at gmail.com  Mon Jan  2 11:04:02 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 08:04:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: call 'unicode.encode(x,
 "idna")' instead of 'x.encode("idna")', with comment
Message-ID: <586a79f2.876ec20a.e50e2.9661@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89304:8da39ae6ab57
Date: 2017-01-02 17:03 +0100
http://bitbucket.org/pypy/pypy/changeset/8da39ae6ab57/

Log:	call 'unicode.encode(x, "idna")' instead of 'x.encode("idna")', with
	comment

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -22,6 +22,8 @@
     return space.fsdecode(space.newbytes(res))
 
 def encode_idna(space, w_host):
+    # call unicode.encode(host, 'idna'), and not host.encode('idna') in
+    # case type(host) is not unicode
     return space.bytes_w(space.call_method(space.w_unicode, 'encode',
                                            w_host, space.wrap('idna')))
 
@@ -276,8 +278,7 @@
     elif space.isinstance_w(w_host, space.w_bytes):
         host = space.bytes_w(w_host)
     elif space.isinstance_w(w_host, space.w_unicode):
-        w_shost = space.call_method(w_host, "encode", space.wrap("idna"))
-        host = space.bytes_w(w_shost)
+        host = encode_idna(space, w_host)
     else:
         raise oefmt(space.w_TypeError,
                     "getaddrinfo() argument 1 must be string or None")

From pypy.commits at gmail.com  Mon Jan  2 11:23:24 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 02 Jan 2017 08:23:24 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: fix
Message-ID: <586a7e7c.cf3fc20a.aa7e8.9aed@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89305:e2ecb4d3858d
Date: 2017-01-02 16:22 +0000
http://bitbucket.org/pypy/pypy/changeset/e2ecb4d3858d/

Log:	fix

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -526,6 +526,8 @@
     'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor',
     'PyCapsule_SetContext', 'PyCapsule_Import', 'PyCapsule_Type', '_Py_get_capsule_type',
 
+    'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
+
     'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
 
     'PyOS_getsig', 'PyOS_setsig',
diff --git a/pypy/module/cpyext/src/complexobject.c b/pypy/module/cpyext/src/complexobject.c
--- a/pypy/module/cpyext/src/complexobject.c
+++ b/pypy/module/cpyext/src/complexobject.c
@@ -4,7 +4,6 @@
 Py_complex
 PyComplex_AsCComplex(PyObject *obj)
 {
-    printf("asdfgg");
     Py_complex result;
     _PyComplex_AsCComplex(obj, &result);
     return result;

From pypy.commits at gmail.com  Mon Jan  2 11:54:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 08:54:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: idna encoding for gethostbyaddr(). Like
 CPython, hard to test
Message-ID: <586a85c0.e6b0c20a.5560f.caae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89306:a61e1b437115
Date: 2017-01-02 17:13 +0100
http://bitbucket.org/pypy/pypy/changeset/a61e1b437115/

Log:	idna encoding for gethostbyaddr(). Like CPython, hard to test

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -60,13 +60,13 @@
         raise converted_error(space, e)
     return common_wrapgethost(space, res)
 
- at unwrap_spec(host=str)
-def gethostbyaddr(space, host):
+def gethostbyaddr(space, w_host):
     """gethostbyaddr(host) -> (name, aliaslist, addresslist)
 
     Return the true host name, a list of aliases, and a list of IP addresses,
     for a host.  The host argument is a string giving a host name or IP number.
     """
+    host = encode_idna(space, w_host)
     try:
         res = rsocket.gethostbyaddr(host)
     except SocketError as e:

From pypy.commits at gmail.com  Mon Jan  2 11:54:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 08:54:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Starting to support idna hostnames in the
 socket methods
Message-ID: <586a85c2.08301c0a.65616.224c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89307:9705352d5190
Date: 2017-01-02 17:53 +0100
http://bitbucket.org/pypy/pypy/changeset/9705352d5190/

Log:	Starting to support idna hostnames in the socket methods

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -22,8 +22,8 @@
     return space.fsdecode(space.newbytes(res))
 
 def encode_idna(space, w_host):
-    # call unicode.encode(host, 'idna'), and not host.encode('idna') in
-    # case type(host) is not unicode
+    # call unicode.encode(host, 'idna'), and not host.encode('idna') in case
+    # type(host) is not unicode.  See also interp_socket.idna_converter()
     return space.bytes_w(space.call_method(space.w_unicode, 'encode',
                                            w_host, space.wrap('idna')))
 
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -85,11 +85,35 @@
     else:
         raise NotImplementedError
 
+def idna_converter(space, w_host):
+    # Converts w_host to a byte string.  Similar to encode_idna()
+    # but accepts more types and refuses NULL bytes.
+    if space.isinstance_w(w_host, space.w_unicode):
+        try:
+            w_s = space.encode_unicode_object(w_host, 'ascii', None)
+        except OperationError as e:
+            if not e.match(space, space.w_UnicodeEncodeError):
+                raise
+            w_s = space.encode_unicode_object(w_host, 'idna', None)
+        s = space.bytes_w(w_s)
+    elif space.isinstance_w(w_host, space.w_bytes):
+        s = space.bytes_w(w_host)
+    elif space.isinstance_w(w_host, space.w_bytearray):
+        s = space.charbuf_w(w_host)
+    else:
+        raise oefmt(space.w_TypeError,
+                    "string or unicode text buffer expected, not %T", w_host)
+    if '\x00' in s:
+        raise oefmt(space.w_TypeError,
+                    "host name must not contain null character")
+    return s
+
+
 # XXX Hack to seperate rpython and pypy
 def addr_from_object(family, fd, space, w_address):
     if family == rsocket.AF_INET:
         w_host, w_port = space.unpackiterable(w_address, 2)
-        host = space.str_w(w_host)
+        host = idna_converter(space, w_host)
         port = space.int_w(w_port)
         port = make_ushort_port(space, port)
         return rsocket.INETAddress(host, port)
@@ -99,7 +123,7 @@
             raise oefmt(space.w_TypeError,
                         "AF_INET6 address must be a tuple of length 2 "
                         "to 4, not %d", len(pieces_w))
-        host = space.str_w(pieces_w[0])
+        host = idna_converter(space, pieces_w[0])
         port = space.int_w(pieces_w[1])
         port = make_ushort_port(space, port)
         if len(pieces_w) > 2: flowinfo = space.int_w(pieces_w[2])
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -428,7 +428,8 @@
     def test_socket_connect_typeerrors(self):
         tests = [
             "",
-            ("80"),
+            "80",
+            ("80",),
             ("80", "80"),
             (80, 80),
         ]
@@ -682,12 +683,16 @@
         s1.close()
         s2.close()
 
-    def test_gethostbyname_unicode(self):
+    def test_hostname_unicode(self):
         import _socket
         domain = u"&#1080;&#1089;&#1087;&#1099;&#1090;&#1072;&#1085;&#1080;&#1077;.pythontest.net"
         _socket.gethostbyname(domain)
         _socket.gethostbyname_ex(domain)
         _socket.getaddrinfo(domain, 0, _socket.AF_UNSPEC, _socket.SOCK_STREAM)
+        s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
+        s.connect((domain, 80))
+        s.close()
+        raises(TypeError, s.connect, (domain + '\x00', 80))
 
 
 class AppTestNetlink:

From pypy.commits at gmail.com  Mon Jan  2 12:01:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 09:01:16 -0800 (PST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <586a875c.4dd41c0a.c02f0.37d4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89308:aa9ae054b625
Date: 2017-01-02 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/aa9ae054b625/

Log:	fix test

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -495,7 +495,8 @@
     def test_socket_connect_typeerrors(self):
         tests = [
             "",
-            ("80"),
+            "80",
+            ("80",),
             ("80", "80"),
             (80, 80),
         ]

From pypy.commits at gmail.com  Mon Jan  2 12:01:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 02 Jan 2017 09:01:26 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Put the defines for functions
 only in decl files, not in pypy_macros.h.
Message-ID: <586a8766.e6b0c20a.5560f.cdd9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89309:4f26ce693579
Date: 2017-01-02 17:00 +0000
http://bitbucket.org/pypy/pypy/changeset/4f26ce693579/

Log:	Put the defines for functions only in decl files, not in
	pypy_macros.h.

	NB: 'header == DEFAULT_HEADER' was never true, so the function
	defines in pypy_macros.h were already also in pypy_decl.h.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1220,7 +1220,7 @@
 def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
-    export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
+    export_symbols = sorted(SYMBOLS_C) + sorted(GLOBALS)
     for name in export_symbols:
         if '#' in name:
             name, header = name.split('#')
@@ -1272,13 +1272,9 @@
         for name, func in sorted(header_functions.iteritems()):
             if not func:
                 continue
-            if header == DEFAULT_HEADER:
-                _name = name
-            else:
-                # this name is not included in pypy_macros.h
-                _name = mangle_name(prefix, name)
-                assert _name is not None, 'error converting %s' % name
-                header.append("#define %s %s" % (name, _name))
+            _name = mangle_name(prefix, name)
+            assert _name is not None, 'error converting %s' % name
+            header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
             header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, _name, args))
             if api_struct:
@@ -1476,7 +1472,7 @@
         for name, func in header_functions.iteritems():
             if not func:
                 continue
-            newname = mangle_name('PyPy', name) or name
+            newname = mangle_name(prefix, name) or name
             deco = entrypoint_lowlevel("cpyext", func.argtypes, newname,
                                         relax=True)
             deco(func.get_wrapper(space))

From pypy.commits at gmail.com  Mon Jan  2 12:09:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 09:09:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: XXX fix me later
Message-ID: <586a8937.2738c20a.6d2db.e15b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89310:850d383f1b23
Date: 2017-01-02 18:08 +0100
http://bitbucket.org/pypy/pypy/changeset/850d383f1b23/

Log:	XXX fix me later

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -686,6 +686,8 @@
     def test_hostname_unicode(self):
         import _socket
         domain = u"&#1080;&#1089;&#1087;&#1099;&#1090;&#1072;&#1085;&#1080;&#1077;.pythontest.net"
+        # XXX figure out why the idna encoding is sometimes missing in
+        # tests, notably if we run all tests instead of just this one
         _socket.gethostbyname(domain)
         _socket.gethostbyname_ex(domain)
         _socket.getaddrinfo(domain, 0, _socket.AF_UNSPEC, _socket.SOCK_STREAM)

From pypy.commits at gmail.com  Mon Jan  2 12:11:49 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 02 Jan 2017 09:11:49 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Remove duplication between
 FUNCTIONS and FUNCTIONS_BY_HEADER
Message-ID: <586a89d5.e626c20a.f471c.cb9c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89311:3b17e46d5898
Date: 2017-01-02 17:11 +0000
http://bitbucket.org/pypy/pypy/changeset/3b17e46d5898/

Log:	Remove duplication between FUNCTIONS and FUNCTIONS_BY_HEADER

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1,5 +1,6 @@
 import ctypes
 import sys, os
+from collections import defaultdict
 
 import py
 
@@ -364,8 +365,8 @@
         func_name = func.func_name
         if header is not None:
             c_name = None
-            assert func_name not in FUNCTIONS, (
-                "%s already registered" % func_name)
+            if func_name in FUNCTIONS_BY_HEADER[header]:
+                raise ValueError("%s already registered" % func_name)
         else:
             c_name = func_name
         api_function = ApiFunction(argtypes, restype, func, error,
@@ -463,9 +464,7 @@
             return res
 
         if header is not None:
-            if header == DEFAULT_HEADER:
-                FUNCTIONS[func_name] = api_function
-            FUNCTIONS_BY_HEADER.setdefault(header, {})[func_name] = api_function
+            FUNCTIONS_BY_HEADER[header][func_name] = api_function
         INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
         return unwrapper  # used in 'normal' RPython code.
     return decorate
@@ -489,8 +488,7 @@
     GLOBALS[name] = (typ, expr)
 
 INTERPLEVEL_API = {}
-FUNCTIONS = {}
-FUNCTIONS_BY_HEADER = {}
+FUNCTIONS_BY_HEADER = defaultdict(dict)
 
 # These are C symbols which cpyext will export, but which are defined in .c
 # files somewhere in the implementation of cpyext (rather than being defined in
@@ -1404,7 +1402,7 @@
     from pypy.module.cpyext.ndarrayobject import HEADER
     global FUNCTIONS_BY_HEADER, separate_module_files
     for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
-        FUNCTIONS_BY_HEADER.setdefault(HEADER, {})[func_name] = None
+        FUNCTIONS_BY_HEADER[HEADER][func_name] = None
     register_global("PyArray_Type",
         'PyTypeObject*',  "space.gettypeobject(W_NDimArray.typedef)",
         header=HEADER)

From pypy.commits at gmail.com  Mon Jan  2 13:44:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 10:44:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <586a9fa1.c11d1c0a.5313b.b681@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89312:020911d610d9
Date: 2017-01-02 19:42 +0100
http://bitbucket.org/pypy/pypy/changeset/020911d610d9/

Log:	hg merge default

diff too long, truncating to 2000 out of 4555 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
diff --git a/ctypes_configure/__init__.py b/ctypes_configure/__init__.py
deleted file mode 100644
diff --git a/ctypes_configure/cbuild.py b/ctypes_configure/cbuild.py
deleted file mode 100644
--- a/ctypes_configure/cbuild.py
+++ /dev/null
@@ -1,456 +0,0 @@
-
-import os, sys, inspect, re, imp, py
-from ctypes_configure import stdoutcapture
-import distutils
-
-debug = 0
-
-configdir = py.path.local.make_numbered_dir(prefix='ctypes_configure-')
-
-class ExternalCompilationInfo(object):
-
-    _ATTRIBUTES = ['pre_include_lines', 'includes', 'include_dirs',
-                   'post_include_lines', 'libraries', 'library_dirs',
-                   'separate_module_sources', 'separate_module_files']
-    _AVOID_DUPLICATES = ['separate_module_files', 'libraries', 'includes',
-                         'include_dirs', 'library_dirs', 'separate_module_sources']
-
-    def __init__(self,
-                 pre_include_lines       = [],
-                 includes                = [],
-                 include_dirs            = [],
-                 post_include_lines      = [],
-                 libraries               = [],
-                 library_dirs            = [],
-                 separate_module_sources = [],
-                 separate_module_files   = []):
-        """
-        pre_include_lines: list of lines that should be put at the top
-        of the generated .c files, before any #include.  They shouldn't
-        contain an #include themselves.
-
-        includes: list of .h file names to be #include'd from the
-        generated .c files.
-
-        include_dirs: list of dir names that is passed to the C compiler
-
-        post_include_lines: list of lines that should be put at the top
-        of the generated .c files, after the #includes.
-
-        libraries: list of library names that is passed to the linker
-
-        library_dirs: list of dir names that is passed to the linker
-
-        separate_module_sources: list of multiline strings that are
-        each written to a .c file and compiled separately and linked
-        later on.  (If function prototypes are needed for other .c files
-        to access this, they can be put in post_include_lines.)
-
-        separate_module_files: list of .c file names that are compiled
-        separately and linked later on.  (If an .h file is needed for
-        other .c files to access this, it can be put in includes.)
-        """
-        for name in self._ATTRIBUTES:
-            value = locals()[name]
-            assert isinstance(value, (list, tuple))
-            setattr(self, name, tuple(value))
-
-    def _value(self):
-        return tuple([getattr(self, x) for x in self._ATTRIBUTES])
-
-    def __hash__(self):
-        return hash(self._value())
-
-    def __eq__(self, other):
-        return self.__class__ is other.__class__ and \
-               self._value() == other._value()
-
-    def __ne__(self, other):
-        return not self == other
-
-    def __repr__(self):
-        info = []
-        for attr in self._ATTRIBUTES:
-            val = getattr(self, attr)
-            info.append("%s=%s" % (attr, repr(val)))
-        return "<ExternalCompilationInfo (%s)>" % ", ".join(info)
-
-    def merge(self, *others):
-        others = list(others)
-        attrs = {}
-        for name in self._ATTRIBUTES:
-            if name not in self._AVOID_DUPLICATES:
-                s = []
-                for i in [self] + others:
-                    s += getattr(i, name)
-                attrs[name] = s
-            else:
-                s = set()
-                attr = []
-                for one in [self] + others:
-                    for elem in getattr(one, name):
-                        if elem not in s:
-                            s.add(elem)
-                            attr.append(elem)
-                attrs[name] = attr
-        return ExternalCompilationInfo(**attrs)
-
-    def write_c_header(self, fileobj):
-        for line in self.pre_include_lines:
-            print >> fileobj, line
-        for path in self.includes:
-            print >> fileobj, '#include <%s>' % (path,)
-        for line in self.post_include_lines:
-            print >> fileobj, line
-
-    def _copy_attributes(self):
-        d = {}
-        for attr in self._ATTRIBUTES:
-            d[attr] = getattr(self, attr)
-        return d
-
-    def convert_sources_to_files(self, cache_dir=None, being_main=False):
-        if not self.separate_module_sources:
-            return self
-        if cache_dir is None:
-            cache_dir = configdir.join('module_cache').ensure(dir=1)
-        num = 0
-        files = []
-        for source in self.separate_module_sources:
-            while 1:
-                filename = cache_dir.join('module_%d.c' % num)
-                num += 1
-                if not filename.check():
-                    break
-            f = filename.open("w")
-            if being_main:
-                f.write("#define PYPY_NOT_MAIN_FILE\n")
-            self.write_c_header(f)
-            source = str(source)
-            f.write(source)
-            if not source.endswith('\n'):
-                f.write('\n')
-            f.close()
-            files.append(str(filename))
-        d = self._copy_attributes()
-        d['separate_module_sources'] = ()
-        d['separate_module_files'] += tuple(files)
-        return ExternalCompilationInfo(**d)
-
-    def compile_shared_lib(self):
-        self = self.convert_sources_to_files()
-        if not self.separate_module_files:
-            return self
-        lib = compile_c_module([], 'externmod', self)
-        d = self._copy_attributes()
-        d['libraries'] += (lib,)
-        d['separate_module_files'] = ()
-        d['separate_module_sources'] = ()
-        return ExternalCompilationInfo(**d)
-
-if sys.platform == 'win32':
-    so_ext = '.dll'
-else:
-    so_ext = '.so'
-
-def compiler_command():
-    # e.g. for tcc, you might set this to
-    #    "tcc -shared -o %s.so %s.c"
-    return os.getenv('PYPY_CC')
-
-def enable_fast_compilation():
-    if sys.platform == 'win32':
-        dash = '/'
-    else:
-        dash = '-'
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt:
-        opt = re.sub('%sO\d+' % dash, '%sO0' % dash, opt)
-    else:
-        opt = '%sO0' % dash
-    gcv['OPT'] = opt
-
-def ensure_correct_math():
-    if sys.platform != 'win32':
-        return # so far
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt and '/Op' not in opt:
-        opt += '/Op'
-    gcv['OPT'] = opt
-
-
-def try_compile(c_files, eci):
-    try:
-        build_executable(c_files, eci)
-        result = True
-    except (distutils.errors.CompileError,
-            distutils.errors.LinkError):
-        result = False
-    return result
-
-def compile_c_module(cfiles, modbasename, eci, tmpdir=None):
-    #try:
-    #    from distutils.log import set_threshold
-    #    set_threshold(10000)
-    #except ImportError:
-    #    print "ERROR IMPORTING"
-    #    pass
-    cfiles = [py.path.local(f) for f in cfiles]
-    if tmpdir is None:
-        tmpdir = configdir.join("module_cache").ensure(dir=1)
-    num = 0
-    cfiles += eci.separate_module_files
-    include_dirs = list(eci.include_dirs)
-    library_dirs = list(eci.library_dirs)
-    if (sys.platform == 'darwin' or    # support Fink & Darwinports
-            sys.platform.startswith('freebsd')):
-        for s in ('/sw/', '/opt/local/', '/usr/local/'):
-            if s + 'include' not in include_dirs and \
-               os.path.exists(s + 'include'):
-                include_dirs.append(s + 'include')
-            if s + 'lib' not in library_dirs and \
-               os.path.exists(s + 'lib'):
-                library_dirs.append(s + 'lib')
-
-    num = 0
-    modname = modbasename
-    while 1:
-        if not tmpdir.join(modname + so_ext).check():
-            break
-        num += 1
-        modname = '%s_%d' % (modbasename, num)
-
-    lastdir = tmpdir.chdir()
-    libraries = eci.libraries
-    ensure_correct_math()
-    try:
-        if debug: print "modname", modname
-        c = stdoutcapture.Capture(mixed_out_err = True)
-        try:
-            try:
-                if compiler_command():
-                    # GCC-ish options only
-                    from distutils import sysconfig
-                    gcv = sysconfig.get_config_vars()
-                    cmd = compiler_command().replace('%s',
-                                                     str(tmpdir.join(modname)))
-                    for dir in [gcv['INCLUDEPY']] + list(include_dirs):
-                        cmd += ' -I%s' % dir
-                    for dir in library_dirs:
-                        cmd += ' -L%s' % dir
-                    os.system(cmd)
-                else:
-                    from distutils.dist import Distribution
-                    from distutils.extension import Extension
-                    from distutils.ccompiler import get_default_compiler
-                    saved_environ = os.environ.items()
-                    try:
-                        # distutils.core.setup() is really meant for end-user
-                        # interactive usage, because it eats most exceptions and
-                        # turn them into SystemExits.  Instead, we directly
-                        # instantiate a Distribution, which also allows us to
-                        # ignore unwanted features like config files.
-                        extra_compile_args = []
-                        # ensure correct math on windows
-                        if sys.platform == 'win32':
-                            extra_compile_args.append('/Op') # get extra precision
-                        if get_default_compiler() == 'unix':
-                            old_version = False
-                            try:
-                                g = os.popen('gcc --version', 'r')
-                                verinfo = g.read()
-                                g.close()
-                            except (OSError, IOError):
-                                pass
-                            else:
-                                old_version = verinfo.startswith('2')
-                            if not old_version:
-                                extra_compile_args.extend(["-Wno-unused-label",
-                                                        "-Wno-unused-variable"])
-                        attrs = {
-                            'name': "testmodule",
-                            'ext_modules': [
-                                Extension(modname, [str(cfile) for cfile in cfiles],
-                                    include_dirs=include_dirs,
-                                    library_dirs=library_dirs,
-                                    extra_compile_args=extra_compile_args,
-                                    libraries=list(libraries),)
-                                ],
-                            'script_name': 'setup.py',
-                            'script_args': ['-q', 'build_ext', '--inplace', '--force'],
-                            }
-                        dist = Distribution(attrs)
-                        if not dist.parse_command_line():
-                            raise ValueError, "distutils cmdline parse error"
-                        dist.run_commands()
-                    finally:
-                        for key, value in saved_environ:
-                            if os.environ.get(key) != value:
-                                os.environ[key] = value
-            finally:
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = open("%s.errors" % modname, "w")
-                    fdump.write(data)
-                    fdump.close()
-            # XXX do we need to do some check on fout/ferr?
-            # XXX not a nice way to import a module
-        except:
-            print >>sys.stderr, data
-            raise
-    finally:
-        lastdir.chdir()
-    return str(tmpdir.join(modname) + so_ext)
-
-def make_module_from_c(cfile, eci):
-    cfile = py.path.local(cfile)
-    modname = cfile.purebasename
-    compile_c_module([cfile], modname, eci)
-    return import_module_from_directory(cfile.dirpath(), modname)
-
-def import_module_from_directory(dir, modname):
-    file, pathname, description = imp.find_module(modname, [str(dir)])
-    try:
-        mod = imp.load_module(modname, file, pathname, description)
-    finally:
-        if file:
-            file.close()
-    return mod
-
-
-def log_spawned_cmd(spawn):
-    def spawn_and_log(cmd, *args, **kwds):
-        if debug:
-            print ' '.join(cmd)
-        return spawn(cmd, *args, **kwds)
-    return spawn_and_log
-
-
-class ProfOpt(object):
-    #XXX assuming gcc style flags for now
-    name = "profopt"
-    
-    def __init__(self, compiler):
-        self.compiler = compiler
-
-    def first(self):
-        self.build('-fprofile-generate')
-
-    def probe(self, exe, args):
-        # 'args' is a single string typically containing spaces
-        # and quotes, which represents several arguments.
-        os.system("'%s' %s" % (exe, args))
-
-    def after(self):
-        self.build('-fprofile-use')
-
-    def build(self, option):
-        compiler = self.compiler
-        compiler.compile_extra.append(option)
-        compiler.link_extra.append(option)
-        try:
-            compiler._build()
-        finally:
-            compiler.compile_extra.pop()
-            compiler.link_extra.pop()
-            
-class CCompiler:
-
-    def __init__(self, cfilenames, eci, outputfilename=None,
-                 compiler_exe=None, profbased=None):
-        self.cfilenames = cfilenames
-        ext = ''
-        self.compile_extra = []
-        self.link_extra = []
-        self.libraries = list(eci.libraries)
-        self.include_dirs = list(eci.include_dirs)
-        self.library_dirs = list(eci.library_dirs)
-        self.compiler_exe = compiler_exe
-        self.profbased = profbased
-        if not sys.platform in ('win32', 'darwin', 'cygwin'): # xxx
-            if 'm' not in self.libraries:
-                self.libraries.append('m')
-            if 'pthread' not in self.libraries:
-                self.libraries.append('pthread')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer', '-pthread']
-            self.link_extra += ['-pthread']
-        if sys.platform == 'win32':
-            self.link_extra += ['/DEBUG'] # generate .pdb file
-        if (sys.platform == 'darwin' or    # support Fink & Darwinports
-                sys.platform.startswith('freebsd')):
-            for s in ('/sw/', '/opt/local/', '/usr/local/'):
-                if s + 'include' not in self.include_dirs and \
-                   os.path.exists(s + 'include'):
-                    self.include_dirs.append(s + 'include')
-                if s + 'lib' not in self.library_dirs and \
-                   os.path.exists(s + 'lib'):
-                    self.library_dirs.append(s + 'lib')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer']
-
-        if outputfilename is None:
-            self.outputfilename = py.path.local(cfilenames[0]).new(ext=ext)
-        else: 
-            self.outputfilename = py.path.local(outputfilename)
-
-    def build(self, noerr=False):
-        basename = self.outputfilename.new(ext='')
-        data = ''
-        try:
-            saved_environ = os.environ.copy()
-            c = stdoutcapture.Capture(mixed_out_err = True)
-            try:
-                self._build()
-            finally:
-                # workaround for a distutils bugs where some env vars can
-                # become longer and longer every time it is used
-                for key, value in saved_environ.items():
-                    if os.environ.get(key) != value:
-                        os.environ[key] = value
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = basename.new(ext='errors').open("w")
-                    fdump.write(data)
-                    fdump.close()
-        except:
-            if not noerr:
-                print >>sys.stderr, data
-            raise
- 
-    def _build(self):
-        from distutils.ccompiler import new_compiler 
-        compiler = new_compiler(force=1)
-        if self.compiler_exe is not None:
-            for c in '''compiler compiler_so compiler_cxx
-                        linker_exe linker_so'''.split():
-                compiler.executables[c][0] = self.compiler_exe
-        compiler.spawn = log_spawned_cmd(compiler.spawn)
-        objects = []
-        for cfile in self.cfilenames: 
-            cfile = py.path.local(cfile)
-            old = cfile.dirpath().chdir() 
-            try: 
-                res = compiler.compile([cfile.basename], 
-                                       include_dirs=self.include_dirs,
-                                       extra_preargs=self.compile_extra)
-                assert len(res) == 1
-                cobjfile = py.path.local(res[0]) 
-                assert cobjfile.check()
-                objects.append(str(cobjfile))
-            finally: 
-                old.chdir() 
-        compiler.link_executable(objects, str(self.outputfilename),
-                                 libraries=self.libraries,
-                                 extra_preargs=self.link_extra,
-                                 library_dirs=self.library_dirs)
-
-def build_executable(*args, **kwds):
-    noerr = kwds.pop('noerr', False)
-    compiler = CCompiler(*args, **kwds)
-    compiler.build(noerr=noerr)
-    return str(compiler.outputfilename)
diff --git a/ctypes_configure/configure.py b/ctypes_configure/configure.py
deleted file mode 100755
--- a/ctypes_configure/configure.py
+++ /dev/null
@@ -1,621 +0,0 @@
-#! /usr/bin/env python
-
-import os, py, sys
-import ctypes
-from ctypes_configure.cbuild import build_executable, configdir, try_compile
-from ctypes_configure.cbuild import ExternalCompilationInfo
-import distutils
-
-# ____________________________________________________________
-#
-# Helpers for simple cases
-
-def eci_from_header(c_header_source):
-    return ExternalCompilationInfo(
-        pre_include_lines=c_header_source.split("\n")
-    )
-
-
-def getstruct(name, c_header_source, interesting_fields):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        STRUCT = Struct(name, interesting_fields)
-    return configure(CConfig)['STRUCT']
-
-def getsimpletype(name, c_header_source, ctype_hint=ctypes.c_int):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        TYPE = SimpleType(name, ctype_hint)
-    return configure(CConfig)['TYPE']
-
-def getconstantinteger(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        CONST = ConstantInteger(name)
-    return configure(CConfig)['CONST']
-
-def getdefined(macro, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        DEFINED = Defined(macro)
-    return configure(CConfig)['DEFINED']
-
-def has(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        HAS = Has(name)
-    return configure(CConfig)['HAS']
-
-def check_eci(eci):
-    """Check if a given ExternalCompilationInfo compiles and links."""
-    class CConfig:
-        _compilation_info_ = eci
-        WORKS = Works()
-    return configure(CConfig)['WORKS']
-
-def sizeof(name, eci, **kwds):
-    class CConfig:
-        _compilation_info_ = eci
-        SIZE = SizeOf(name)
-    for k, v in kwds.items():
-        setattr(CConfig, k, v)
-    return configure(CConfig)['SIZE']
-
-def memory_alignment():
-    """Return the alignment (in bytes) of memory allocations.
-    This is enough to make sure a structure with pointers and 'double'
-    fields is properly aligned."""
-    global _memory_alignment
-    if _memory_alignment is None:
-        S = getstruct('struct memory_alignment_test', """
-           struct memory_alignment_test {
-               double d;
-               void* p;
-           };
-        """, [])
-        result = ctypes.alignment(S)
-        assert result & (result-1) == 0, "not a power of two??"
-        _memory_alignment = result
-    return _memory_alignment
-_memory_alignment = None
-
-# ____________________________________________________________
-#
-# General interface
-
-class ConfigResult:
-    def __init__(self, CConfig, info, entries):
-        self.CConfig = CConfig
-        self.result = {}
-        self.info = info
-        self.entries = entries
-        
-    def get_entry_result(self, entry):
-        try:
-            return self.result[entry]
-        except KeyError:
-            pass
-        name = self.entries[entry]
-        info = self.info[name]
-        self.result[entry] = entry.build_result(info, self)
-
-    def get_result(self):
-        return dict([(name, self.result[entry])
-                     for entry, name in self.entries.iteritems()])
-
-
-class _CWriter(object):
-    """ A simple class which aggregates config parts
-    """
-    def __init__(self, CConfig):
-        self.path = uniquefilepath()
-        self.f = self.path.open("w")
-        self.config = CConfig
-
-    def write_header(self):
-        f = self.f
-        CConfig = self.config
-        CConfig._compilation_info_.write_c_header(f)
-        print >> f, C_HEADER
-        print >> f
-
-    def write_entry(self, key, entry):
-        f = self.f
-        print >> f, 'void dump_section_%s(void) {' % (key,)
-        for line in entry.prepare_code():
-            if line and line[0] != '#':
-                line = '\t' + line
-            print >> f, line
-        print >> f, '}'
-        print >> f
-
-    def write_entry_main(self, key):
-        print >> self.f, '\tprintf("-+- %s\\n");' % (key,)
-        print >> self.f, '\tdump_section_%s();' % (key,)
-        print >> self.f, '\tprintf("---\\n");'
-
-    def start_main(self):
-        print >> self.f, 'int main(int argc, char *argv[]) {'
-
-    def close(self):
-        f = self.f
-        print >> f, '\treturn 0;'
-        print >> f, '}'
-        f.close()
-
-    def ask_gcc(self, question):
-        self.start_main()
-        self.f.write(question + "\n")
-        self.close()
-        eci = self.config._compilation_info_
-        return try_compile([self.path], eci)
-
-        
-def configure(CConfig, noerr=False):
-    """Examine the local system by running the C compiler.
-    The CConfig class contains CConfigEntry attribues that describe
-    what should be inspected; configure() returns a dict mapping
-    names to the results.
-    """
-    for attr in ['_includes_', '_libraries_', '_sources_', '_library_dirs_',
-                 '_include_dirs_', '_header_']:
-        assert not hasattr(CConfig, attr), "Found legacy attribut %s on CConfig" % (attr,)
-    entries = []
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigEntry):
-            entries.append((key, value))
-
-    if entries:   # can be empty if there are only CConfigSingleEntries
-        writer = _CWriter(CConfig)
-        writer.write_header()
-        for key, entry in entries:
-            writer.write_entry(key, entry)
-
-        f = writer.f
-        writer.start_main()
-        for key, entry in entries:
-            writer.write_entry_main(key)
-        writer.close()
-
-        eci = CConfig._compilation_info_
-        infolist = list(run_example_code(writer.path, eci, noerr=noerr))
-        assert len(infolist) == len(entries)
-
-        resultinfo = {}
-        resultentries = {}
-        for info, (key, entry) in zip(infolist, entries):
-            resultinfo[key] = info
-            resultentries[entry] = key
-
-        result = ConfigResult(CConfig, resultinfo, resultentries)
-        for name, entry in entries:
-            result.get_entry_result(entry)
-        res = result.get_result()
-    else:
-        res = {}
-
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigSingleEntry):
-            writer = _CWriter(CConfig)
-            writer.write_header()
-            res[key] = value.question(writer.ask_gcc)
-    return res
-
-# ____________________________________________________________
-
-
-class CConfigEntry(object):
-    "Abstract base class."
-
-class Struct(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined structure.
-    """
-    def __init__(self, name, interesting_fields, ifdef=None):
-        self.name = name
-        self.interesting_fields = interesting_fields
-        self.ifdef = ifdef
-
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield 'typedef struct {'
-        yield '    char c;'
-        yield '    ctypesplatcheck_t s;'
-        yield '} ctypesplatcheck2_t;'
-        yield ''
-        yield 'ctypesplatcheck_t s;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("align", offsetof(ctypesplatcheck2_t, s));'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        for fieldname, fieldtype in self.interesting_fields:
-            yield 'dump("fldofs %s", offsetof(ctypesplatcheck_t, %s));'%(
-                fieldname, fieldname)
-            yield 'dump("fldsize %s",   sizeof(s.%s));' % (
-                fieldname, fieldname)
-            if fieldtype in integer_class:
-                yield 's.%s = 0; s.%s = ~s.%s;' % (fieldname,
-                                                   fieldname,
-                                                   fieldname)
-                yield 'dump("fldunsigned %s", s.%s > 0);' % (fieldname,
-                                                             fieldname)
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None:
-            if not info['defined']:
-                return None
-        alignment = 1
-        layout = [None] * info['size']
-        for fieldname, fieldtype in self.interesting_fields:
-            if isinstance(fieldtype, Struct):
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                c_fieldtype = config_result.get_entry_result(fieldtype)
-                layout_addfield(layout, offset, c_fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(c_fieldtype))
-            else:
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                sign   = info.get('fldunsigned ' + fieldname, False)
-                if (size, sign) != size_and_sign(fieldtype):
-                    fieldtype = fixup_ctype(fieldtype, fieldname, (size, sign))
-                layout_addfield(layout, offset, fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(fieldtype))
-
-        # try to enforce the same alignment as the one of the original
-        # structure
-        if alignment < info['align']:
-            choices = [ctype for ctype in alignment_types
-                             if ctype_alignment(ctype) == info['align']]
-            assert choices, "unsupported alignment %d" % (info['align'],)
-            choices = [(ctypes.sizeof(ctype), i, ctype)
-                       for i, ctype in enumerate(choices)]
-            csize, _, ctype = min(choices)
-            for i in range(0, info['size'] - csize + 1, info['align']):
-                if layout[i:i+csize] == [None] * csize:
-                    layout_addfield(layout, i, ctype, '_alignment')
-                    break
-            else:
-                raise AssertionError("unenforceable alignment %d" % (
-                    info['align'],))
-
-        n = 0
-        for i, cell in enumerate(layout):
-            if cell is not None:
-                continue
-            layout_addfield(layout, i, ctypes.c_char, '_pad%d' % (n,))
-            n += 1
-
-        # build the ctypes Structure
-        seen = {}
-        fields = []
-        for cell in layout:
-            if cell in seen:
-                continue
-            fields.append((cell.name, cell.ctype))
-            seen[cell] = True
-
-        class S(ctypes.Structure):
-            _fields_ = fields
-        name = self.name
-        if name.startswith('struct '):
-            name = name[7:]
-        S.__name__ = name
-        return S
-
-class SimpleType(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined simple numeric type.
-    """
-    def __init__(self, name, ctype_hint=ctypes.c_int, ifdef=None):
-        self.name = name
-        self.ctype_hint = ctype_hint
-        self.ifdef = ifdef
-        
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield ''
-        yield 'ctypesplatcheck_t x;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        if self.ctype_hint in integer_class:
-            yield 'x = 0; x = ~x;'
-            yield 'dump("unsigned", x > 0);'
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None and not info['defined']:
-            return None
-        size = info['size']
-        sign = info.get('unsigned', False)
-        ctype = self.ctype_hint
-        if (size, sign) != size_and_sign(ctype):
-            ctype = fixup_ctype(ctype, self.name, (size, sign))
-        return ctype
-
-class ConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant.
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'if ((%s) < 0) {' % (self.name,)
-        yield '    long long x = (long long)(%s);' % (self.name,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.name,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-
-    def build_result(self, info, config_result):
-        return info['value']
-
-class DefinedConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant. If not #defined the value will be None.
-    """
-    def __init__(self, macro):
-        self.name = self.macro = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'if ((%s) < 0) {' % (self.macro,)
-        yield '    long long x = (long long)(%s);' % (self.macro,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.macro,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            return info['value']
-        return None
-
-
-class DefinedConstantString(CConfigEntry):
-    """
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'int i;'
-        yield 'char *p = %s;' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'for (i = 0; p[i] != 0; i++ ) {'
-        yield '  printf("value_%d: %d\\n", i, (int)(unsigned char)p[i]);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            string = ''
-            d = 0
-            while info.has_key('value_%d' % d):
-                string += chr(info['value_%d' % d])
-                d += 1
-            return string
-        return None
-
-
-class Defined(CConfigEntry):
-    """A boolean, corresponding to an #ifdef.
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % (self.macro,)
-        yield 'dump("defined", 1);'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        return bool(info['defined'])
-
-class CConfigSingleEntry(object):
-    """ An abstract class of type which requires
-    gcc succeeding/failing instead of only asking
-    """
-    pass
-
-class Has(CConfigSingleEntry):
-    def __init__(self, name):
-        self.name = name
-    
-    def question(self, ask_gcc):
-        return ask_gcc(self.name + ';')
-
-class Works(CConfigSingleEntry):
-    def question(self, ask_gcc):
-        return ask_gcc("")
-
-class SizeOf(CConfigEntry):
-    """An entry in a CConfig class that stands for
-    some external opaque type
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'dump("size",  sizeof(%s));' % self.name
-
-    def build_result(self, info, config_result):
-        return info['size']
-
-# ____________________________________________________________
-#
-# internal helpers
-
-def ctype_alignment(c_type):
-    if issubclass(c_type, ctypes.Structure):
-        return max([ctype_alignment(fld_type)
-                     for fld_name, fld_type in c_type._fields_])
-    
-    return ctypes.alignment(c_type)
-
-def uniquefilepath(LAST=[0]):
-    i = LAST[0]
-    LAST[0] += 1
-    return configdir.join('ctypesplatcheck_%d.c' % i)
-
-alignment_types = [
-    ctypes.c_short,
-    ctypes.c_int,
-    ctypes.c_long,
-    ctypes.c_float,
-    ctypes.c_double,
-    ctypes.c_char_p,
-    ctypes.c_void_p,
-    ctypes.c_longlong,
-    ctypes.c_wchar,
-    ctypes.c_wchar_p,
-    ]
-
-integer_class = [ctypes.c_byte,     ctypes.c_ubyte,
-                 ctypes.c_short,    ctypes.c_ushort,
-                 ctypes.c_int,      ctypes.c_uint,
-                 ctypes.c_long,     ctypes.c_ulong,
-                 ctypes.c_longlong, ctypes.c_ulonglong,
-                 ]
-float_class = [ctypes.c_float, ctypes.c_double]
-
-class Field(object):
-    def __init__(self, name, ctype):
-        self.name = name
-        self.ctype = ctype
-    def __repr__(self):
-        return '<field %s: %s>' % (self.name, self.ctype)
-
-def layout_addfield(layout, offset, ctype, prefix):
-    size = ctypes.sizeof(ctype)
-    name = prefix
-    i = 0
-    while name in layout:
-        i += 1
-        name = '%s_%d' % (prefix, i)
-    field = Field(name, ctype)
-    for i in range(offset, offset+size):
-        assert layout[i] is None, "%s overlaps %r" % (fieldname, layout[i])
-        layout[i] = field
-    return field
-
-def size_and_sign(ctype):
-    return (ctypes.sizeof(ctype),
-            ctype in integer_class and ctype(-1).value > 0)
-
-def fixup_ctype(fieldtype, fieldname, expected_size_and_sign):
-    for typeclass in [integer_class, float_class]:
-        if fieldtype in typeclass:
-            for ctype in typeclass:
-                if size_and_sign(ctype) == expected_size_and_sign:
-                    return ctype
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_char):
-        # for now, assume it is an array of chars; otherwise we'd also
-        # have to check the exact integer type of the elements of the array
-        size, sign = expected_size_and_sign
-        return ctypes.c_char * size
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_ubyte):
-        # grumble, fields of type 'c_char array' have automatic cast-to-
-        # Python-string behavior in ctypes, which may not be what you
-        # want, so here is the same with c_ubytes instead...
-        size, sign = expected_size_and_sign
-        return ctypes.c_ubyte * size
-    raise TypeError("conflicting field type %r for %r" % (fieldtype,
-                                                          fieldname))
-
-
-C_HEADER = """
-#include <stdio.h>
-#include <stddef.h>   /* for offsetof() */
-#ifndef _WIN32
-#  include <stdint.h>   /* FreeBSD: for uint64_t */
-#endif
-
-void dump(char* key, int value) {
-    printf("%s: %d\\n", key, value);
-}
-"""
-
-def run_example_code(filepath, eci, noerr=False):
-    executable = build_executable([filepath], eci, noerr=noerr)
-    output = py.process.cmdexec(executable)
-    section = None
-    for line in output.splitlines():
-        line = line.strip()
-        if line.startswith('-+- '):      # start of a new section
-            section = {}
-        elif line == '---':              # section end
-            assert section is not None
-            yield section
-            section = None
-        elif line:
-            assert section is not None
-            key, value = line.split(': ')
-            section[key] = int(value)
-
-# ____________________________________________________________
-
-def get_python_include_dir():
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    return gcv['INCLUDEPY']
-
-if __name__ == '__main__':
-    doc = """Example:
-    
-       ctypes_platform.py  -h sys/types.h  -h netinet/in.h
-                           'struct sockaddr_in'
-                           sin_port  c_int
-    """
-    import sys, getopt
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'h:')
-    if not args:
-        print >> sys.stderr, doc
-    else:
-        assert len(args) % 2 == 1
-        headers = []
-        for opt, value in opts:
-            if opt == '-h':
-                headers.append('#include <%s>' % (value,))
-        name = args[0]
-        fields = []
-        for i in range(1, len(args), 2):
-            ctype = getattr(ctypes, args[i+1])
-            fields.append((args[i], ctype))
-
-        S = getstruct(name, '\n'.join(headers), fields)
-
-        for key, value in S._fields_:
-            print key, value
diff --git a/ctypes_configure/doc/configure.html b/ctypes_configure/doc/configure.html
deleted file mode 100644
--- a/ctypes_configure/doc/configure.html
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="latin1" ?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=latin1" />
-<meta name="generator" content="Docutils 0.4.1: http://docutils.sourceforge.net/" />
-<title>ctypes configure</title>
-</head>
-<body>
-<div class="document" id="ctypes-configure">
-<h1 class="title">ctypes configure</h1>
-<div class="section">
-<h1><a id="idea" name="idea">idea</a></h1>
-<p>One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes gcc
-for various platform-dependent details like
-exact sizes of types (for example size_t), #defines, exact outline
-of structures etc. It replaces in this regard code generator (h2py).</p>
-</div>
-<div class="section">
-<h1><a id="installation" name="installation">installation</a></h1>
-<p><tt class="docutils literal"><span class="pre">easy_install</span> <span class="pre">ctypes_configure</span></tt></p>
-</div>
-<div class="section">
-<h1><a id="usage" name="usage">usage</a></h1>
-<p><a class="reference" href="http://codespeak.net/svn/pypy/dist/ctypes_configure/doc/sample.py">sample.py</a> explains in details how to use it.</p>
-</div>
-</div>
-</body>
-</html>
diff --git a/ctypes_configure/doc/configure.txt b/ctypes_configure/doc/configure.txt
deleted file mode 100644
--- a/ctypes_configure/doc/configure.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-=================
-ctypes configure
-=================
-
-idea
-====
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes gcc
-for various platform-dependent details like
-exact sizes of types (for example size\_t), #defines, exact outline
-of structures etc. It replaces in this regard code generator (h2py).
-
-installation
-============
-
-``easy_install ctypes_configure``
-
-usage
-=====
-
-:source:`sample.py <ctypes_configure/doc/sample.py>` explains in details how to use it.
diff --git a/ctypes_configure/doc/sample.py b/ctypes_configure/doc/sample.py
deleted file mode 100644
--- a/ctypes_configure/doc/sample.py
+++ /dev/null
@@ -1,72 +0,0 @@
-
-from ctypes_configure import configure
-import ctypes
-
-class CConfigure:
-    _compilation_info_ = configure.ExternalCompilationInfo(
-        
-        # all lines landing in C header before includes
-        pre_include_lines = [],
-
-        # list of .h files to include
-        includes = ['time.h', 'sys/time.h', 'unistd.h'],
-
-        # list of directories to search for include files
-        include_dirs = [],
-
-        # all lines landing in C header after includes
-        post_include_lines = [],
-
-        # libraries to link with
-        libraries = [],
-
-        # library directories
-        library_dirs = [],
-
-        # additional C sources to compile with (that go to
-        # created .c files)
-        separate_module_sources = [],
-
-        # additional existing C source file names
-        separate_module_files = [],
-        )
-
-    # get real int type out of hint and name
-    size_t = configure.SimpleType('size_t', ctypes.c_int)
-
-    # grab value of numerical #define
-    NULL = configure.ConstantInteger('NULL')
-
-    # grab #define, whether it's defined or not
-    EXISTANT = configure.Defined('NULL')
-    NOT_EXISTANT = configure.Defined('XXXNOTNULL')
-
-    # check for existance of C functions
-    has_write = configure.Has('write')
-    no_xxxwrite = configure.Has('xxxwrite')
-
-    # check for size of type
-    sizeof_size_t = configure.SizeOf('size_t')
-
-    # structure, with given hints for interesting fields,
-    # types does not need to be too specific.
-    # all interesting fields would end up with right offset
-    # size and order
-    struct_timeval = configure.Struct('struct timeval',[
-        ('tv_sec', ctypes.c_int),
-        ('tv_usec', ctypes.c_int)])
-
-info = configure.configure(CConfigure)
-
-assert info['has_write']
-assert not info['no_xxxwrite']
-assert info['NULL'] == 0
-size_t = info['size_t']
-print "size_t in ctypes is ", size_t
-assert ctypes.sizeof(size_t) == info['sizeof_size_t']
-assert info['EXISTANT']
-assert not info['NOT_EXISTANT']
-print
-print "fields of struct timeval are "
-for name, value in info['struct_timeval']._fields_:
-    print "  ", name, " ", value
diff --git a/ctypes_configure/dumpcache.py b/ctypes_configure/dumpcache.py
deleted file mode 100644
--- a/ctypes_configure/dumpcache.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os, sys
-import ctypes
-
-
-def dumpcache(referencefilename, filename, config):
-    dirname = os.path.dirname(referencefilename)
-    filename = os.path.join(dirname, filename)
-    f = open(filename, 'w')
-    print >> f, 'import ctypes'
-    print >> f
-    names = config.keys()
-    names.sort()
-    print >> f, '__all__ = %r' % (tuple(names),)
-    print >> f
-    for key in names:
-        val = config[key]
-        if isinstance(val, (int, long)):
-            f.write("%s = %d\n" % (key, val))
-        elif val is None:
-            f.write("%s = None\n" % key)
-        elif isinstance(val, ctypes.Structure.__class__):
-            f.write("class %s(ctypes.Structure):\n" % key)
-            f.write("    _fields_ = [\n")
-            for k, v in val._fields_:
-                f.write("        ('%s', %s),\n" % (k, ctypes_repr(v)))
-            f.write("    ]\n")
-        elif isinstance(val, (tuple, list)):
-            for x in val:
-                assert isinstance(x, (int, long, str)), \
-                       "lists of integers or strings only"
-            f.write("%s = %r\n" % (key, val))
-        else:
-            # a simple type, hopefully
-            f.write("%s = %s\n" % (key, ctypes_repr(val)))
-    f.close()
-    print 'Wrote %s.' % (filename,)
-    sys.stdout.flush()
-
-def ctypes_repr(cls):
-    # ctypes_configure does not support nested structs so far
-    # so let's ignore it
-    if isinstance(cls, ctypes._SimpleCData.__class__):
-        return "ctypes." + cls.__name__
-    if hasattr(cls, '_length_') and hasattr(cls, '_type_'):  # assume an array
-        return '%s*%d' % (ctypes_repr(cls._type_), cls._length_)
-    raise NotImplementedError("saving of object with type %r" % type(cls))
diff --git a/ctypes_configure/stdoutcapture.py b/ctypes_configure/stdoutcapture.py
deleted file mode 100644
--- a/ctypes_configure/stdoutcapture.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-A quick hack to capture stdout/stderr.
-"""
-
-import os, sys
-
-
-class Capture:
-    
-    def __init__(self, mixed_out_err = False):
-        "Start capture of the Unix-level stdout and stderr."
-        if (not hasattr(os, 'tmpfile') or
-            not hasattr(os, 'dup') or
-            not hasattr(os, 'dup2') or
-            not hasattr(os, 'fdopen')):
-            self.dummy = 1
-        else:
-            try:
-                self.tmpout = os.tmpfile()
-                if mixed_out_err:
-                    self.tmperr = self.tmpout
-                else:
-                    self.tmperr = os.tmpfile()
-            except OSError:     # bah?  on at least one Windows box
-                self.dummy = 1
-                return
-            self.dummy = 0
-            # make new stdout/stderr files if needed
-            self.localoutfd = os.dup(1)
-            self.localerrfd = os.dup(2)
-            if hasattr(sys.stdout, 'fileno') and sys.stdout.fileno() == 1:
-                self.saved_stdout = sys.stdout
-                sys.stdout = os.fdopen(self.localoutfd, 'w', 1)
-            else:
-                self.saved_stdout = None
-            if hasattr(sys.stderr, 'fileno') and sys.stderr.fileno() == 2:
-                self.saved_stderr = sys.stderr
-                sys.stderr = os.fdopen(self.localerrfd, 'w', 0)
-            else:
-                self.saved_stderr = None
-            os.dup2(self.tmpout.fileno(), 1)
-            os.dup2(self.tmperr.fileno(), 2)
-
-    def done(self):
-        "End capture and return the captured text (stdoutfile, stderrfile)."
-        if self.dummy:
-            import cStringIO
-            return cStringIO.StringIO(), cStringIO.StringIO()
-        else:
-            os.dup2(self.localoutfd, 1)
-            os.dup2(self.localerrfd, 2)
-            if self.saved_stdout is not None:
-                f = sys.stdout
-                sys.stdout = self.saved_stdout
-                f.close()
-            else:
-                os.close(self.localoutfd)
-            if self.saved_stderr is not None:
-                f = sys.stderr
-                sys.stderr = self.saved_stderr
-                f.close()
-            else:
-                os.close(self.localerrfd)
-            self.tmpout.seek(0)
-            self.tmperr.seek(0)
-            return self.tmpout, self.tmperr
-
-
-if __name__ == '__main__':
-    # test
-    c = Capture()
-    try:
-        os.system('echo hello')
-    finally:
-        fout, ferr = c.done()
-    print 'Output:', `fout.read()`
-    print 'Error:', `ferr.read()`
diff --git a/ctypes_configure/test/__init__.py b/ctypes_configure/test/__init__.py
deleted file mode 100644
diff --git a/ctypes_configure/test/test_configure.py b/ctypes_configure/test/test_configure.py
deleted file mode 100644
--- a/ctypes_configure/test/test_configure.py
+++ /dev/null
@@ -1,212 +0,0 @@
-import py, sys, struct
-from ctypes_configure import configure
-from ctypes_configure.cbuild import ExternalCompilationInfo
-import ctypes
-
-def test_dirent():
-    dirent = configure.getstruct("struct dirent",
-                                       """
-           struct dirent  /* for this example only, not the exact dirent */
-           {
-               long d_ino;
-               int d_off;
-               unsigned short d_reclen;
-               char d_name[32];
-           };
-                                       """,
-                                       [("d_reclen", ctypes.c_ushort)])
-    assert issubclass(dirent, ctypes.Structure)
-    ssize = (ctypes.sizeof(ctypes.c_long) +
-             ctypes.sizeof(ctypes.c_int) +
-             ctypes.sizeof(ctypes.c_ushort) +
-             32)
-    extra_padding = (-ssize) % ctypes.alignment(ctypes.c_long)
-
-    assert dirent._fields_ == [('_alignment', ctypes.c_long),
-                               ('_pad0', ctypes.c_char),
-                               ('_pad1', ctypes.c_char),
-                               ('_pad2', ctypes.c_char),
-                               ('_pad3', ctypes.c_char),
-                               ('d_reclen', ctypes.c_ushort),
-                               ] + [
-                               ('_pad%d' % n, ctypes.c_char)
-                                    for n in range(4, 4+32+extra_padding)]
-    assert ctypes.sizeof(dirent) == ssize + extra_padding
-    assert ctypes.alignment(dirent) == ctypes.alignment(ctypes.c_long)
-
-def test_fit_type():
-    S = configure.getstruct("struct S",
-                                  """
-           struct S {
-               signed char c;
-               unsigned char uc;
-               short s;
-               unsigned short us;
-               int i;
-               unsigned int ui;
-               long l;
-               unsigned long ul;
-               long long ll;
-               unsigned long long ull;
-               float f;
-               double d;
-           };
-                                  """,
-                                  [("c",   ctypes.c_int),
-                                   ("uc",  ctypes.c_int),
-                                   ("s",   ctypes.c_uint),
-                                   ("us",  ctypes.c_int),
-                                   ("i",   ctypes.c_int),
-                                   ("ui",  ctypes.c_int),
-                                   ("l",   ctypes.c_int),
-                                   ("ul",  ctypes.c_int),
-                                   ("ll",  ctypes.c_int),
-                                   ("ull", ctypes.c_int),
-                                   ("f",   ctypes.c_double),
-                                   ("d",   ctypes.c_float)])
-    assert issubclass(S, ctypes.Structure)
-    fields = dict(S._fields_)
-    assert fields["c"] == ctypes.c_byte
-    assert fields["uc"] == ctypes.c_ubyte
-    assert fields["s"] == ctypes.c_short
-    assert fields["us"] == ctypes.c_ushort
-    assert fields["i"] == ctypes.c_int
-    assert fields["ui"] == ctypes.c_uint
-    assert fields["l"] == ctypes.c_long
-    assert fields["ul"] == ctypes.c_ulong
-    assert fields["ll"] == ctypes.c_longlong
-    assert fields["ull"] == ctypes.c_ulonglong
-    assert fields["f"] == ctypes.c_float
-    assert fields["d"] == ctypes.c_double
-
-def test_simple_type():
-    ctype = configure.getsimpletype('test_t',
-                                          'typedef unsigned short test_t;',
-                                          ctypes.c_int)
-    assert ctype == ctypes.c_ushort
-
-def test_constant_integer():
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (6*7)')
-    assert value == 42
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (-2147483648LL)')
-    assert value == -2147483648
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (3333333333ULL)')
-    assert value == 3333333333
-
-def test_defined():
-    res = configure.getdefined('ALFKJLKJFLKJFKLEJDLKEWMECEE', '')
-    assert not res
-    res = configure.getdefined('ALFKJLKJFLKJFKLEJDLKEWMECEE',
-                                     '#define ALFKJLKJFLKJFKLEJDLKEWMECEE')
-    assert res
-
-def test_configure():
-    configdir = configure.configdir
-    test_h = configdir.join('test_ctypes_platform.h')
-    test_h.write('#define XYZZY 42\n')
-
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            pre_include_lines = ["/* a C comment */",
-                                 "#include <stdio.h>",
-                                 "#include <test_ctypes_platform.h>"],
-            include_dirs = [str(configdir)]
-        )
-
-        FILE = configure.Struct('FILE', [])
-        ushort = configure.SimpleType('unsigned short')
-        XYZZY = configure.ConstantInteger('XYZZY')
-
-    res = configure.configure(CConfig)
-    assert issubclass(res['FILE'], ctypes.Structure)
-    assert res == {'FILE': res['FILE'],
-                   'ushort': ctypes.c_ushort,
-                   'XYZZY': 42}
-
-def test_ifdef():
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            post_include_lines = ['/* a C comment */',
-                                  '#define XYZZY 42',
-                                  'typedef int foo;',
-                                  'struct s {',
-                                  'int i;',
-                                  'double f;'
-                                  '};'])
-
-
-        s = configure.Struct('struct s', [('i', ctypes.c_int)],
-                                   ifdef='XYZZY')
-        z = configure.Struct('struct z', [('i', ctypes.c_int)],
-                                   ifdef='FOOBAR')
-
-        foo = configure.SimpleType('foo', ifdef='XYZZY')
-        bar = configure.SimpleType('bar', ifdef='FOOBAR')
-
-    res = configure.configure(CConfig)
-    assert res['s'] is not None
-    assert res['z'] is None
-    assert res['foo'] is not None
-    assert res['bar'] is None
-
-def test_nested_structs():
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            post_include_lines="""
-            struct x {
-            int foo;
-            unsigned long bar;
-            };
-            struct y {
-            char c;
-            struct x x;
-            };
-            """.split("\n"))
-
-        x = configure.Struct("struct x", [("bar", ctypes.c_short)])
-        y = configure.Struct("struct y", [("x", x)])
-
-    res = configure.configure(CConfig)
-    c_x = res["x"]
-    c_y = res["y"]
-    c_y_fields = dict(c_y._fields_)
-    assert issubclass(c_x , ctypes.Structure)
-    assert issubclass(c_y, ctypes.Structure)
-    assert c_y_fields["x"] is c_x
-
-def test_array():
-    dirent = configure.getstruct("struct dirent",
-                                       """
-           struct dirent  /* for this example only, not the exact dirent */
-           {
-               long d_ino;
-               int d_off;
-               unsigned short d_reclen;
-               char d_name[32];
-           };
-                                       """,
-                                       [("d_name", ctypes.c_char * 0)])
-    assert dirent.d_name.size == 32
-
-def test_has():
-    assert configure.has("x", "int x = 3;")
-    assert not configure.has("x", "")
-    # has() should also not crash if it is given an invalid #include
-    assert not configure.has("x", "#include <some/path/which/cannot/exist>")
-
-def test_check_eci():
-    eci = ExternalCompilationInfo()
-    assert configure.check_eci(eci)
-    eci = ExternalCompilationInfo(libraries=['some_name_that_doesnt_exist_'])
-    assert not configure.check_eci(eci)
-
-def test_sizeof():
-    assert configure.sizeof("char", ExternalCompilationInfo()) == 1
-
-def test_memory_alignment():
-    a = configure.memory_alignment()
-    print a
-    assert a % struct.calcsize("P") == 0
diff --git a/ctypes_configure/test/test_dumpcache.py b/ctypes_configure/test/test_dumpcache.py
deleted file mode 100644
--- a/ctypes_configure/test/test_dumpcache.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import ctypes
-from ctypes_configure import configure, dumpcache
-from ctypes_configure.cbuild import ExternalCompilationInfo
-
-
-def test_cache():
-    configdir = configure.configdir
-    test_h = configdir.join('test_ctypes_platform2.h')
-    test_h.write('#define XYZZY 42\n'
-                 "#define large 2147483648L\n")
-
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            pre_include_lines = ["/* a C comment */",
-                                 "#include <stdio.h>",
-                                 "#include <test_ctypes_platform2.h>"],
-            include_dirs = [str(configdir)]
-        )
-
-        FILE = configure.Struct('FILE', [])
-        ushort = configure.SimpleType('unsigned short')
-        XYZZY = configure.ConstantInteger('XYZZY')
-        XUZ = configure.Has('XUZ')
-        large = configure.DefinedConstantInteger('large')
-        undef = configure.Defined('really_undefined')
-
-    res = configure.configure(CConfig)
-
-    cachefile = configdir.join('cache')
-    dumpcache.dumpcache('', str(cachefile), res)
-
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['XYZZY'] == res['XYZZY']
-    assert d['ushort'] == res['ushort']
-    assert d['FILE']._fields_ == res['FILE']._fields_
-    assert d['FILE'].__mro__[1:] == res['FILE'].__mro__[1:]
-    assert d['undef'] == res['undef']
-    assert d['large'] == res['large']
-    assert d['XUZ'] == res['XUZ']
-
-
-def test_cache_array():
-    configdir = configure.configdir
-    res = {'foo': ctypes.c_short * 27}
-    cachefile = configdir.join('cache_array')
-    dumpcache.dumpcache('', str(cachefile), res)
-    #
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['foo'] == res['foo']
-
-def test_cache_array_array():
-    configdir = configure.configdir
-    res = {'foo': (ctypes.c_int * 2) * 3}
-    cachefile = configdir.join('cache_array_array')
-    dumpcache.dumpcache('', str(cachefile), res)
-    #
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['foo'] == res['foo']
diff --git a/lib-python/2.7/sqlite3/test/regression.py b/lib-python/2.7/sqlite3/test/regression.py
--- a/lib-python/2.7/sqlite3/test/regression.py
+++ b/lib-python/2.7/sqlite3/test/regression.py
@@ -351,7 +351,10 @@
         self.assertRaises(ValueError, cur.execute, " \0select 2")
         self.assertRaises(ValueError, cur.execute, "select 2\0")
 
+    @test_support.impl_detail(pypy=False)
     def CheckCommitCursorReset(self):
+        # This test is for logic added in 2.7.13 which PyPy doesn't
+        # implement.  See http://bugs.python.org/issue29006
         """
         Connection.commit() did reset cursors, which made sqlite3
         to return rows multiple times when fetched from cursors
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -374,12 +374,6 @@
                 if cursor is not None:
                     cursor._reset = True
 
-    def _reset_other_statements(self, excepted):
-        for weakref in self.__statements:
-            statement = weakref()
-            if statement is not None and statement is not excepted:
-                statement._reset()
-
     @_check_thread_wrap
     @_check_closed_wrap
     def __call__(self, sql):
@@ -435,6 +429,16 @@
         if not self._in_transaction:
             return
 
+        # The following line is a KNOWN DIFFERENCE with CPython 2.7.13.
+        # More precisely, the corresponding line was removed in the
+        # version 2.7.13 of CPython, but this is causing troubles for
+        # PyPy (and potentially for CPython too):
+        #
+        #     http://bugs.python.org/issue29006
+        #
+        # So for now, we keep this line.
+        self.__do_all_statements(Statement._reset, False)
+
         statement_star = _ffi.new('sqlite3_stmt **')
         ret = _lib.sqlite3_prepare_v2(self._db, b"COMMIT", -1,
                                       statement_star, _ffi.NULL)
@@ -862,24 +866,7 @@
                 self.__statement._set_params(params)
 
                 # Actually execute the SQL statement
-
-                # NOTE: if we get SQLITE_LOCKED, it's probably because
-                # one of the cursors created previously is still alive
-                # and not reset and the operation we're trying to do
-                # makes Sqlite unhappy about that.  In that case, we
-                # automatically reset all cursors and try again.  This
-                # is not what CPython does!  It is a workaround for a
-                # new feature of 2.7.13.  Previously, all cursors would
-                # be reset at commit(), which makes it unlikely to have
-                # cursors lingering around.  Since 2.7.13, cursors stay
-                # around instead.  This causes problems here---at least:
-                # this is the only place shown by pysqlite tests, and I
-                # can only hope there is no other.
-
                 ret = _lib.sqlite3_step(self.__statement._statement)
-                if ret == _lib.SQLITE_LOCKED:
-                    self.__connection._reset_other_statements(self.__statement)
-                    ret = _lib.sqlite3_step(self.__statement._statement)
 
                 if ret == _lib.SQLITE_ROW:
                     if multiple:
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.9.1
+Version: 1.9.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.9.1"
-__version_info__ = (1, 9, 1)
+__version__ = "1.9.2"
+__version_info__ = (1, 9, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.9.1"
+                               "\ncompiled with cffi version: 1.9.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/py/_io/saferepr.py b/py/_io/saferepr.py
--- a/py/_io/saferepr.py
+++ b/py/_io/saferepr.py
@@ -21,6 +21,11 @@
                 return py.builtin._totext('"%s"') % u
             else:
                 return py.builtin._totext("'%s'") % u.replace("'", r"\'")
+
+        repr = builtin_repr
+        # ^^^ it's very annoying to display 'xx' instead of u'xx' when
+        # the difference can be essential, particularly in PyPy
+
         s = repr(x[:self.maxstring])
         if len(s) > self.maxstring:
             i = max(0, (self.maxstring-3)//2)
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -494,6 +494,11 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
+* The ``sqlite`` module was updated on 2.7.13 to no longer reset all
+  cursors when there is a commit.  This causes troubles for PyPy (and
+  potentially for CPython too), and so for now we didn't port this change:
+  see http://bugs.python.org/issue29006.
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -77,3 +77,16 @@
 .. branch: stdlib-2.7.13
 
 Updated the implementation to match CPython 2.7.13 instead of 2.7.13.
+
+.. branch: issue2444
+
+Fix ``PyObject_GetBuffer`` and ``PyMemoryView_GET_BUFFER``, which leaked
+memory and held references. Add a finalizer to CPyBuffer, add a
+PyMemoryViewObject with a PyBuffer attached so that the call to 
+``PyMemoryView_GET_BUFFER`` does not leak a PyBuffer-sized piece of memory.
+Properly call ``bf_releasebuffer`` when not ``NULL``.
+
+.. branch: boehm-rawrefcount
+
+Support translations of cpyext with the Boehm GC (for special cases like
+revdb).
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -309,9 +309,9 @@
             config.translation.jit = True
 
         if config.objspace.usemodules.cpyext:
-            if config.translation.gc != 'incminimark':
+            if config.translation.gc not in ('incminimark', 'boehm'):
                 raise Exception("The 'cpyext' module requires the 'incminimark'"
-                                " GC.  You need either 'targetpypystandalone.py"
+                                " 'boehm' GC.  You need either 'targetpypystandalone.py"
                                 " --withoutmod-cpyext' or '--gc=incminimark'")
 
         config.translating = True
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.9.1"
+VERSION = "1.9.2"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -17,7 +17,7 @@
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrBase, W_CTypePointer
 from pypy.module._cffi_backend.ctypevoid import W_CTypeVoid
-from pypy.module._cffi_backend.ctypestruct import W_CTypeStruct
+from pypy.module._cffi_backend.ctypestruct import W_CTypeStruct, W_CTypeUnion
 from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned,
     W_CTypePrimitiveUnsigned, W_CTypePrimitiveCharOrUniChar,
     W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble)
@@ -231,6 +231,11 @@
         return cifbuilder.fb_struct_ffi_type(self, is_result_type)
     return _missing_ffi_type(self, cifbuilder, is_result_type)
 
+def _union_ffi_type(self, cifbuilder, is_result_type):
+    if self.size >= 0:   # only for a better error message
+        return cifbuilder.fb_union_ffi_type(self, is_result_type)
+    return _missing_ffi_type(self, cifbuilder, is_result_type)
+
 def _primsigned_ffi_type(self, cifbuilder, is_result_type):
     size = self.size
     if   size == 1: return clibffi.ffi_type_sint8
@@ -266,6 +271,7 @@
 
 W_CType._get_ffi_type                       = _missing_ffi_type
 W_CTypeStruct._get_ffi_type                 = _struct_ffi_type
+W_CTypeUnion._get_ffi_type                  = _union_ffi_type
 W_CTypePrimitiveSigned._get_ffi_type        = _primsigned_ffi_type
 W_CTypePrimitiveCharOrUniChar._get_ffi_type = _primunsigned_ffi_type
 W_CTypePrimitiveUnsigned._get_ffi_type      = _primunsigned_ffi_type
@@ -276,6 +282,12 @@
 # ----------
 
 
+_SUPPORTED_IN_API_MODE = (
+        " are only supported as %s if the function is "
+        "'API mode' and non-variadic (i.e. declared inside ffibuilder"
+        ".cdef()+ffibuilder.set_source() and not taking a final '...' "
+        "argument)")
+
 class CifDescrBuilder(object):
     rawmem = lltype.nullptr(rffi.CCHARP.TO)
 
@@ -297,6 +309,20 @@
     def fb_fill_type(self, ctype, is_result_type):
         return ctype._get_ffi_type(self, is_result_type)
 
+    def fb_unsupported(self, ctype, is_result_type, detail):
+        place = "return value" if is_result_type else "argument"
+        raise oefmt(self.space.w_NotImplementedError,
+            "ctype '%s' not supported as %s.  %s.  "
+            "Such structs" + _SUPPORTED_IN_API_MODE,
+            ctype.name, place, detail, place)
+
+    def fb_union_ffi_type(self, ctype, is_result_type=False):
+        place = "return value" if is_result_type else "argument"
+        raise oefmt(self.space.w_NotImplementedError,
+            "ctype '%s' not supported as %s by libffi.  "
+            "Unions" + _SUPPORTED_IN_API_MODE,
+            ctype.name, place, place)
+
     def fb_struct_ffi_type(self, ctype, is_result_type=False):
         # We can't pass a struct that was completed by verify().
         # Issue: assume verify() is given "struct { long b; ...; }".
@@ -309,37 +335,40 @@
         # Another reason for 'custom_field_pos' would be anonymous
         # nested structures: we lost the information about having it
         # here, so better safe (and forbid it) than sorry (and maybe
-        # crash).
+        # crash).  Note: it seems we only get in this case with
+        # ffi.verify().
         space = self.space
         ctype.force_lazy_struct()
         if ctype._custom_field_pos:
             # these NotImplementedErrors may be caught and ignored until
             # a real call is made to a function of this type
-            place = "return value" if is_result_type else "argument"
-            raise oefmt(space.w_NotImplementedError,
-                "ctype '%s' not supported as %s (it is a struct declared "
-                "with \"...;\", but the C calling convention may depend "
-                "on the missing fields)", ctype.name, place)
+            raise self.fb_unsupported(ctype, is_result_type,
+                "It is a struct declared with \"...;\", but the C "
+                "calling convention may depend on the missing fields; "
+                "or, it contains anonymous struct/unions")
+        # Another reason: __attribute__((packed)) is not supported by libffi.
+        if ctype._with_packed_change:
+            raise self.fb_unsupported(ctype, is_result_type,
+                "It is a 'packed' structure, with a different layout than "
+                "expected by libffi")
 
         # walk the fields, expanding arrays into repetitions; first,
         # only count how many flattened fields there are
         nflat = 0
         for i, cf in enumerate(ctype._fields_list):
             if cf.is_bitfield():
-                place = "return value" if is_result_type else "argument"
-                raise oefmt(space.w_NotImplementedError,
-                    "ctype '%s' not supported as %s"
-                    " (it is a struct with bit fields)", ctype.name, place)
+                raise self.fb_unsupported(ctype, is_result_type,
+                    "It is a struct with bit fields, which libffi does not "
+                    "support")
             flat = 1
             ct = cf.ctype
             while isinstance(ct, ctypearray.W_CTypeArray):
                 flat *= ct.length
                 ct = ct.ctitem
             if flat <= 0:
-                place = "return value" if is_result_type else "argument"
-                raise oefmt(space.w_NotImplementedError,
-                    "ctype '%s' not supported as %s (it is a struct"
-                    " with a zero-length array)", ctype.name, place)
+                raise self.fb_unsupported(ctype, is_result_type,
+                    "It is a struct with a zero-length array, which libffi "
+                    "does not support")
             nflat += flat
 
         if USE_C_LIBFFI_MSVC and is_result_type:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -34,6 +34,7 @@
     _fields_dict = None
     _custom_field_pos = False
     _with_var_array = False
+    _with_packed_changed = False
 
     def __init__(self, space, name):
         W_CType.__init__(self, space, -1, name, len(name))
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -303,6 +303,7 @@
     fields_dict = {}
     w_ctype._custom_field_pos = False
     with_var_array = False
+    with_packed_change = False
 
     for i in range(len(fields_w)):
         w_field = fields_w[i]
@@ -333,7 +334,8 @@
         #
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield or if SF_PACKED
-        falign = 1 if sflags & SF_PACKED else ftype.alignof()
+        falignorg = ftype.alignof()
+        falign = 1 if sflags & SF_PACKED else falignorg
         do_align = True
         if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
@@ -359,7 +361,10 @@
                 bs_flag = ctypestruct.W_CField.BS_REGULAR
 

From pypy.commits at gmail.com  Mon Jan  2 13:48:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 10:48:07 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <586aa067.61c9c20a.d7016.5811@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r842:53c36248b7ee
Date: 2017-01-02 19:47 +0100
http://bitbucket.org/pypy/pypy.org/changeset/53c36248b7ee/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66493 of $105000 (63.3%)
+   $66502 of $105000 (63.3%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2331 left
+    we have $2339 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Mon Jan  2 14:45:03 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 02 Jan 2017 11:45:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix merge in test_memoryobject.py
Message-ID: <586aadbf.c9b3c20a.bbe06.1349@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89313:5fee3be01d9e
Date: 2017-01-02 19:44 +0000
http://bitbucket.org/pypy/pypy/changeset/5fee3be01d9e/

Log:	fix merge in test_memoryobject.py

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -5,7 +5,7 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
 
-only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
@@ -30,10 +30,10 @@
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
         w_mv = api.PyMemoryView_FromBuffer(view)
-        for f in ('format', 'itemsize', 'ndim', 'readonly', 
+        for f in ('format', 'itemsize', 'ndim', 'readonly',
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)
-            assert space.eq_w(space.getattr(w_mv, w_f), 
+            assert space.eq_w(space.getattr(w_mv, w_f),
                               space.getattr(w_memoryview, w_f))
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
@@ -87,11 +87,6 @@
         assert y.shape == (10,)
         assert len(y) == 10
         assert y[3] == 3
-        s = y[3]
-        assert len(s) == struct.calcsize('i')
-        assert s == struct.pack('i', 3)
-        viewlen = module.test_buffer(arr)
-        assert viewlen == y.itemsize * len(y)
 
     def test_buffer_protocol_capi(self):
         foo = self.import_extension('foo', [
@@ -106,7 +101,7 @@
                     return NULL;
                 vlen = view.len / view.itemsize;
                 PyBuffer_Release(&view);
-                return PyInt_FromLong(vlen);
+                return PyLong_FromLong(vlen);
              """),
             ("test_buffer", "METH_VARARGS",
              """
@@ -114,16 +109,16 @@
                 PyObject* obj = PyTuple_GetItem(args, 0);
                 PyObject* memoryview = PyMemoryView_FromObject(obj);
                 if (memoryview == NULL)
-                    return PyInt_FromLong(-1);
+                    return PyLong_FromLong(-1);
                 view = PyMemoryView_GET_BUFFER(memoryview);
                 Py_DECREF(memoryview);
-                return PyInt_FromLong(view->len / view->itemsize);
+                return PyLong_FromLong(view->len / view->itemsize);
             """)])
         module = self.import_module(name='buffer_test')
         arr = module.PyMyArray(10)
         ten = foo.get_len(arr)
         assert ten == 10
-        ten = foo.get_len('1234567890')
+        ten = foo.get_len(b'1234567890')
         assert ten == 10
         ten = foo.test_buffer(arr)
         assert ten == 10
@@ -145,15 +140,15 @@
         shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
         assert strides[-1] == 8
         dt1 = np.dtype(
-             [('a', 'b'), ('b', 'i'), 
-              ('sub0', np.dtype('b,i')), 
-              ('sub1', np.dtype('b,i')), 
-              ('sub2', np.dtype('b,i')), 
-              ('sub3', np.dtype('b,i')), 
-              ('sub4', np.dtype('b,i')), 
-              ('sub5', np.dtype('b,i')), 
-              ('sub6', np.dtype('b,i')), 
-              ('sub7', np.dtype('b,i')), 
+             [('a', 'b'), ('b', 'i'),
+              ('sub0', np.dtype('b,i')),
+              ('sub1', np.dtype('b,i')),
+              ('sub2', np.dtype('b,i')),
+              ('sub3', np.dtype('b,i')),
+              ('sub4', np.dtype('b,i')),
+              ('sub5', np.dtype('b,i')),
+              ('sub6', np.dtype('b,i')),
+              ('sub7', np.dtype('b,i')),
               ('c', 'i')],
              )
         x = np.arange(dt1.itemsize, dtype='int8').view(dt1)

From pypy.commits at gmail.com  Mon Jan  2 14:54:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 11:54:48 -0800 (PST)
Subject: [pypy-commit] cffi default: support more obscure compilers. source
 is https://github.com/matricks/bam/pull/61/files
Message-ID: <586ab008.96a61c0a.46f2d.ba88@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2847:80a8b3ce3902
Date: 2017-01-02 20:53 +0100
http://bitbucket.org/cffi/cffi/changeset/80a8b3ce3902/

Log:	support more obscure compilers. source is
	https://github.com/matricks/bam/pull/61/files

diff --git a/c/call_python.c b/c/call_python.c
--- a/c/call_python.c
+++ b/c/call_python.c
@@ -177,7 +177,20 @@
 #if (defined(WITH_THREAD) && !defined(_MSC_VER) &&   \
      !defined(__amd64__) && !defined(__x86_64__) &&   \
      !defined(__i386__) && !defined(__i386))
-# define read_barrier()  __sync_synchronize()
+# if defined(__GNUC__)
+#   define read_barrier()  __sync_synchronize()
+# elif defined(_AIX)
+#   define read_barrier()  __lwsync()
+# elif defined(__SUNPRO_C) || defined(__SUNPRO_CC)
+#   include <mbarrier.h>
+#   define read_barrier()  __compiler_barrier()
+# elif defined(__hpux)
+#   define read_barrier()  _Asm_mf()
+# else
+#   define read_barrier()  /* missing */
+#   warning "no definition for read_barrier(), missing synchronization for\
+ multi-thread initialization in embedded mode"
+# endif
 #else
 # define read_barrier()  (void)0
 #endif

From pypy.commits at gmail.com  Mon Jan  2 14:54:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 11:54:49 -0800 (PST)
Subject: [pypy-commit] cffi default: document
Message-ID: <586ab009.2aa9c20a.32025.125c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2848:eeadf4414558
Date: 2017-01-02 20:54 +0100
http://bitbucket.org/cffi/cffi/changeset/eeadf4414558/

Log:	document

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -20,6 +20,9 @@
   called with libffi because an argument is a struct that is "too
   complicated" (not a struct *pointer*).
 
+* Add support for some obscure compilers (non-msvc, non-gcc, non-icc,
+  non-clang)
+
 
 v1.9
 ====

From pypy.commits at gmail.com  Mon Jan  2 15:02:55 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 02 Jan 2017 12:02:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Remove commented-out duplicate
 implementation of PyMemoryView_GET_BUFFER
Message-ID: <586ab1ef.4c9d1c0a.74f50.ae2f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89314:084daed91c42
Date: 2017-01-02 20:02 +0000
http://bitbucket.org/pypy/pypy/changeset/084daed91c42/

Log:	Remove commented-out duplicate implementation of
	PyMemoryView_GET_BUFFER

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -214,30 +214,3 @@
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
     # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BASE')
-
-#XXX this function was removed in PyPy2, double-check that XXX
-#@cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
-#def PyMemoryView_GET_BUFFER(space, w_obj):
-#    """Return a pointer to the buffer-info structure wrapped by the given
-#    object.  The object must be a memoryview instance; this macro doesn't
-#    check its type, you must do it yourself or you will risk crashes."""
-#    view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-#    if not isinstance(w_obj, W_MemoryView):
-#        return view
-#    ndim = w_obj.buf.getndim()
-#    if ndim >= Py_MAX_NDIMS:
-#        # XXX warn?
-#        return view
-#    fill_Py_buffer(space, w_obj.buf, view)
-#    try:
-#        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-#        view.c_obj = make_ref(space, w_obj)
-#        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
-#        isstr = False
-#    except ValueError:
-#        w_s = w_obj.descr_tobytes(space)
-#        view.c_obj = make_ref(space, w_s)
-#        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
-#        rffi.setintfield(view, 'c_readonly', 1)
-#        isstr = True
-#    return view

From pypy.commits at gmail.com  Mon Jan  2 15:19:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 12:19:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: py3 compat
Message-ID: <586ab5c7.542e1c0a.8fa96.951c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89315:a42b878433f7
Date: 2017-01-02 21:18 +0100
http://bitbucket.org/pypy/pypy/changeset/a42b878433f7/

Log:	py3 compat

diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1975,7 +1975,7 @@
                 return s;
             }
         """, packed=True, min_version=(1, 8, 3))
-        assert lib.f().y == chr(40)
+        assert ord(lib.f().y) == 40
         assert lib.f().x == 200
         e = raises(NotImplementedError, lib.g, 0)
         assert str(e.value) == (

From pypy.commits at gmail.com  Mon Jan  2 15:24:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 12:24:50 -0800 (PST)
Subject: [pypy-commit] cffi default: py3 test fix
Message-ID: <586ab712.46bb1c0a.a84b0.d6e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2849:863081891de4
Date: 2017-01-02 21:24 +0100
http://bitbucket.org/cffi/cffi/changeset/863081891de4/

Log:	py3 test fix

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2166,7 +2166,7 @@
             return s;
         }
     """)
-    assert lib.f().y == chr(40)
+    assert ord(lib.f().y) == 40
     assert lib.f().x == 200
     e = py.test.raises(NotImplementedError, lib.g, 0)
     assert str(e.value) == (

From pypy.commits at gmail.com  Mon Jan  2 16:58:59 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 13:58:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: remove file
Message-ID: <586acd23.06891c0a.4d4b.c237@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r89318:7096ceaddf76
Date: 2017-01-02 21:48 +0200
http://bitbucket.org/pypy/pypy/changeset/7096ceaddf76/

Log:	remove file

	on pypy2 implements PyObject_CheckBuffer which is a C macro in
	abstract.h, CBuffer is not now used and it seems at some point the
	file was rewritten but not renamed

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
deleted file mode 100644
--- a/pypy/module/cpyext/buffer.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from rpython.rtyper.lltypesystem import rffi
-from rpython.rlib import buffer
-from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL)
-from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
-
-class CBuffer(buffer.Buffer):
-
-    _immutable_ = True
-
-    def __init__(self, space, c_buf, c_len, c_obj):
-        self.space = space
-        self.c_buf = c_buf
-        self.c_len = c_len
-        self.c_obj = c_obj
-
-    def __del__(self):
-        Py_DecRef(self.space, self.c_obj)
-
-    def getlength(self):
-        return self.c_len
-
-    def getitem(self, index):
-        return self.c_buf[index]
-
-    def as_str(self):
-        return rffi.charpsize2str(rffi.cast(rffi.CCHARP, self.c_buf),
-                                  self.c_len)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -11,7 +11,6 @@
 from pypy.objspace.std.memoryobject import W_MemoryView
 from pypy.module.cpyext.object import _dealloc
 from pypy.module.cpyext.import_ import PyImport_Import
-from pypy.module.cpyext.buffer import CBuffer
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView")
 

From pypy.commits at gmail.com  Mon Jan  2 16:58:55 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 13:58:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: function moved around in the same file
Message-ID: <586acd1f.88711c0a.281e6.ad1a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r89316:eedfc89de786
Date: 2017-01-02 21:22 +0200
http://bitbucket.org/pypy/pypy/changeset/eedfc89de786/

Log:	function moved around in the same file

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -215,29 +215,3 @@
     # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BASE')
 
-#XXX this function was removed in PyPy2, double-check that XXX
-#@cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
-#def PyMemoryView_GET_BUFFER(space, w_obj):
-#    """Return a pointer to the buffer-info structure wrapped by the given
-#    object.  The object must be a memoryview instance; this macro doesn't
-#    check its type, you must do it yourself or you will risk crashes."""
-#    view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-#    if not isinstance(w_obj, W_MemoryView):
-#        return view
-#    ndim = w_obj.buf.getndim()
-#    if ndim >= Py_MAX_NDIMS:
-#        # XXX warn?
-#        return view
-#    fill_Py_buffer(space, w_obj.buf, view)
-#    try:
-#        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-#        view.c_obj = make_ref(space, w_obj)
-#        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
-#        isstr = False
-#    except ValueError:
-#        w_s = w_obj.descr_tobytes(space)
-#        view.c_obj = make_ref(space, w_s)
-#        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
-#        rffi.setintfield(view, 'c_readonly', 1)
-#        isstr = True
-#    return view

From pypy.commits at gmail.com  Mon Jan  2 16:58:57 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 13:58:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: PyInt_FromLong -> PyLong_FromLong
Message-ID: <586acd21.6249c20a.a829a.3ca2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r89317:af6570151311
Date: 2017-01-02 21:23 +0200
http://bitbucket.org/pypy/pypy/changeset/af6570151311/

Log:	PyInt_FromLong -> PyLong_FromLong

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -106,7 +106,7 @@
                     return NULL;
                 vlen = view.len / view.itemsize;
                 PyBuffer_Release(&view);
-                return PyInt_FromLong(vlen);
+                return PyLong_FromLong(vlen);
              """),
             ("test_buffer", "METH_VARARGS",
              """
@@ -114,10 +114,10 @@
                 PyObject* obj = PyTuple_GetItem(args, 0);
                 PyObject* memoryview = PyMemoryView_FromObject(obj);
                 if (memoryview == NULL)
-                    return PyInt_FromLong(-1);
+                    return PyLong_FromLong(-1);
                 view = PyMemoryView_GET_BUFFER(memoryview);
                 Py_DECREF(memoryview);
-                return PyInt_FromLong(view->len / view->itemsize);
+                return PyLong_FromLong(view->len / view->itemsize);
             """)])
         module = self.import_module(name='buffer_test')
         arr = module.PyMyArray(10)

From pypy.commits at gmail.com  Mon Jan  2 16:59:02 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 13:59:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: tweak tests
Message-ID: <586acd26.d5091c0a.6942f.a35d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r89320:e4c3e3dff2bf
Date: 2017-01-02 23:26 +0200
http://bitbucket.org/pypy/pypy/changeset/e4c3e3dff2bf/

Log:	tweak tests

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -10,7 +10,8 @@
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
-        assert api.PyObject_CheckBuffer(w_hello)
+        # implemented as a C macro
+        #assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
         assert space.eq_w(w_char, space.wrap('h'))
@@ -123,7 +124,7 @@
         arr = module.PyMyArray(10)
         ten = foo.get_len(arr)
         assert ten == 10
-        ten = foo.get_len('1234567890')
+        ten = foo.get_len(b'1234567890')
         assert ten == 10
         ten = foo.test_buffer(arr)
         assert ten == 10

From pypy.commits at gmail.com  Mon Jan  2 16:59:04 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 13:59:04 -0800 (PST)
Subject: [pypy-commit] pypy default: PyInt_FromLong -> PyLong_FromLong
Message-ID: <586acd28.c5311c0a.d6d87.b0b2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r89321:4bc7410d9efc
Date: 2017-01-02 23:32 +0200
http://bitbucket.org/pypy/pypy/changeset/4bc7410d9efc/

Log:	PyInt_FromLong -> PyLong_FromLong

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -62,7 +62,7 @@
                     return NULL;
                 vlen = view.len / view.itemsize;
                 PyBuffer_Release(&view);
-                return PyInt_FromLong(vlen);
+                return PyLong_FromLong(vlen);
              """),
             ("test_buffer", "METH_VARARGS",
              """
@@ -70,10 +70,10 @@
                 PyObject* obj = PyTuple_GetItem(args, 0);
                 PyObject* memoryview = PyMemoryView_FromObject(obj);
                 if (memoryview == NULL)
-                    return PyInt_FromLong(-1);
+                    return PyLong_FromLong(-1);
                 view = PyMemoryView_GET_BUFFER(memoryview);
                 Py_DECREF(memoryview);
-                return PyInt_FromLong(view->len / view->itemsize);
+                return PyLong_FromLong(view->len / view->itemsize);
             """)])
         module = self.import_module(name='buffer_test')
         arr = module.PyMyArray(10)

From pypy.commits at gmail.com  Mon Jan  2 16:59:01 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 13:59:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: remove another buffer.py import,
 and a duplicate declaration of PyObject_GetBuffer
Message-ID: <586acd25.07941c0a.61a51.a775@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r89319:2fe6166e5b03
Date: 2017-01-02 22:54 +0200
http://bitbucket.org/pypy/pypy/changeset/2fe6166e5b03/

Log:	remove another buffer.py import, and a duplicate declaration of
	PyObject_GetBuffer

diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -49,7 +49,6 @@
 import pypy.module.cpyext.longobject
 import pypy.module.cpyext.listobject
 import pypy.module.cpyext.sequence
-import pypy.module.cpyext.buffer
 import pypy.module.cpyext.eval
 import pypy.module.cpyext.import_
 import pypy.module.cpyext.mapping
diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h
--- a/pypy/module/cpyext/include/abstract.h
+++ b/pypy/module/cpyext/include/abstract.h
@@ -10,16 +10,6 @@
     /* Return 1 if the getbuffer function is available, otherwise
        return 0 */
 
-     PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, Py_buffer *view,
-                                        int flags);
-
-    /* This is a C-API version of the getbuffer function call.  It checks
-       to make sure object has the required function pointer and issues the
-       call.  Returns -1 and raises an error on failure and returns 0 on
-       success
-    */
-
-
      PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view);
 
     /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*.

From pypy.commits at gmail.com  Mon Jan  2 16:59:06 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 13:59:06 -0800 (PST)
Subject: [pypy-commit] pypy default: add passing test from py3.5
Message-ID: <586acd2a.12ad1c0a.593aa.c543@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r89322:be6558eb1f0b
Date: 2017-01-02 23:49 +0200
http://bitbucket.org/pypy/pypy/changeset/be6558eb1f0b/

Log:	add passing test from py3.5

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -36,6 +36,30 @@
             assert space.eq_w(space.getattr(w_mv, w_f), 
                               space.getattr(w_memoryview, w_f))
 
+
+class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
+    def test_fillWithObject(self):
+        module = self.import_extension('foo', [
+                ("fillinfo", "METH_VARARGS",
+                 """
+                 Py_buffer buf;
+                 PyObject *str = PyBytes_FromString("hello, world.");
+                 if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
+                                       0, 0)) {
+                     return NULL;
+                 }
+
+                 /* Get rid of our own reference to the object, but
+                  * the Py_buffer should still have a reference.
+                  */
+                 Py_DECREF(str);
+
+                 return PyMemoryView_FromBuffer(&buf);
+                 """)])
+        result = module.fillinfo()
+        assert b"hello, world." == result
+        del result
+
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol_app(self):
         import struct

From pypy.commits at gmail.com  Mon Jan  2 16:59:08 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 13:59:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: delete outdated and wrong test
Message-ID: <586acd2c.8a281c0a.b5464.ad7b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r89323:d99c5e19b571
Date: 2017-01-02 23:51 +0200
http://bitbucket.org/pypy/pypy/changeset/d99c5e19b571/

Log:	delete outdated and wrong test

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -60,24 +60,6 @@
         assert b"hello, world." == result
         del result
 
-    def test_fill_from_NULL_pointer(self):
-        module = self.import_extension('foo', [
-                ("fillinfo_NULL", "METH_VARARGS",
-                 """
-                 Py_buffer info;
-                 if (PyBuffer_FillInfo(&info, NULL, NULL, 1, 1,
-                                       PyBUF_FULL_RO) < 0) {
-                     return NULL;
-                 }
-                 return PyMemoryView_FromBuffer(&info);
-                 """)])
-        exc = raises(ValueError, module.fillinfo_NULL)
-        assert 'NULL' in str(exc.value)
-
-    @pytest.mark.skipif(True, reason='write a test for this')
-    def test_get_base_and_get_buffer(self, space, api):
-        assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER
-
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol_app(self):
         import struct

From pypy.commits at gmail.com  Mon Jan  2 16:59:09 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 02 Jan 2017 13:59:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <586acd2d.c19d1c0a.e9a01.be08@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r89324:08682b9332c7
Date: 2017-01-02 23:55 +0200
http://bitbucket.org/pypy/pypy/changeset/08682b9332c7/

Log:	merge heads

diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1975,7 +1975,7 @@
                 return s;
             }
         """, packed=True, min_version=(1, 8, 3))
-        assert lib.f().y == chr(40)
+        assert ord(lib.f().y) == 40
         assert lib.f().x == 200
         e = raises(NotImplementedError, lib.g, 0)
         assert str(e.value) == (
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -213,4 +213,3 @@
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
     # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BASE')
-
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -5,7 +5,7 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
 
-only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
@@ -31,10 +31,10 @@
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
         w_mv = api.PyMemoryView_FromBuffer(view)
-        for f in ('format', 'itemsize', 'ndim', 'readonly', 
+        for f in ('format', 'itemsize', 'ndim', 'readonly',
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)
-            assert space.eq_w(space.getattr(w_mv, w_f), 
+            assert space.eq_w(space.getattr(w_mv, w_f),
                               space.getattr(w_memoryview, w_f))
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
@@ -70,11 +70,6 @@
         assert y.shape == (10,)
         assert len(y) == 10
         assert y[3] == 3
-        s = y[3]
-        assert len(s) == struct.calcsize('i')
-        assert s == struct.pack('i', 3)
-        viewlen = module.test_buffer(arr)
-        assert viewlen == y.itemsize * len(y)
 
     def test_buffer_protocol_capi(self):
         foo = self.import_extension('foo', [
@@ -128,15 +123,15 @@
         shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
         assert strides[-1] == 8
         dt1 = np.dtype(
-             [('a', 'b'), ('b', 'i'), 
-              ('sub0', np.dtype('b,i')), 
-              ('sub1', np.dtype('b,i')), 
-              ('sub2', np.dtype('b,i')), 
-              ('sub3', np.dtype('b,i')), 
-              ('sub4', np.dtype('b,i')), 
-              ('sub5', np.dtype('b,i')), 
-              ('sub6', np.dtype('b,i')), 
-              ('sub7', np.dtype('b,i')), 
+             [('a', 'b'), ('b', 'i'),
+              ('sub0', np.dtype('b,i')),
+              ('sub1', np.dtype('b,i')),
+              ('sub2', np.dtype('b,i')),
+              ('sub3', np.dtype('b,i')),
+              ('sub4', np.dtype('b,i')),
+              ('sub5', np.dtype('b,i')),
+              ('sub6', np.dtype('b,i')),
+              ('sub7', np.dtype('b,i')),
               ('c', 'i')],
              )
         x = np.arange(dt1.itemsize, dtype='int8').view(dt1)

From pypy.commits at gmail.com  Tue Jan  3 02:19:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 23:19:53 -0800 (PST)
Subject: [pypy-commit] pypy default: Next attempt at 7a93d726ec2a
Message-ID: <586b5099.0bba1c0a.43c1d.1a64@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89325:9104ca035a56
Date: 2017-01-03 08:19 +0100
http://bitbucket.org/pypy/pypy/changeset/9104ca035a56/

Log:	Next attempt at 7a93d726ec2a

diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
--- a/rpython/rlib/test/test_rawrefcount_boehm.py
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -3,17 +3,26 @@
 from rpython.tool.udir import udir
 
 
+def compile_test(basename):
+    srcdir = os.path.dirname(os.path.dirname(
+        os.path.abspath(os.path.join(__file__))))
+    srcdir = os.path.join(srcdir, 'src')
+
+    err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o %s %s.c"
+                    % (udir, srcdir, basename, basename))
+    return err
+
 def setup_module():
     filename = str(udir.join("test-rawrefcount-boehm-check.c"))
     with open(filename, "w") as f:
         print >> f, '#include "gc/gc_mark.h"'
-        print >> f, 'void *testing(void) {'
-        print >> f, '    return &GC_set_start_callback;'
+        print >> f, '#include <stdio.h>'
+        print >> f, 'int main(void) {'
+        print >> f, '    printf("%p", &GC_set_start_callback);'
+        print >> f, '    return 0;'
         print >> f, '}'
 
-    err = os.system("cd '%s' && gcc -c test-rawrefcount-boehm-check.c"
-                    % (udir,))
-    if err != 0:
+    if compile_test("test-rawrefcount-boehm-check") != 0:
         py.test.skip("Boehm GC not installed or too old version")
 
 
@@ -180,13 +189,9 @@
         print >> f, code
         print >> f, '}'
 
-    srcdir = os.path.dirname(os.path.dirname(
-        os.path.abspath(os.path.join(__file__))))
-    srcdir = os.path.join(srcdir, 'src')
-
-    err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o test-rawrefcount-boehm"
-                    " test-rawrefcount-boehm.c" % (udir, srcdir))
-    assert err == 0
+    err = compile_test("test-rawrefcount-boehm")
+    if err != 0:
+        raise OSError("gcc failed")
     p = subprocess.Popen("./test-rawrefcount-boehm", stdout=subprocess.PIPE,
                          cwd=str(udir))
     stdout, _ = p.communicate()

From pypy.commits at gmail.com  Tue Jan  3 02:25:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 02 Jan 2017 23:25:27 -0800 (PST)
Subject: [pypy-commit] pypy default: Move TestBoehmTranslated into
 test_rawrefcount_boehm.py, where it
Message-ID: <586b51e7.d32f1c0a.9e8ad.0a2a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89326:57ce16a5cccf
Date: 2017-01-03 08:24 +0100
http://bitbucket.org/pypy/pypy/changeset/57ce16a5cccf/

Log:	Move TestBoehmTranslated into test_rawrefcount_boehm.py, where it
	belongs, to be properly skipped by setup_module() if needed

diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -1,7 +1,7 @@
 import weakref
 from rpython.rlib import rawrefcount, objectmodel, rgc
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT
-from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.translator.c.test.test_standalone import StandaloneTests
 from rpython.config.translationoption import get_combined_translation_config
@@ -286,55 +286,3 @@
         t, cbuilder = self.compile(entry_point)
         data = cbuilder.cmdexec('hi there')
         assert data.startswith('OK!\n')
-
-
-class TestBoehmTranslated(StandaloneTests):
-
-    def test_full_translation(self):
-
-        def make_ob():
-            p = W_Root(42)
-            ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
-            rawrefcount.create_link_pypy(p, ob)
-            ob.c_ob_refcnt += REFCNT_FROM_PYPY
-            assert rawrefcount.from_obj(PyObject, p) == ob
-            assert rawrefcount.to_obj(W_Root, ob) == p
-            return ob
-
-        prebuilt_p = W_Root(-42)
-        prebuilt_ob = lltype.malloc(PyObjectS, flavor='raw', zero=True,
-                                    immortal=True)
-
-        def entry_point(argv):
-            rawrefcount.create_link_pypy(prebuilt_p, prebuilt_ob)
-            prebuilt_ob.c_ob_refcnt += REFCNT_FROM_PYPY
-            oblist = [make_ob() for i in range(50)]
-            rgc.collect()
-            deadlist = []
-            while True:
-                ob = rawrefcount.next_dead(PyObject)
-                if not ob: break
-                if ob.c_ob_refcnt != 1:
-                    print "next_dead().ob_refcnt != 1"
-                    return 1
-                deadlist.append(ob)
-            if len(deadlist) == 0:
-                print "no dead object"
-                return 1
-            if len(deadlist) < 30:
-                print "not enough dead objects"
-                return 1
-            for ob in deadlist:
-                if ob not in oblist:
-                    print "unexpected value for dead pointer"
-                    return 1
-                oblist.remove(ob)
-            print "OK!"
-            lltype.free(ob, flavor='raw')
-            return 0
-
-        self.config = get_combined_translation_config(translating=True)
-        self.config.translation.gc = "boehm"
-        t, cbuilder = self.compile(entry_point)
-        data = cbuilder.cmdexec('hi there')
-        assert data.startswith('OK!\n')
diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
--- a/rpython/rlib/test/test_rawrefcount_boehm.py
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -1,6 +1,12 @@
 import itertools, os, subprocess, py
 from hypothesis import given, strategies
 from rpython.tool.udir import udir
+from rpython.rlib import rawrefcount, rgc
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.test.test_rawrefcount import W_Root, PyObject, PyObjectS
+from rpython.rtyper.lltypesystem import lltype
+from rpython.translator.c.test.test_standalone import StandaloneTests
+from rpython.config.translationoption import get_combined_translation_config
 
 
 def compile_test(basename):
@@ -248,3 +254,55 @@
             del links_p2g[p]
         else:
             assert False, repr(line)
+
+
+class TestBoehmTranslated(StandaloneTests):
+
+    def test_full_translation(self):
+
+        def make_ob():
+            p = W_Root(42)
+            ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+            rawrefcount.create_link_pypy(p, ob)
+            ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            assert rawrefcount.from_obj(PyObject, p) == ob
+            assert rawrefcount.to_obj(W_Root, ob) == p
+            return ob
+
+        prebuilt_p = W_Root(-42)
+        prebuilt_ob = lltype.malloc(PyObjectS, flavor='raw', zero=True,
+                                    immortal=True)
+
+        def entry_point(argv):
+            rawrefcount.create_link_pypy(prebuilt_p, prebuilt_ob)
+            prebuilt_ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            oblist = [make_ob() for i in range(50)]
+            rgc.collect()
+            deadlist = []
+            while True:
+                ob = rawrefcount.next_dead(PyObject)
+                if not ob: break
+                if ob.c_ob_refcnt != 1:
+                    print "next_dead().ob_refcnt != 1"
+                    return 1
+                deadlist.append(ob)
+            if len(deadlist) == 0:
+                print "no dead object"
+                return 1
+            if len(deadlist) < 30:
+                print "not enough dead objects"
+                return 1
+            for ob in deadlist:
+                if ob not in oblist:
+                    print "unexpected value for dead pointer"
+                    return 1
+                oblist.remove(ob)
+            print "OK!"
+            lltype.free(ob, flavor='raw')
+            return 0
+
+        self.config = get_combined_translation_config(translating=True)
+        self.config.translation.gc = "boehm"
+        t, cbuilder = self.compile(entry_point)
+        data = cbuilder.cmdexec('hi there')
+        assert data.startswith('OK!\n')

From pypy.commits at gmail.com  Tue Jan  3 03:27:43 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 00:27:43 -0800 (PST)
Subject: [pypy-commit] cffi strbuf-as-buffer: readd two lines accidentaly
 removed
Message-ID: <586b607f.0bba1c0a.43c1d.32dc@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: strbuf-as-buffer
Changeset: r2850:b301c37c91bb
Date: 2017-01-03 09:27 +0100
http://bitbucket.org/cffi/cffi/changeset/b301c37c91bb/

Log:	readd two lines accidentaly removed

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3426,6 +3426,8 @@
     BCharA = new_array_type(BCharP, None)
     p1 = from_buffer(BCharA, b"foo")
     assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer

From pypy.commits at gmail.com  Tue Jan  3 03:52:23 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 00:52:23 -0800 (PST)
Subject: [pypy-commit] pypy strbuf-as-buffer: added comments and remove typo,
 update error message
Message-ID: <586b6647.07941c0a.61a51.4bb9@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: strbuf-as-buffer
Changeset: r89327:a22f7d7e919f
Date: 2017-01-03 09:51 +0100
http://bitbucket.org/pypy/pypy/changeset/a22f7d7e919f/

Log:	added comments and remove typo, update error message

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -88,9 +88,12 @@
         if isinstance(obj, (str, unicode)):
             # hack, buffer(str) will always return a readonly buffer.
             # CPython calls PyObject_AsWriteBuffer(...) here!
-            # str cannot be modified, thus rase a type error in this case
+            # str cannot be modified, thus raise a type error in this case
             raise TypeError("Cannot use %s as modifiable buffer" % str(type(obj)))
 
+        # why not just call memoryview(obj)[offset:]?
+        # array in Python 2.7 does not support the buffer protocol and will
+        # fail, even though buffer is supported
         buf = buffer(obj, offset, size)
 
         if len(buf) < size:
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -143,8 +143,7 @@
 def _from_buffer(space, w_ctype, w_x):
     if invalid_input_buffer_type(space, w_x):
         raise oefmt(space.w_TypeError,
-                        "from_buffer() cannot return the address of the "
-                        "raw string within a str or unicode object")
+                        "from_buffer() cannot return the address a unicode")
     buf = _fetch_as_read_buffer(space, w_x)
     if space.isinstance_w(w_x, space.w_str):
         _cdata = get_raw_address_of_string(space, w_x)

From pypy.commits at gmail.com  Tue Jan  3 03:54:08 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 00:54:08 -0800 (PST)
Subject: [pypy-commit] pypy strbuf-as-buffer: merge default
Message-ID: <586b66b0.ca10c20a.b0762.d745@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: strbuf-as-buffer
Changeset: r89328:494bd552b4dd
Date: 2017-01-03 09:53 +0100
http://bitbucket.org/pypy/pypy/changeset/494bd552b4dd/

Log:	merge default

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -76,3 +76,8 @@
 PyMemoryViewObject with a PyBuffer attached so that the call to 
 ``PyMemoryView_GET_BUFFER`` does not leak a PyBuffer-sized piece of memory.
 Properly call ``bf_releasebuffer`` when not ``NULL``.
+
+.. branch: boehm-rawrefcount
+
+Support translations of cpyext with the Boehm GC (for special cases like
+revdb).
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -305,9 +305,9 @@
             config.objspace.lonepycfiles = False
 
         if config.objspace.usemodules.cpyext:
-            if config.translation.gc != 'incminimark':
+            if config.translation.gc not in ('incminimark', 'boehm'):
                 raise Exception("The 'cpyext' module requires the 'incminimark'"
-                                " GC.  You need either 'targetpypystandalone.py"
+                                " 'boehm' GC.  You need either 'targetpypystandalone.py"
                                 " --withoutmod-cpyext' or '--gc=incminimark'")
 
         config.translating = True
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -1,6 +1,6 @@
 from rpython.rlib import rsocket
 from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, r_longlong, r_uint32
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -98,7 +98,8 @@
         proto = space.str_w(w_proto)
 
     if port < 0 or port > 0xffff:
-        raise oefmt(space.w_ValueError, "getservbyport: port must be 0-65535.")
+        raise oefmt(space.w_OverflowError,
+                    "getservbyport: port must be 0-65535.")
 
     try:
         service = rsocket.getservbyport(port, proto)
@@ -163,40 +164,58 @@
         space.wrap(W_Socket(space, sock2))
     ])
 
-# The following 4 functions refuse all negative numbers, like CPython 2.6.
-# They could also check that the argument is not too large, but CPython 2.6
-# is not doing that consistently.
- at unwrap_spec(x="c_uint")
+# The following 4 functions refuse all negative numbers.
+# They also check that the argument is not too large, but note that
+# CPython 2.7 is not doing that consistently (CPython 3.x does).
+LONGLONG_UINT32_MAX = r_longlong(2**32-1)
+
+ at unwrap_spec(x="c_int")
 def ntohs(space, x):
     """ntohs(integer) -> integer
 
     Convert a 16-bit integer from network to host byte order.
     """
+    if x < 0:
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
     return space.wrap(rsocket.ntohs(intmask(x)))
 
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x=r_longlong)
 def ntohl(space, x):
     """ntohl(integer) -> integer
 
     Convert a 32-bit integer from network to host byte order.
     """
-    return space.wrap(rsocket.ntohl(x))
+    if x < r_longlong(0):
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
+    if x > LONGLONG_UINT32_MAX:
+        raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
+    return space.wrap(rsocket.ntohl(r_uint32(x)))
 
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x="c_int")
 def htons(space, x):
     """htons(integer) -> integer
 
     Convert a 16-bit integer from host to network byte order.
     """
-    return space.wrap(rsocket.htons(intmask(x)))
+    if x < 0:
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
+    return space.wrap(rsocket.htons(x))
 
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x=r_longlong)
 def htonl(space, x):
     """htonl(integer) -> integer
 
     Convert a 32-bit integer from host to network byte order.
     """
-    return space.wrap(rsocket.htonl(x))
+    if x < r_longlong(0):
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
+    if x > LONGLONG_UINT32_MAX:
+        raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
+    return space.wrap(rsocket.htonl(r_uint32(x)))
 
 @unwrap_spec(ip=str)
 def inet_aton(space, ip):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -83,11 +83,6 @@
                          "(_socket, port): return _socket.getservbyport(port)")
     assert space.unwrap(name) == "smtp"
 
-    from pypy.interpreter.error import OperationError
-    exc = raises(OperationError, space.appexec,
-           [w_socket], "(_socket): return _socket.getservbyport(-1)")
-    assert exc.value.match(space, space.w_ValueError)
-
 def test_getprotobyname():
     name = "tcp"
     w_n = space.appexec([w_socket, space.wrap(name)],
@@ -325,6 +320,11 @@
         assert _socket.socket.__name__ == 'socket'
         assert _socket.socket.__module__ == '_socket'
 
+    def test_overflow_errors(self):
+        import _socket
+        raises(OverflowError, _socket.getservbyport, -1)
+        raises(OverflowError, _socket.getservbyport, 65536)
+
     def test_ntoa_exception(self):
         import _socket
         raises(_socket.error, _socket.inet_ntoa, b"ab")
@@ -495,7 +495,8 @@
     def test_socket_connect_typeerrors(self):
         tests = [
             "",
-            ("80"),
+            "80",
+            ("80",),
             ("80", "80"),
             (80, 80),
         ]
@@ -519,44 +520,25 @@
     def test_NtoH(self):
         import sys
         import _socket as socket
-        # This just checks that htons etc. are their own inverse,
-        # when looking at the lower 16 or 32 bits.
+        # This checks that htons etc. are their own inverse,
+        # when looking at the lower 16 or 32 bits.  It also
+        # checks that we get OverflowErrors when calling with -1,
+        # or (for XtoXl()) with too large values.  For XtoXs()
+        # large values are silently truncated instead, like CPython.
         sizes = {socket.htonl: 32, socket.ntohl: 32,
                  socket.htons: 16, socket.ntohs: 16}
         for func, size in sizes.items():
             mask = (1 << size) - 1
-            for i in (0, 1, 0xffff, ~0xffff, 2, 0x01234567, 0x76543210):
+            for i in (0, 1, 0xffff, 0xffff0000, 2, 0x01234567, 0x76543210):
                 assert i & mask == func(func(i&mask)) & mask
 
             swapped = func(mask)
             assert swapped & mask == mask
-            try:
-                func(-1)
-            except (OverflowError, ValueError):
-                pass
-            else:
-                assert False
-            try:
-                func(sys.maxint*2+2)
-            except OverflowError:
-                pass
-            else:
-                assert False
-
-    def test_NtoH_overflow(self):
-        skip("we are not checking for overflowing values yet")
-        import _socket as socket
-        # Checks that we cannot give too large values to htons etc.
-        # Skipped for now; CPython 2.6 is also not consistent.
-        sizes = {socket.htonl: 32, socket.ntohl: 32,
-                 socket.htons: 16, socket.ntohs: 16}
-        for func, size in sizes.items():
-            try:
-                func(1 << size)
-            except OverflowError:
-                pass
-            else:
-                assert False
+            raises(OverflowError, func, -1)
+            raises(OverflowError, func, -1L)
+            if size > 16:    # else, values too large are ignored
+                raises(OverflowError, func, 2 ** size)
+                raises(OverflowError, func, 2L ** size)
 
     def test_newsocket(self):
         import socket
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -2,12 +2,6 @@
 
 @py.test.mark.tryfirst
 def pytest_runtest_setup(item):
-    if 'linux' in sys.platform:
-        # tests require minimally std=c++11
-        cc_info = py.process.cmdexec('gcc -v --help')
-        if not '-std=c++11' in cc_info:
-            py.test.skip('skipping tests because gcc does not support C++11')
-
     if py.path.local.sysfind('genreflex') is None:
         import pypy.module.cppyy.capi.loadable_capi as lcapi
         if 'dummy' in lcapi.reflection_library:
diff --git a/pypy/module/cppyy/test/support.py b/pypy/module/cppyy/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cppyy/test/support.py
@@ -0,0 +1,16 @@
+import py, sys, subprocess
+
+currpath = py.path.local(__file__).dirpath()
+
+
+def setup_make(targetname):
+    if sys.platform == 'win32':
+        py.test.skip("win32 not supported so far")
+    import pypy.module.cppyy.capi.loadable_capi as lcapi
+    popen = subprocess.Popen(["make", targetname], cwd=str(currpath),
+                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    stdout, _ = popen.communicate()
+    if popen.returncode:
+        if '-std=c++11' in stdout:
+            py.test.skip("gcc does not seem to support -std=c++11")
+        raise OSError("'make' failed:\n%s" % (stdout,))
diff --git a/pypy/module/cppyy/test/test_cppyy.py b/pypy/module/cppyy/test/test_cppyy.py
--- a/pypy/module/cppyy/test/test_cppyy.py
+++ b/pypy/module/cppyy/test/test_cppyy.py
@@ -1,18 +1,15 @@
 import py, os, sys
+import subprocess
 
 from pypy.module.cppyy import interp_cppyy, executor
+from .support import setup_make
 
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict.so"))
 
 def setup_module(mod):
-    if sys.platform == 'win32':
-        py.test.skip("win32 not supported so far")
-    import pypy.module.cppyy.capi.loadable_capi as lcapi
-    err = os.system("cd '%s' && make example01Dict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    setup_make("example01Dict.so")
 
 class TestCPPYYImplementation:
     def test01_class_query(self, space):
diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py
--- a/pypy/module/cppyy/test/test_datatypes.py
+++ b/pypy/module/cppyy/test/test_datatypes.py
@@ -1,15 +1,12 @@
 import py, os, sys
+from .support import setup_make
 
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("datatypesDict.so"))
 
 def setup_module(mod):
-    if sys.platform == 'win32':
-        py.test.skip("win32 not supported so far")
-    err = os.system("cd '%s' && make datatypesDict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    setup_make("datatypesDict.so")
 
 class AppTestDATATYPES:
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py
--- a/pypy/module/cppyy/test/test_pythonify.py
+++ b/pypy/module/cppyy/test/test_pythonify.py
@@ -1,17 +1,14 @@
 import py, os, sys
 
 from pypy.module.cppyy import interp_cppyy, executor
+from .support import setup_make
 
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict.so"))
 
 def setup_module(mod):
-    if sys.platform == 'win32':
-        py.test.skip("win32 not supported so far")
-    err = os.system("cd '%s' && make example01Dict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    setup_make("example01Dict.so")
 
 class AppTestPYTHONIFY:
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1,6 +1,5 @@
 import ctypes
 import sys, os
-import atexit
 
 import py
 
@@ -18,7 +17,6 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import NameManager
 from rpython.tool.udir import udir
-from rpython.translator import platform
 from pypy.module.cpyext.state import State
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.baseobjspace import W_Root
@@ -484,6 +482,12 @@
     TYPES[configname] = forward
     return forward
 
+GLOBALS = {}
+def register_global(name, typ, expr, header=None):
+    if header is not None:
+        name = '%s#%s' % (name, header)
+    GLOBALS[name] = (typ, expr)
+
 INTERPLEVEL_API = {}
 FUNCTIONS = {}
 FUNCTIONS_BY_HEADER = {}
@@ -544,18 +548,23 @@
     '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext',
 ]
 TYPES = {}
-GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur
-    '_Py_NoneStruct#%s' % pypy_decl: ('PyObject*', 'space.w_None'),
-    '_Py_TrueStruct#%s' % pypy_decl: ('PyIntObject*', 'space.w_True'),
-    '_Py_ZeroStruct#%s' % pypy_decl: ('PyIntObject*', 'space.w_False'),
-    '_Py_NotImplementedStruct#%s' % pypy_decl: ('PyObject*', 'space.w_NotImplemented'),
-    '_Py_EllipsisObject#%s' % pypy_decl: ('PyObject*', 'space.w_Ellipsis'),
-    'PyDateTimeAPI': ('PyDateTime_CAPI*', 'None'),
-    }
 FORWARD_DECLS = []
 INIT_FUNCTIONS = []
 BOOTSTRAP_FUNCTIONS = []
 
+# this needs to include all prebuilt pto, otherwise segfaults occur
+register_global('_Py_NoneStruct',
+    'PyObject*', 'space.w_None', header=pypy_decl)
+register_global('_Py_TrueStruct',
+    'PyIntObject*', 'space.w_True', header=pypy_decl)
+register_global('_Py_ZeroStruct',
+    'PyIntObject*', 'space.w_False', header=pypy_decl)
+register_global('_Py_NotImplementedStruct',
+    'PyObject*', 'space.w_NotImplemented', header=pypy_decl)
+register_global('_Py_EllipsisObject',
+    'PyObject*', 'space.w_Ellipsis', header=pypy_decl)
+register_global('PyDateTimeAPI', 'PyDateTime_CAPI*', 'None')
+
 def build_exported_objects():
     # Standard exceptions
     # PyExc_BaseException, PyExc_Exception, PyExc_ValueError, PyExc_KeyError,
@@ -564,7 +573,7 @@
     # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError,
     # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ...
     for exc_name in exceptions.Module.interpleveldefs.keys():
-        GLOBALS['PyExc_' + exc_name] = (
+        register_global('PyExc_' + exc_name,
             'PyTypeObject*',
             'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, ))
 
@@ -599,7 +608,7 @@
         'PyCFunction_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)',
         'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)'
         }.items():
-        GLOBALS['%s#%s' % (cpyname, pypy_decl)] = ('PyTypeObject*', pypyexpr)
+        register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
 
     for cpyname in '''PyMethodObject PyListObject PyLongObject
                       PyClassObject'''.split():
@@ -1019,14 +1028,12 @@
 def build_bridge(space):
     "NOT_RPYTHON"
     from pypy.module.cpyext.pyobject import make_ref
+    from rpython.translator.c.database import LowLevelDatabase
+    use_micronumpy = setup_micronumpy(space)
+    db = LowLevelDatabase()
+    prefix ='cpyexttest'
 
-    use_micronumpy = setup_micronumpy(space)
-
-    export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS)
-    from rpython.translator.c.database import LowLevelDatabase
-    db = LowLevelDatabase()
-
-    generate_macros(export_symbols, prefix='cpyexttest')
+    functions = generate_decls_and_callbacks(db, prefix=prefix)
 
     # Structure declaration code
     members = []
@@ -1047,9 +1054,6 @@
     RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
-    functions = generate_decls_and_callbacks(db, export_symbols,
-                                            prefix='cpyexttest')
-
     global_objects = []
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
@@ -1076,7 +1080,7 @@
             '\n' +
             '\n'.join(functions))
 
-    eci = build_eci(True, export_symbols, code, use_micronumpy)
+    eci = build_eci(True, code, use_micronumpy)
     eci = eci.compile_shared_lib(
         outputfilename=str(udir / "module_cache" / "pypyapi"))
     modulename = py.path.local(eci.libraries[-1])
@@ -1097,7 +1101,6 @@
     run_bootstrap_functions(space)
 
     # load the bridge, and init structure
-    import ctypes
     bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
 
     space.fromcache(State).install_dll(eci)
@@ -1117,7 +1120,7 @@
 
         INTERPLEVEL_API[name] = w_obj
 
-        name = name.replace('Py', 'cpyexttest')
+        name = name.replace('Py', prefix)
         if isptr:
             ptr = ctypes.c_void_p.in_dll(bridge, name)
             if typ == 'PyObject*':
@@ -1145,12 +1148,6 @@
     pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
 
     # implement structure initialization code
-    #for name, func in FUNCTIONS.iteritems():
-    #    if name.startswith('cpyext_'): # XXX hack
-    #        continue
-    #    pypyAPI[structindex[name]] = ctypes.cast(
-    #        ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
-    #        ctypes.c_void_p)
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
             if name.startswith('cpyext_') or func is None: # XXX hack
@@ -1218,13 +1215,13 @@
     else:
         return None
 
-def generate_macros(export_symbols, prefix):
+def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
-    renamed_symbols = []
+    export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
     for name in export_symbols:
         if '#' in name:
-            name,header = name.split('#')
+            name, header = name.split('#')
         else:
             header = pypy_decl
         newname = mangle_name(prefix, name)
@@ -1233,8 +1230,6 @@
             pypy_macros.append('#define %s %s' % (name, newname))
         if name.startswith("PyExc_"):
             pypy_macros.append('#define _%s _%s' % (name, newname))
-        renamed_symbols.append(newname)
-    export_symbols[:] = renamed_symbols
 
     # Generate defines
     for macro_name, size in [
@@ -1254,8 +1249,6 @@
     pypy_macros_h = udir.join('pypy_macros.h')
     pypy_macros_h.write('\n'.join(pypy_macros))
 
-def generate_decls_and_callbacks(db, export_symbols, api_struct=True, prefix=''):
-    "NOT_RPYTHON"
     # implement function callbacks and generate function decls
     functions = []
     decls = {}
@@ -1341,7 +1334,7 @@
                          source_dir / "pymem.c",
                          ]
 
-def build_eci(building_bridge, export_symbols, code, use_micronumpy=False):
+def build_eci(building_bridge, code, use_micronumpy=False):
     "NOT_RPYTHON"
     # Build code and get pointer to the structure
     kwds = {}
@@ -1410,31 +1403,29 @@
         return use_micronumpy
     # import registers api functions by side-effect, we also need HEADER
     from pypy.module.cpyext.ndarrayobject import HEADER
-    global GLOBALS, FUNCTIONS_BY_HEADER, separate_module_files
+    global FUNCTIONS_BY_HEADER, separate_module_files
     for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
         FUNCTIONS_BY_HEADER.setdefault(HEADER, {})[func_name] = None
-    GLOBALS["PyArray_Type#%s" % HEADER] = ('PyTypeObject*', "space.gettypeobject(W_NDimArray.typedef)")
+    register_global("PyArray_Type",
+        'PyTypeObject*',  "space.gettypeobject(W_NDimArray.typedef)",
+        header=HEADER)
     separate_module_files.append(source_dir / "ndarrayobject.c")
     return use_micronumpy
 
 def setup_library(space):
     "NOT_RPYTHON"
+    from rpython.translator.c.database import LowLevelDatabase
     use_micronumpy = setup_micronumpy(space)
-    export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
-    from rpython.translator.c.database import LowLevelDatabase
     db = LowLevelDatabase()
     prefix = 'PyPy'
 
-    generate_macros(export_symbols, prefix=prefix)
-
-    functions = generate_decls_and_callbacks(db, [], api_struct=False,
-                                            prefix=prefix)
+    functions = generate_decls_and_callbacks(db, api_struct=False, prefix=prefix)
     code = "#include <Python.h>\n"
     if use_micronumpy:
         code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
     code  += "\n".join(functions)
 
-    eci = build_eci(False, export_symbols, code, use_micronumpy)
+    eci = build_eci(False, code, use_micronumpy)
 
     space.fromcache(State).install_dll(eci)
 
@@ -1586,7 +1577,7 @@
 
 @specialize.memo()
 def make_generic_cpy_call(FT, expect_null):
-    from pypy.module.cpyext.pyobject import make_ref, from_ref, Py_DecRef
+    from pypy.module.cpyext.pyobject import make_ref, from_ref
     from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
     from pypy.module.cpyext.pyobject import get_w_obj_and_decref
     from pypy.module.cpyext.pyerrors import PyErr_Occurred
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -1,7 +1,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.executioncontext import AsyncAction
+from pypy.interpreter import executioncontext
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
@@ -14,8 +14,9 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
-        pyobj_dealloc_action = PyObjDeallocAction(space)
-        self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+        if space.config.translation.gc != "boehm":
+            pyobj_dealloc_action = PyObjDeallocAction(space)
+            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -67,6 +68,11 @@
             state.api_lib = str(api.build_bridge(self.space))
         else:
             api.setup_library(self.space)
+            #
+            if self.space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(self.space)
+                self.space.actionflag.register_periodic_action(action,
+                    use_bytecode_counter=True)
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -84,8 +90,10 @@
         from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
-            rawrefcount.init(llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
-                                      self.dealloc_trigger))
+            if space.config.translation.gc != "boehm":
+                rawrefcount.init(
+                    llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
+                    self.dealloc_trigger))
             init_static_data_translated(space)
 
         setup_new_method_def(space)
@@ -143,15 +151,23 @@
         self.extensions[path] = w_copy
 
 
-class PyObjDeallocAction(AsyncAction):
+def _rawrefcount_perform(space):
+    from pypy.module.cpyext.pyobject import PyObject, decref
+    while True:
+        py_obj = rawrefcount.next_dead(PyObject)
+        if not py_obj:
+            break
+        decref(space, py_obj)
+
+class PyObjDeallocAction(executioncontext.AsyncAction):
     """An action that invokes _Py_Dealloc() on the dying PyObjects.
     """
+    def perform(self, executioncontext, frame):
+        _rawrefcount_perform(self.space)
 
+class BoehmPyObjDeallocAction(executioncontext.PeriodicAsyncAction):
+    # This variant is used with Boehm, which doesn't have the explicit
+    # callback.  Instead we must periodically check ourselves.
     def perform(self, executioncontext, frame):
-        from pypy.module.cpyext.pyobject import PyObject, decref
-
-        while True:
-            py_obj = rawrefcount.next_dead(PyObject)
-            if not py_obj:
-                break
-            decref(self.space, py_obj)
+        if we_are_translated():
+            _rawrefcount_perform(self.space)
diff --git a/pypy/module/cpyext/stubgen.py b/pypy/module/cpyext/stubgen.py
deleted file mode 100644
--- a/pypy/module/cpyext/stubgen.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# -*- coding: utf-8 -*-
-from os import path
-
-from pypy.module.cpyext import api
-
-from sphinx import addnodes
-
-
-TEMPLATE = """
- at cpython_api([%(paramtypes)s], %(rettype)s)
-def %(functionname)s(%(params)s):
-%(docstring)s    raise NotImplementedError
-    %(borrows)s
-"""
-
-C_TYPE_TO_PYPY_TYPE = {
-        "void": "lltype.Void",
-        "int": "rffi.INT_real",
-        "PyTypeObject*": "PyTypeObjectPtr",
-        "PyVarObject*": "PyObject",
-        "const char*": "rffi.CCHARP",
-        "double": "rffi.DOUBLE",
-        "PyObject*": "PyObject",
-        "PyObject**": "PyObjectP",
-        "char*": "rffi.CCHARP",
-        "PyMethodDef*": "PyMethodDef",
-        "Py_ssize_t": "Py_ssize_t",
-        "Py_ssize_t*": "Py_ssize_t",
-        "size_t": "rffi.SIZE_T",
-        "...": "...",
-        "char": "lltype.Char",
-        "long": "lltype.Signed",
-        "Py_buffer*": "Py_buffer",
-        "": "",
-        }
-
-C_TYPE_TO_PYPY_TYPE_ARGS = C_TYPE_TO_PYPY_TYPE.copy()
-C_TYPE_TO_PYPY_TYPE_ARGS.update({
-    "void": "rffi.VOIDP",
-    })
-
-
-def c_param_to_type_and_name(string, is_arg=True):
-    string = string.replace(" **", "** ").replace(" *", "* ")
-    try:
-        typ, name = string.rsplit(" ", 1)
-    except ValueError:
-        typ = string
-        name = ""
-    return [C_TYPE_TO_PYPY_TYPE, C_TYPE_TO_PYPY_TYPE_ARGS][is_arg]\
-            .get(typ, "{" + typ + "}"), name
-
-
-def process_doctree(app, doctree):
-    for node in doctree.traverse(addnodes.desc_content):
-        par = node.parent
-        if par['desctype'] != 'cfunction':
-            continue
-        if not par[0].has_key('names') or not par[0]['names']:
-            continue
-        functionname = par[0]['names'][0]
-        if (functionname in api.FUNCTIONS or
-            functionname in api.SYMBOLS_C):
-            print "Wow, you implemented already", functionname
-            continue
-        borrows = docstring = ""
-        crettype, _, cparameters = par[0]
-        crettype = crettype.astext()
-        cparameters = cparameters.astext()
-        rettype, _ = c_param_to_type_and_name(crettype, False)
-        params = ["space"]
-        paramtypes = []
-        for param in cparameters.split(","):
-            typ, name = c_param_to_type_and_name(param.strip())
-            params.append(name)
-            paramtypes.append(typ)
-        params = ", ".join(params)
-        paramtypes = ", ".join(paramtypes)
-        docstring = node.astext()
-        entry = app._refcounts.get(functionname)
-        if entry and entry.result_type in ("PyObject*", "PyVarObject*"):
-            if entry.result_refs is None:
-                docstring += "\nReturn value: always NULL."
-            else:
-                borrows = ("borrow_from()", "")[entry.result_refs]
-        docstring = "\n    ".join(docstring.splitlines())
-        if docstring:
-            docstring = '    """%s"""\n' % (docstring,)
-        code = TEMPLATE % locals()
-        app._stubgen_f.write(code)
-
-
-def init_apidump(app):
-    fname = path.join(path.dirname(api.__file__), "stubs.py")
-    app._stubgen_f = file(fname, "w")
-    app.connect('doctree-read', process_doctree)
-
-
-def setup(app):
-    app.connect('builder-inited', init_apidump)
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -36,6 +36,30 @@
             assert space.eq_w(space.getattr(w_mv, w_f), 
                               space.getattr(w_memoryview, w_f))
 
+
+class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
+    def test_fillWithObject(self):
+        module = self.import_extension('foo', [
+                ("fillinfo", "METH_VARARGS",
+                 """
+                 Py_buffer buf;
+                 PyObject *str = PyBytes_FromString("hello, world.");
+                 if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
+                                       0, 0)) {
+                     return NULL;
+                 }
+
+                 /* Get rid of our own reference to the object, but
+                  * the Py_buffer should still have a reference.
+                  */
+                 Py_DECREF(str);
+
+                 return PyMemoryView_FromBuffer(&buf);
+                 """)])
+        result = module.fillinfo()
+        assert b"hello, world." == result
+        del result
+
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol_app(self):
         import struct
@@ -62,7 +86,7 @@
                     return NULL;
                 vlen = view.len / view.itemsize;
                 PyBuffer_Release(&view);
-                return PyInt_FromLong(vlen);
+                return PyLong_FromLong(vlen);
              """),
             ("test_buffer", "METH_VARARGS",
              """
@@ -70,10 +94,10 @@
                 PyObject* obj = PyTuple_GetItem(args, 0);
                 PyObject* memoryview = PyMemoryView_FromObject(obj);
                 if (memoryview == NULL)
-                    return PyInt_FromLong(-1);
+                    return PyLong_FromLong(-1);
                 view = PyMemoryView_GET_BUFFER(memoryview);
                 Py_DECREF(memoryview);
-                return PyInt_FromLong(view->len / view->itemsize);
+                return PyLong_FromLong(view->len / view->itemsize);
             """)])
         module = self.import_module(name='buffer_test')
         arr = module.PyMyArray(10)
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -4,10 +4,11 @@
 #  This is meant for pypy's cpyext module, but is a generally
 #  useful interface over our GC.  XXX "pypy" should be removed here
 #
-import sys, weakref
-from rpython.rtyper.lltypesystem import lltype, llmemory
+import sys, weakref, py
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.objectmodel import we_are_translated, specialize, not_rpython
 from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib import rgc
 
 
@@ -245,6 +246,11 @@
         v_p, v_ob = hop.inputargs(*hop.args_r)
         hop.exception_cannot_occur()
         hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)])
+        #
+        if hop.rtyper.annotator.translator.config.translation.gc == "boehm":
+            c_func = hop.inputconst(lltype.typeOf(func_boehm_eci),
+                                    func_boehm_eci)
+            hop.genop('direct_call', [c_func])
 
 
 class Entry(ExtRegistryEntry):
@@ -297,3 +303,10 @@
         v_ob = hop.genop('gc_rawrefcount_next_dead', [],
                          resulttype = llmemory.Address)
         return _spec_ob(hop, v_ob)
+
+src_dir = py.path.local(__file__).dirpath() / 'src'
+boehm_eci = ExternalCompilationInfo(
+    post_include_bits     = [(src_dir / 'boehm-rawrefcount.h').read()],
+    separate_module_files = [(src_dir / 'boehm-rawrefcount.c')],
+)
+func_boehm_eci = rffi.llexternal_use_eci(boehm_eci)
diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -0,0 +1,282 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <assert.h>
+#include <limits.h>
+#include <gc/gc.h>
+#include <gc/gc_mark.h>
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+#  define RPY_EXTERN  /* nothing */
+#else
+#  include "common_header.h"
+#endif
+
+
+#define REFCNT_FROM_PYPY  (LONG_MAX / 4 + 1)
+
+typedef struct pypy_header0 gcobj_t;    /* opaque here */
+
+#ifndef _WIN32
+typedef intptr_t Py_ssize_t;
+#else
+typedef long Py_ssize_t;
+#endif
+
+/* this is the first two words of the PyObject structure used in
+   pypy/module/cpyext */
+typedef struct {
+    Py_ssize_t ob_refcnt;
+    Py_ssize_t ob_pypy_link;
+} pyobj_t;
+
+struct link_s {
+    pyobj_t *pyobj;    /* NULL if entry unused */
+    uintptr_t gcenc;
+    struct link_s *next_in_bucket;
+};
+
+#define MARKER_LIST_START  ((pyobj_t *)-1)
+
+static struct link_s **hash_buckets, *hash_list, *hash_free_list;
+static uintptr_t hash_mask_bucket;
+static intptr_t hash_list_walk_next = -1;
+
+static uintptr_t hash_get_hash(gcobj_t *gcobj)
+{
+    assert(gcobj != NULL);
+    uintptr_t h = (uintptr_t)gcobj;
+    assert((h & 1) == 0);
+    h -= (h >> 6);
+    return h & hash_mask_bucket;
+}
+
+static gcobj_t *decode_gcenc(uintptr_t gcenc)
+{
+    if (gcenc & 1)
+        gcenc = ~gcenc;
+    return (gcobj_t *)gcenc;
+}
+
+static void hash_link(struct link_s *lnk)
+{
+    uintptr_t h = hash_get_hash(decode_gcenc(lnk->gcenc));
+    lnk->next_in_bucket = hash_buckets[h];
+    hash_buckets[h] = lnk;
+}
+
+static void boehm_is_about_to_collect(void);
+
+static void hash_grow_table(void)
+{
+    static int rec = 0;
+    assert(!rec);   /* recursive hash_grow_table() */
+    rec = 1;
+
+    if (hash_buckets == NULL)
+        GC_set_start_callback(boehm_is_about_to_collect);
+
+    uintptr_t i, num_buckets = (hash_mask_bucket + 1) * 2;
+    if (num_buckets < 16) num_buckets = 16;
+    assert((num_buckets & (num_buckets - 1)) == 0);  /* power of two */
+
+    /* The new hash_buckets: an array of pointers to struct link_s, of
+       length a power of two, used as a dictionary hash table.  It is
+       not allocated with Boehm because there is no point in Boehm looking
+       in it.
+     */
+    struct link_s **new_buckets = calloc(num_buckets, sizeof(struct link_s *));
+    assert(new_buckets);
+
+    /* The new hash_list: the array of all struct link_s.  Their order
+       is irrelevant.  There is a GC_register_finalizer() on the 'gcenc'
+       field, so we don't move the array; instead we allocate a new array
+       to use in addition to the old one.  There are a total of 2 to 4
+       times as many 'struct link_s' as the length of 'buckets'.
+     */
+    uintptr_t num_list = num_buckets * 2;
+    struct link_s *new_list = GC_MALLOC(num_list * sizeof(struct link_s));
+    for (i = num_list; i-- > 1; ) {
+        new_list[i].next_in_bucket = hash_free_list;
+        hash_free_list = &new_list[i];
+    }
+    /* list[0] is abused to store a pointer to the previous list and
+       the length of the current list */
+    struct link_s *old_list = hash_list;
+    new_list[0].next_in_bucket = old_list;
+    new_list[0].gcenc = num_list;
+    new_list[0].pyobj = MARKER_LIST_START;
+
+    hash_list = new_list;
+    free(hash_buckets);
+    hash_buckets = new_buckets;
+    hash_mask_bucket = num_buckets - 1;
+    hash_list_walk_next = hash_mask_bucket;
+
+    /* re-add all old 'struct link_s' to the hash_buckets */
+    struct link_s *plist = old_list;
+    while (plist != NULL) {
+        uintptr_t count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc != 0)
+                hash_link(&plist[i]);
+        }
+        plist = plist[0].next_in_bucket;
+    }
+    GC_reachable_here(old_list);
+
+    rec = 0;
+}
+
+static void hash_add_entry(gcobj_t *gcobj, pyobj_t *pyobj)
+{
+    if (hash_free_list == NULL) {
+        hash_grow_table();
+    }
+    assert(pyobj->ob_pypy_link == 0);
+
+    struct link_s *lnk = hash_free_list;
+    hash_free_list = lnk->next_in_bucket;
+    lnk->pyobj = pyobj;
+    lnk->gcenc = (uintptr_t)gcobj;
+    pyobj->ob_pypy_link = (Py_ssize_t)lnk;
+
+    hash_link(lnk);
+
+    if (GC_base(gcobj) == NULL) {
+        /* 'gcobj' is probably a prebuilt object - it makes no */
+        /* sense to register it then, and it crashes Boehm in */
+        /* quite obscure ways */
+    }
+    else {
+        int j = GC_general_register_disappearing_link(
+                                    (void **)&lnk->gcenc, gcobj);
+        assert(j == GC_SUCCESS);
+    }
+}
+
+static pyobj_t *hash_get_entry(gcobj_t *gcobj)
+{
+    if (hash_buckets == NULL)
+        return NULL;
+    uintptr_t h = hash_get_hash(gcobj);
+    struct link_s *lnk = hash_buckets[h];
+    while (lnk != NULL) {
+        assert(lnk->pyobj != NULL);
+        if (decode_gcenc(lnk->gcenc) == gcobj)
+            return lnk->pyobj;
+        lnk = lnk->next_in_bucket;
+    }
+    return NULL;
+}
+
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_next_dead(void)
+{
+    while (hash_list_walk_next >= 0) {
+        struct link_s *p, **pp = &hash_buckets[hash_list_walk_next];
+        while (1) {
+            p = *pp;
+            if (p == NULL)
+                break;
+            assert(p->pyobj != NULL);
+            if (p->gcenc == 0) {
+                /* quadratic time on the number of links from the same
+                   bucket chain, but it should be small with very high
+                   probability */
+                pyobj_t *result = p->pyobj;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+                printf("next_dead: %p\n", result);
+#endif
+                assert(result->ob_refcnt == REFCNT_FROM_PYPY);
+                result->ob_refcnt = 1;
+                p->pyobj = NULL;
+                *pp = p->next_in_bucket;
+                p->next_in_bucket = hash_free_list;
+                hash_free_list = p;
+                return result;
+            }
+            else {
+                assert(p->gcenc != ~(uintptr_t)0);
+                pp = &p->next_in_bucket;
+            }
+        }
+        hash_list_walk_next--;
+    }
+    return NULL;
+}
+
+RPY_EXTERN
+void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                     /*pyobj_t*/void *pyobj)
+{
+    gcobj_t *gcobj1 = (gcobj_t *)gcobj;
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    assert(pyobj1->ob_pypy_link == 0);
+    /*assert(pyobj1->ob_refcnt >= REFCNT_FROM_PYPY);*/
+    /*^^^ could also be fixed just after the call to create_link_pypy()*/
+
+    hash_add_entry(gcobj1, pyobj1);
+}
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj)
+{
+    return hash_get_entry((gcobj_t *)gcobj);
+}
+
+RPY_EXTERN
+/*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj)
+{
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    if (pyobj1->ob_pypy_link == 0)
+        return NULL;
+
+    struct link_s *lnk = (struct link_s *)pyobj1->ob_pypy_link;
+    assert(lnk->pyobj == pyobj1);
+    
+    gcobj_t *g = decode_gcenc(lnk->gcenc);
+    assert(g != NULL);
+    return g;
+}
+
+static void boehm_is_about_to_collect(void)
+{
+    struct link_s *plist = hash_list;
+    uintptr_t gcenc_union = 0;
+    while (plist != NULL) {
+        uintptr_t i, count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc == 0)
+                continue;
+
+            pyobj_t *p = plist[i].pyobj;
+            assert(p != NULL);
+            assert(p->ob_refcnt >= REFCNT_FROM_PYPY);
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+            printf("plist[%d].gcenc: %p ", (int)i, (void *)plist[i].gcenc);
+#endif
+
+            if ((plist[i].gcenc & 1) ^ (p->ob_refcnt == REFCNT_FROM_PYPY)) {
+                /* ob_refcnt > FROM_PYPY: non-zero regular refcnt, 
+                   the gc obj must stay alive.  decode gcenc.
+                   ---OR---
+                   ob_refcnt == FROM_PYPY: no refs from C code, the
+                   gc obj must not (necessarily) stay alive.  encode gcenc.
+                */
+                plist[i].gcenc = ~plist[i].gcenc;
+            }
+            gcenc_union |= plist[i].gcenc;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+            printf("-> %p\n", (void *)plist[i].gcenc);
+#endif
+    }
+        plist = plist[0].next_in_bucket;
+    }
+    if (gcenc_union & 1)   /* if there is at least one item potentially dead */
+        hash_list_walk_next = hash_mask_bucket;
+}
diff --git a/rpython/rlib/src/boehm-rawrefcount.h b/rpython/rlib/src/boehm-rawrefcount.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.h
@@ -0,0 +1,26 @@
+
+/* Missing:
+   OP_GC_RAWREFCOUNT_INIT(callback, r): the callback is not supported here
+   OP_GC_RAWREFCOUNT_CREATE_LINK_PYOBJ(): not implemented, maybe not needed
+*/
+
+#define OP_GC_RAWREFCOUNT_CREATE_LINK_PYPY(gcobj, pyobj, r)   \
+    gc_rawrefcount_create_link_pypy(gcobj, pyobj)
+
+#define OP_GC_RAWREFCOUNT_FROM_OBJ(gcobj, r)   \
+    r = gc_rawrefcount_from_obj(gcobj)
+
+#define OP_GC_RAWREFCOUNT_TO_OBJ(pyobj, r)   \
+    r = gc_rawrefcount_to_obj(pyobj)
+
+#define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
+    r = gc_rawrefcount_next_dead()
+
+#define OP_GC_RAWREFCOUNT_MARK_DEALLOCATING(gcobj, pyobj, r)  /* nothing */
+
+
+RPY_EXTERN void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                                /*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj);
+RPY_EXTERN /*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_next_dead(void);
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -1,7 +1,7 @@
 import weakref
 from rpython.rlib import rawrefcount, objectmodel, rgc
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT
-from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.translator.c.test.test_standalone import StandaloneTests
 from rpython.config.translationoption import get_combined_translation_config
@@ -264,6 +264,9 @@
             if rawrefcount.next_dead(PyObject) != ob:
                 print "NEXT_DEAD != OB"
                 return 1
+            if ob.c_ob_refcnt != 1:
+                print "next_dead().ob_refcnt != 1"
+                return 1
             if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
                 print "NEXT_DEAD second time != NULL"
                 return 1
diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -0,0 +1,308 @@
+import itertools, os, subprocess, py
+from hypothesis import given, strategies
+from rpython.tool.udir import udir
+from rpython.rlib import rawrefcount, rgc
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.test.test_rawrefcount import W_Root, PyObject, PyObjectS
+from rpython.rtyper.lltypesystem import lltype
+from rpython.translator.c.test.test_standalone import StandaloneTests
+from rpython.config.translationoption import get_combined_translation_config
+
+
+def compile_test(basename):
+    srcdir = os.path.dirname(os.path.dirname(
+        os.path.abspath(os.path.join(__file__))))
+    srcdir = os.path.join(srcdir, 'src')
+
+    err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o %s %s.c"
+                    % (udir, srcdir, basename, basename))
+    return err
+
+def setup_module():
+    filename = str(udir.join("test-rawrefcount-boehm-check.c"))
+    with open(filename, "w") as f:
+        print >> f, '#include "gc/gc_mark.h"'
+        print >> f, '#include <stdio.h>'
+        print >> f, 'int main(void) {'
+        print >> f, '    printf("%p", &GC_set_start_callback);'
+        print >> f, '    return 0;'
+        print >> f, '}'
+
+    if compile_test("test-rawrefcount-boehm-check") != 0:
+        py.test.skip("Boehm GC not installed or too old version")
+
+
+
+TEST_CODE = r"""
+#define TEST_BOEHM_RAWREFCOUNT
+#include "boehm-rawrefcount.c"
+
+static gcobj_t *alloc_gcobj(void)   /* for tests */
+{
+    gcobj_t *g = GC_MALLOC(1000);
+    printf("gc obj: %p\n", g);
+    return g;
+}
+
+static pyobj_t *alloc_pyobj(void)   /* for tests */
+{
+    pyobj_t *p = malloc(1000);
+    p->ob_refcnt = 1;
+    p->ob_pypy_link = 0;
+    printf("py obj: %p\n", p);
+    return p;
+}
+
+static void decref(pyobj_t *p)      /* for tests */
+{
+    p->ob_refcnt--;
+    if (p->ob_refcnt == 0) {
+        printf("decref to zero: %p\n", p);
+        free(p);
+    }
+    assert(p->ob_refcnt >= REFCNT_FROM_PYPY ||
+           p->ob_refcnt < REFCNT_FROM_PYPY * 0.99);
+}
+
+void run_test(void);     /* forward declaration, produced by the test */
+
+int main(void)
+{
+    run_test();
+    while (gc_rawrefcount_next_dead() != NULL)
+        ;
+    return 0;
+}
+"""
+
+
+operations = strategies.sampled_from([
+    'new_pyobj',
+    'new_gcobj',
+    'create_link',
+    'from_obj',
+    'to_obj',
+    'forget_pyobj',
+    'forget_gcobj',
+    'collect',
+    'dead',
+    ])
+
+
+ at strategies.composite
+def make_code(draw):
+    code = []
+    pyobjs = []
+    gcobjs = []
+    num_gcobj = itertools.count()
+    num_pyobj = itertools.count()
+    links_g2p = {}
+    links_p2g = {}
+
+    def new_gcobj():
+        varname = 'g%d' % next(num_gcobj)
+        code.append('gcobj_t *volatile %s = alloc_gcobj();' % varname)
+        gcobjs.append(varname)
+        return varname
+
+    def new_pyobj():
+        varname = 'p%d' % next(num_pyobj)
+        code.append('pyobj_t *%s = alloc_pyobj();' % varname)
+        pyobjs.append(varname)
+        return varname
+
+    for op in draw(strategies.lists(operations, average_size=250)):
+        if op == 'new_gcobj':
+            new_gcobj()
+        elif op == 'new_pyobj':
+            new_pyobj()
+        elif op == 'create_link':
+            gvars = [varname for varname in gcobjs if varname not in links_g2p]
+            if gvars == []:
+                gvars.append(new_gcobj())
+            pvars = [varname for varname in pyobjs if varname not in links_p2g]
+            if pvars == []:
+                pvars.append(new_pyobj())
+            gvar = draw(strategies.sampled_from(gvars))
+            pvar = draw(strategies.sampled_from(pvars))
+            code.append(r'printf("create_link %%p-%%p\n", %s, %s); '
+                            % (gvar, pvar) +
+                        "%s->ob_refcnt += REFCNT_FROM_PYPY; " % pvar +
+                        "gc_rawrefcount_create_link_pypy(%s, %s);"
+                            % (gvar, pvar))
+            links_g2p[gvar] = pvar
+            links_p2g[pvar] = gvar
+        elif op == 'from_obj':
+            if gcobjs:
+                prnt = False
+                gvar = draw(strategies.sampled_from(gcobjs))
+                if gvar not in links_g2p:
+                    check = "== NULL"
+                elif links_g2p[gvar] in pyobjs:
+                    check = "== %s" % (links_g2p[gvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_from_obj(%s) %s);"
+                            % (gvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", %s, '
+                        'gc_rawrefcount_from_obj(%s));' % (gvar, gvar))
+        elif op == 'to_obj':
+            if pyobjs:
+                prnt = False
+                pvar = draw(strategies.sampled_from(pyobjs))
+                if pvar not in links_p2g:
+                    check = "== NULL"
+                elif links_p2g[pvar] in gcobjs:
+                    check = "== %s" % (links_p2g[pvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_to_obj(%s) %s);"
+                            % (pvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", '
+                        'gc_rawrefcount_to_obj(%s), %s);' % (pvar, pvar))
+        elif op == 'forget_pyobj':
+            if pyobjs:
+                index = draw(strategies.sampled_from(range(len(pyobjs))))
+                pvar = pyobjs.pop(index)
+                code.append(r'printf("-p%%p\n", %s); ' % pvar +
+                            "decref(%s); %s = NULL;" % (pvar, pvar))
+        elif op == 'forget_gcobj':
+            if gcobjs:
+                index = draw(strategies.sampled_from(range(len(gcobjs))))
+                gvar = gcobjs.pop(index)
+                code.append(r'printf("-g%%p\n", %s); ' % gvar +
+                            "%s = NULL;" % (gvar,))
+        elif op == 'collect':
+            code.append("GC_gcollect();")
+        elif op == 'dead':
+            code.append('gc_rawrefcount_next_dead();')
+        else:
+            assert False, op
+
+    return '\n'.join(code)
+
+
+ at given(make_code())
+def test_random(code):
+    filename = str(udir.join("test-rawrefcount-boehm.c"))
+    with open(filename, "w") as f:
+        print >> f, TEST_CODE
+        print >> f, 'void run_test(void) {'
+        print >> f, code
+        print >> f, '}'
+
+    err = compile_test("test-rawrefcount-boehm")
+    if err != 0:
+        raise OSError("gcc failed")
+    p = subprocess.Popen("./test-rawrefcount-boehm", stdout=subprocess.PIPE,
+                         cwd=str(udir))
+    stdout, _ = p.communicate()
+    assert p.wait() == 0
+
+    gcobjs = {}
+    pyobjs = {}
+    links_p2g = {}
+    links_g2p = {}
+    for line in stdout.splitlines():
+        if line.startswith('py obj: '):
+            p = line[8:]
+            assert not pyobjs.get(p)
+            pyobjs[p] = True
+            assert p not in links_p2g
+        elif line.startswith('gc obj: '):
+            g = line[8:]
+            assert not gcobjs.get(g)
+            gcobjs[g] = True
+            if g in links_g2p: del links_g2p[g]
+        elif line.startswith('-p'):
+            p = line[2:]
+            assert pyobjs[p] == True
+            pyobjs[p] = False
+        elif line.startswith('-g'):
+            g = line[2:]
+            assert gcobjs[g] == True
+            gcobjs[g] = False
+        elif line.startswith('decref to zero: '):
+            p = line[16:]
+            assert pyobjs[p] == False
+            assert p not in links_p2g
+            del pyobjs[p]
+        elif line.startswith('create_link '):
+            g, p = line[12:].split('-')
+            assert g in gcobjs
+            assert p in pyobjs
+            assert g not in links_g2p
+            assert p not in links_p2g
+            links_g2p[g] = p
+            links_p2g[p] = g
+        elif line.startswith('link '):
+            g, p = line[5:].split('-')
+            assert g in gcobjs
+            assert p in pyobjs
+            assert links_g2p[g] == p
+            assert links_p2g[p] == g
+        elif line.startswith('plist['):
+            pass
+        elif line.startswith('next_dead: '):
+            p = line[11:]
+            assert pyobjs[p] == False
+            del pyobjs[p]
+            del links_p2g[p]
+        else:
+            assert False, repr(line)
+
+
+class TestBoehmTranslated(StandaloneTests):
+
+    def test_full_translation(self):
+
+        def make_ob():
+            p = W_Root(42)
+            ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+            rawrefcount.create_link_pypy(p, ob)
+            ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            assert rawrefcount.from_obj(PyObject, p) == ob
+            assert rawrefcount.to_obj(W_Root, ob) == p
+            return ob
+
+        prebuilt_p = W_Root(-42)
+        prebuilt_ob = lltype.malloc(PyObjectS, flavor='raw', zero=True,
+                                    immortal=True)
+
+        def entry_point(argv):
+            rawrefcount.create_link_pypy(prebuilt_p, prebuilt_ob)
+            prebuilt_ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            oblist = [make_ob() for i in range(50)]
+            rgc.collect()
+            deadlist = []
+            while True:
+                ob = rawrefcount.next_dead(PyObject)
+                if not ob: break
+                if ob.c_ob_refcnt != 1:
+                    print "next_dead().ob_refcnt != 1"
+                    return 1
+                deadlist.append(ob)
+            if len(deadlist) == 0:
+                print "no dead object"
+                return 1
+            if len(deadlist) < 30:
+                print "not enough dead objects"
+                return 1
+            for ob in deadlist:
+                if ob not in oblist:
+                    print "unexpected value for dead pointer"
+                    return 1
+                oblist.remove(ob)
+            print "OK!"
+            lltype.free(ob, flavor='raw')
+            return 0
+
+        self.config = get_combined_translation_config(translating=True)
+        self.config.translation.gc = "boehm"
+        t, cbuilder = self.compile(entry_point)
+        data = cbuilder.cmdexec('hi there')
+        assert data.startswith('OK!\n')
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -966,6 +966,9 @@
     def op_gc_rawrefcount_mark_deallocating(self, *args):
         raise NotImplementedError("gc_rawrefcount_mark_deallocating")
 
+    def op_gc_rawrefcount_next_dead(self, *args):
+        raise NotImplementedError("gc_rawrefcount_next_dead")
+
     def op_do_malloc_fixedsize(self):
         raise NotImplementedError("do_malloc_fixedsize")
     def op_do_malloc_fixedsize_clear(self):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -494,6 +494,7 @@
     'gc_rawrefcount_mark_deallocating': LLOp(),
     'gc_rawrefcount_from_obj':          LLOp(sideeffects=False),
     'gc_rawrefcount_to_obj':            LLOp(sideeffects=False),
+    'gc_rawrefcount_next_dead':         LLOp(),
 
     'gc_move_out_of_nursery':           LLOp(),
 

From pypy.commits at gmail.com  Tue Jan  3 04:07:26 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 01:07:26 -0800 (PST)
Subject: [pypy-commit] cffi strbuf-as-buffer: kill invalid_input_buffer_type,
 update docs to state that now only unicode objects are forbidden
Message-ID: <586b69ce.6249c20a.a829a.e6d1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: strbuf-as-buffer
Changeset: r2851:44df83151f36
Date: 2017-01-03 10:04 +0100
http://bitbucket.org/cffi/cffi/changeset/44df83151f36/

Log:	kill invalid_input_buffer_type, update docs to state that now only
	unicode objects are forbidden

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6101,43 +6101,18 @@
     return 0;
 }
 
-static int invalid_input_buffer_type(PyObject *x)
-{
-    /* From PyPy 5.4, from_buffer() accepts strings (but still not buffers
-       or memoryviews on strings). */
-    if (PyBytes_Check(x))
-        return 0;
-
-#if PY_MAJOR_VERSION < 3
-    if (PyBuffer_Check(x)) {
-        return 0;
-    }
-    else
-#endif
-#if PY_MAJOR_VERSION > 2 || PY_MINOR_VERSION > 6
-    if (PyMemoryView_Check(x)) {
-        return 0;
-    }
-    else
-#endif
-        ;
-
-    if (PyBytes_Check(x) || PyUnicode_Check(x))
-        return 1;
-    /* From PyPy 5.2, bytearray buffers can fetch a raw pointer, so
-       there is no reason any more to prevent from_buffer(bytearray()). */
-    return 0;
-}
-
 static PyObject *direct_from_buffer(CTypeDescrObject *ct, PyObject *x)
 {
     CDataObject *cd;
     Py_buffer *view;
 
-    if (invalid_input_buffer_type(x)) {
+    /* PyPy 5.7 can obtain buffers for string (python 2)
+       or bytes (python 3). from_buffer(u"foo") is disallowed.
+     */
+    if (PyUnicode_Check(x)) {
         PyErr_SetString(PyExc_TypeError,
-                        "from_buffer() cannot return the address of the "
-                        "raw string within a "STR_OR_BYTES" or unicode object");
+                        "from_buffer() cannot return the address "
+                        "of a unicode object");
         return NULL;
     }
 
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -175,9 +175,8 @@
 **ffi.from_buffer(python_buffer)**: return a ``<cdata 'char[]'>`` that
 points to the data of the given Python object, which must support the
 buffer interface.  This is the opposite of ``ffi.buffer()``.  It gives
-a reference to the existing data, not a copy; for this
-reason, and for PyPy compatibility, it does not work with the built-in
-type unicode; nor buffers/memoryviews to byte or unicode strings.
+a reference to the existing data, not a copy; It does not work with the built-in
+type unicode.
 It is meant to be used on objects
 containing large quantities of raw data, like bytearrays
 or ``array.array`` or numpy

From pypy.commits at gmail.com  Tue Jan  3 04:07:28 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 01:07:28 -0800 (PST)
Subject: [pypy-commit] cffi strbuf-as-buffer: update whatsnew entry
Message-ID: <586b69d0.12ad1c0a.593aa.7216@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: strbuf-as-buffer
Changeset: r2852:146c486af9a5
Date: 2017-01-03 10:07 +0100
http://bitbucket.org/cffi/cffi/changeset/146c486af9a5/

Log:	update whatsnew entry

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -15,6 +15,8 @@
 
 * some OS/X build fixes ("only with Xcode but without CLT").
 
+* ``ffi.from_buffer`` allows Python 2 strings and Python 3 bytes to be
+  passed. Unicode is the only type disallowed.
 
 v1.9
 ====

From pypy.commits at gmail.com  Tue Jan  3 04:27:28 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 01:27:28 -0800 (PST)
Subject: [pypy-commit] pypy strbuf-as-buffer: kill invalid_input_buffer_type
 and simplify it to space.isinstance_w(w_x, space.w_unicode)
Message-ID: <586b6e80.a285c20a.d98b9.ee97@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: strbuf-as-buffer
Changeset: r89329:fbcff82b11bb
Date: 2017-01-03 10:26 +0100
http://bitbucket.org/pypy/pypy/changeset/fbcff82b11bb/

Log:	kill invalid_input_buffer_type and simplify it to
	space.isinstance_w(w_x, space.w_unicode)

diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -135,13 +135,8 @@
     #
     return _from_buffer(space, w_ctype, w_x)
 
-def invalid_input_buffer_type(space, w_x):
+def _from_buffer(space, w_ctype, w_x):
     if space.isinstance_w(w_x, space.w_unicode):
-        return True
-    return False
-
-def _from_buffer(space, w_ctype, w_x):
-    if invalid_input_buffer_type(space, w_x):
         raise oefmt(space.w_TypeError,
                         "from_buffer() cannot return the address a unicode")
     buf = _fetch_as_read_buffer(space, w_x)

From pypy.commits at gmail.com  Tue Jan  3 04:27:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 01:27:42 -0800 (PST)
Subject: [pypy-commit] cffi strbuf-as-buffer: Fix test
Message-ID: <586b6e8e.cb911c0a.66b4.4996@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: strbuf-as-buffer
Changeset: r2853:dc41e3e1b6e2
Date: 2017-01-03 10:27 +0100
http://bitbucket.org/cffi/cffi/changeset/dc41e3e1b6e2/

Log:	Fix test

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3434,11 +3434,14 @@
     except ImportError:
         pass
     else:
+        # Python 2 only
         contents = from_buffer(BCharA, buffer(b"foo"))
+        assert len(contents) == len(p1)
         for i in range(len(contents)):
             assert contents[i] == p1[i]
-        p4 = from_buffer(BCharA, b"f\x00\x00\x00o\x00\x00\x00o\x00\x00\x00")
+        p4 = buffer(u+"foo")
         contents = from_buffer(BCharA, buffer(u+"foo"))
+        assert len(contents) == len(p4)
         for i in range(len(contents)):
             assert contents[i] == p4[i]
     try:
@@ -3447,6 +3450,7 @@
         pass
     else:
         contents = from_buffer(BCharA, memoryview(b"foo"))
+        assert len(contents) == len(p1)
         for i in range(len(contents)):
             assert contents[i] == p1[i]
 

From pypy.commits at gmail.com  Tue Jan  3 04:43:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 01:43:53 -0800 (PST)
Subject: [pypy-commit] pypy default: Add many checks and tests until I found
 where to insert the two lines that fix 'r_int' support
Message-ID: <586b7259.ca57c20a.96102.cb97@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89330:ba112f125696
Date: 2017-01-03 10:43 +0100
http://bitbucket.org/pypy/pypy/changeset/ba112f125696/

Log:	Add many checks and tests until I found where to insert the two
	lines that fix 'r_int' support

diff --git a/rpython/jit/codewriter/assembler.py b/rpython/jit/codewriter/assembler.py
--- a/rpython/jit/codewriter/assembler.py
+++ b/rpython/jit/codewriter/assembler.py
@@ -5,6 +5,7 @@
 from rpython.jit.codewriter.jitcode import SwitchDictDescr, JitCode
 from rpython.jit.codewriter import heaptracker, longlong
 from rpython.rlib.objectmodel import ComputedIntSymbolic
+from rpython.rlib.rarithmetic import r_int
 from rpython.flowspace.model import Constant
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper import rclass
@@ -82,6 +83,8 @@
             if not isinstance(value, (llmemory.AddressAsInt,
                                       ComputedIntSymbolic)):
                 value = lltype.cast_primitive(lltype.Signed, value)
+                if type(value) is r_int:
+                    value = int(value)
                 if allow_short:
                     try:
                         short_num = -128 <= value <= 127
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -1,6 +1,7 @@
 from rpython.jit.metainterp.history import AbstractDescr, ConstInt
 from rpython.jit.codewriter import heaptracker
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import base_int
 
 
 class JitCode(AbstractDescr):
@@ -21,6 +22,10 @@
               liveness=None, startpoints=None, alllabels=None,
               resulttypes=None):
         self.code = code
+        for x in constants_i:
+            assert not isinstance(x, base_int), (
+                "found constant %r of type %r, must not appear in "
+                "JitCode.constants_i" % (x, type(x)))
         # if the following lists are empty, use a single shared empty list
         self.constants_i = constants_i or self._empty_i
         self.constants_r = constants_r or self._empty_r
diff --git a/rpython/jit/codewriter/test/test_assembler.py b/rpython/jit/codewriter/test/test_assembler.py
--- a/rpython/jit/codewriter/test/test_assembler.py
+++ b/rpython/jit/codewriter/test/test_assembler.py
@@ -7,6 +7,7 @@
 from rpython.jit.metainterp.history import AbstractDescr
 from rpython.flowspace.model import Constant
 from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rlib.rarithmetic import r_int, r_uint
 
 
 def test_assemble_simple():
@@ -239,3 +240,17 @@
         ]
     assembler = Assembler()
     py.test.raises(AssemblerError, assembler.assemble, ssarepr)
+
+def test_assemble_r_int():
+    # r_int is a strange type, which the jit should replace with int.
+    # r_uint is also replaced with int.
+    ssarepr = SSARepr("test")
+    i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2)
+    ssarepr.insns = [
+        ('uint_add', i0, Constant(r_uint(42424242), lltype.Unsigned), '->', i1),
+        ('int_add', i0, Constant(r_int(42424243), lltype.Signed), '->', i2),
+        ]
+    assembler = Assembler()
+    jitcode = assembler.assemble(ssarepr)
+    assert jitcode.constants_i == [42424242, 42424243]
+    assert map(type, jitcode.constants_i) == [int, int]
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -6,6 +6,7 @@
 from rpython.jit.metainterp.history import MissingValue
 from rpython.rlib import longlong2float
 from rpython.rlib.debug import ll_assert, make_sure_not_resized
+from rpython.rlib.debug import check_annotation
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from rpython.rlib.unroll import unrolling_iterable
@@ -183,7 +184,7 @@
                 if lltype.typeOf(result) is lltype.Bool:
                     result = int(result)
                 assert lltype.typeOf(result) is lltype.Signed
-                self.registers_i[ord(code[position])] = result
+                self.registers_i[ord(code[position])] = plain_int(result)
                 position += 1
             elif resulttype == 'r':
                 # argcode should be 'r' too
@@ -213,7 +214,7 @@
                     if lltype.typeOf(result) is lltype.Bool:
                         result = int(result)
                     assert lltype.typeOf(result) is lltype.Signed
-                    self.registers_i[ord(code[position])] = result
+                    self.registers_i[ord(code[position])] = plain_int(result)
                     position += 1
             elif resulttype == 'L':
                 assert result >= 0
@@ -251,6 +252,23 @@
         if b < 0 or b >= LONG_BIT:
             raise ValueError("Shift count, %d,  not in valid range, 0 .. %d." % (b, LONG_BIT-1))
 
+def check_list_of_plain_integers(s_arg, bookkeeper):
+    """Check that 'BlackhopeInterpreter.registers_i' is annotated as a
+    non-resizable list of plain integers (and not r_int's for example)."""
+    from rpython.annotator import model as annmodel
+    assert isinstance(s_arg, annmodel.SomeList)
+    s_arg.listdef.never_resize()
+    assert s_arg.listdef.listitem.s_value.knowntype is int
+
+def _check_int(s_arg, bookkeeper):
+    assert s_arg.knowntype is int
+
+def plain_int(x):
+    """Check that 'x' is annotated as a plain integer (and not r_int)"""
+    check_annotation(x, _check_int)
+    return x
+
+
 class BlackholeInterpreter(object):
 
     def __init__(self, builder, count_interpreter):
@@ -277,6 +295,7 @@
         self.tmpreg_r = default_r
         self.tmpreg_f = default_f
         self.jitcode = None
+        check_annotation(self.registers_i, check_list_of_plain_integers)
 
     def __repr__(self):
         return '<BHInterp #%d>' % self.count_interpreter
@@ -295,7 +314,7 @@
 
     def setarg_i(self, index, value):
         assert lltype.typeOf(value) is lltype.Signed
-        self.registers_i[index] = value
+        self.registers_i[index] = plain_int(value)
 
     def setarg_r(self, index, value):
         assert lltype.typeOf(value) == llmemory.GCREF
@@ -1573,7 +1592,8 @@
     # 'xxx_call_yyy' instructions from the caller frame
     def _setup_return_value_i(self, result):
         assert lltype.typeOf(result) is lltype.Signed
-        self.registers_i[ord(self.jitcode.code[self.position-1])] = result
+        self.registers_i[ord(self.jitcode.code[self.position-1])] = plain_int(
+                                                                        result)
     def _setup_return_value_r(self, result):
         assert lltype.typeOf(result) == llmemory.GCREF
         self.registers_r[ord(self.jitcode.code[self.position-1])] = result

From pypy.commits at gmail.com  Tue Jan  3 04:48:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 01:48:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <586b7358.c515c20a.a3d17.f05b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89331:b6b06ee0fb56
Date: 2017-01-03 10:47 +0100
http://bitbucket.org/pypy/pypy/changeset/b6b06ee0fb56/

Log:	hg merge default

diff --git a/rpython/jit/codewriter/assembler.py b/rpython/jit/codewriter/assembler.py
--- a/rpython/jit/codewriter/assembler.py
+++ b/rpython/jit/codewriter/assembler.py
@@ -5,6 +5,7 @@
 from rpython.jit.codewriter.jitcode import SwitchDictDescr, JitCode
 from rpython.jit.codewriter import heaptracker, longlong
 from rpython.rlib.objectmodel import ComputedIntSymbolic
+from rpython.rlib.rarithmetic import r_int
 from rpython.flowspace.model import Constant
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper import rclass
@@ -82,6 +83,8 @@
             if not isinstance(value, (llmemory.AddressAsInt,
                                       ComputedIntSymbolic)):
                 value = lltype.cast_primitive(lltype.Signed, value)
+                if type(value) is r_int:
+                    value = int(value)
                 if allow_short:
                     try:
                         short_num = -128 <= value <= 127
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -1,6 +1,7 @@
 from rpython.jit.metainterp.history import AbstractDescr, ConstInt
 from rpython.jit.codewriter import heaptracker
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import base_int
 
 
 class JitCode(AbstractDescr):
@@ -21,6 +22,10 @@
               liveness=None, startpoints=None, alllabels=None,
               resulttypes=None):
         self.code = code
+        for x in constants_i:
+            assert not isinstance(x, base_int), (
+                "found constant %r of type %r, must not appear in "
+                "JitCode.constants_i" % (x, type(x)))
         # if the following lists are empty, use a single shared empty list
         self.constants_i = constants_i or self._empty_i
         self.constants_r = constants_r or self._empty_r
diff --git a/rpython/jit/codewriter/test/test_assembler.py b/rpython/jit/codewriter/test/test_assembler.py
--- a/rpython/jit/codewriter/test/test_assembler.py
+++ b/rpython/jit/codewriter/test/test_assembler.py
@@ -7,6 +7,7 @@
 from rpython.jit.metainterp.history import AbstractDescr
 from rpython.flowspace.model import Constant
 from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rlib.rarithmetic import r_int, r_uint
 
 
 def test_assemble_simple():
@@ -239,3 +240,17 @@
         ]
     assembler = Assembler()
     py.test.raises(AssemblerError, assembler.assemble, ssarepr)
+
+def test_assemble_r_int():
+    # r_int is a strange type, which the jit should replace with int.
+    # r_uint is also replaced with int.
+    ssarepr = SSARepr("test")
+    i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2)
+    ssarepr.insns = [
+        ('uint_add', i0, Constant(r_uint(42424242), lltype.Unsigned), '->', i1),
+        ('int_add', i0, Constant(r_int(42424243), lltype.Signed), '->', i2),
+        ]
+    assembler = Assembler()
+    jitcode = assembler.assemble(ssarepr)
+    assert jitcode.constants_i == [42424242, 42424243]
+    assert map(type, jitcode.constants_i) == [int, int]
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -6,6 +6,7 @@
 from rpython.jit.metainterp.history import MissingValue
 from rpython.rlib import longlong2float
 from rpython.rlib.debug import ll_assert, make_sure_not_resized
+from rpython.rlib.debug import check_annotation
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from rpython.rlib.unroll import unrolling_iterable
@@ -183,7 +184,7 @@
                 if lltype.typeOf(result) is lltype.Bool:
                     result = int(result)
                 assert lltype.typeOf(result) is lltype.Signed
-                self.registers_i[ord(code[position])] = result
+                self.registers_i[ord(code[position])] = plain_int(result)
                 position += 1
             elif resulttype == 'r':
                 # argcode should be 'r' too
@@ -213,7 +214,7 @@
                     if lltype.typeOf(result) is lltype.Bool:
                         result = int(result)
                     assert lltype.typeOf(result) is lltype.Signed
-                    self.registers_i[ord(code[position])] = result
+                    self.registers_i[ord(code[position])] = plain_int(result)
                     position += 1
             elif resulttype == 'L':
                 assert result >= 0
@@ -251,6 +252,23 @@
         if b < 0 or b >= LONG_BIT:
             raise ValueError("Shift count, %d,  not in valid range, 0 .. %d." % (b, LONG_BIT-1))
 
+def check_list_of_plain_integers(s_arg, bookkeeper):
+    """Check that 'BlackhopeInterpreter.registers_i' is annotated as a
+    non-resizable list of plain integers (and not r_int's for example)."""
+    from rpython.annotator import model as annmodel
+    assert isinstance(s_arg, annmodel.SomeList)
+    s_arg.listdef.never_resize()
+    assert s_arg.listdef.listitem.s_value.knowntype is int
+
+def _check_int(s_arg, bookkeeper):
+    assert s_arg.knowntype is int
+
+def plain_int(x):
+    """Check that 'x' is annotated as a plain integer (and not r_int)"""
+    check_annotation(x, _check_int)
+    return x
+
+
 class BlackholeInterpreter(object):
 
     def __init__(self, builder, count_interpreter):
@@ -277,6 +295,7 @@
         self.tmpreg_r = default_r
         self.tmpreg_f = default_f
         self.jitcode = None
+        check_annotation(self.registers_i, check_list_of_plain_integers)
 
     def __repr__(self):
         return '<BHInterp #%d>' % self.count_interpreter
@@ -295,7 +314,7 @@
 
     def setarg_i(self, index, value):
         assert lltype.typeOf(value) is lltype.Signed
-        self.registers_i[index] = value
+        self.registers_i[index] = plain_int(value)
 
     def setarg_r(self, index, value):
         assert lltype.typeOf(value) == llmemory.GCREF
@@ -1573,7 +1592,8 @@
     # 'xxx_call_yyy' instructions from the caller frame
     def _setup_return_value_i(self, result):
         assert lltype.typeOf(result) is lltype.Signed
-        self.registers_i[ord(self.jitcode.code[self.position-1])] = result
+        self.registers_i[ord(self.jitcode.code[self.position-1])] = plain_int(
+                                                                        result)
     def _setup_return_value_r(self, result):
         assert lltype.typeOf(result) == llmemory.GCREF
         self.registers_r[ord(self.jitcode.code[self.position-1])] = result
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -1,7 +1,7 @@
 import weakref
 from rpython.rlib import rawrefcount, objectmodel, rgc
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT
-from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.translator.c.test.test_standalone import StandaloneTests
 from rpython.config.translationoption import get_combined_translation_config
@@ -286,55 +286,3 @@
         t, cbuilder = self.compile(entry_point)
         data = cbuilder.cmdexec('hi there')
         assert data.startswith('OK!\n')
-
-
-class TestBoehmTranslated(StandaloneTests):
-
-    def test_full_translation(self):
-
-        def make_ob():
-            p = W_Root(42)
-            ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
-            rawrefcount.create_link_pypy(p, ob)
-            ob.c_ob_refcnt += REFCNT_FROM_PYPY
-            assert rawrefcount.from_obj(PyObject, p) == ob
-            assert rawrefcount.to_obj(W_Root, ob) == p
-            return ob
-
-        prebuilt_p = W_Root(-42)
-        prebuilt_ob = lltype.malloc(PyObjectS, flavor='raw', zero=True,
-                                    immortal=True)
-
-        def entry_point(argv):
-            rawrefcount.create_link_pypy(prebuilt_p, prebuilt_ob)
-            prebuilt_ob.c_ob_refcnt += REFCNT_FROM_PYPY
-            oblist = [make_ob() for i in range(50)]
-            rgc.collect()
-            deadlist = []
-            while True:
-                ob = rawrefcount.next_dead(PyObject)
-                if not ob: break
-                if ob.c_ob_refcnt != 1:
-                    print "next_dead().ob_refcnt != 1"
-                    return 1
-                deadlist.append(ob)
-            if len(deadlist) == 0:
-                print "no dead object"
-                return 1
-            if len(deadlist) < 30:
-                print "not enough dead objects"
-                return 1
-            for ob in deadlist:
-                if ob not in oblist:
-                    print "unexpected value for dead pointer"
-                    return 1
-                oblist.remove(ob)
-            print "OK!"
-            lltype.free(ob, flavor='raw')
-            return 0
-
-        self.config = get_combined_translation_config(translating=True)
-        self.config.translation.gc = "boehm"
-        t, cbuilder = self.compile(entry_point)
-        data = cbuilder.cmdexec('hi there')
-        assert data.startswith('OK!\n')
diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
--- a/rpython/rlib/test/test_rawrefcount_boehm.py
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -1,19 +1,34 @@
 import itertools, os, subprocess, py
 from hypothesis import given, strategies
 from rpython.tool.udir import udir
+from rpython.rlib import rawrefcount, rgc
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.test.test_rawrefcount import W_Root, PyObject, PyObjectS
+from rpython.rtyper.lltypesystem import lltype
+from rpython.translator.c.test.test_standalone import StandaloneTests
+from rpython.config.translationoption import get_combined_translation_config
 
 
+def compile_test(basename):
+    srcdir = os.path.dirname(os.path.dirname(
+        os.path.abspath(os.path.join(__file__))))
+    srcdir = os.path.join(srcdir, 'src')
+
+    err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o %s %s.c"
+                    % (udir, srcdir, basename, basename))
+    return err
+
 def setup_module():
     filename = str(udir.join("test-rawrefcount-boehm-check.c"))
     with open(filename, "w") as f:
         print >> f, '#include "gc/gc_mark.h"'
-        print >> f, 'void *testing(void) {'
-        print >> f, '    return &GC_set_start_callback;'
+        print >> f, '#include <stdio.h>'
+        print >> f, 'int main(void) {'
+        print >> f, '    printf("%p", &GC_set_start_callback);'
+        print >> f, '    return 0;'
         print >> f, '}'
 
-    err = os.system("cd '%s' && gcc -c test-rawrefcount-boehm-check.c"
-                    % (udir,))
-    if err != 0:
+    if compile_test("test-rawrefcount-boehm-check") != 0:
         py.test.skip("Boehm GC not installed or too old version")
 
 
@@ -180,13 +195,9 @@
         print >> f, code
         print >> f, '}'
 
-    srcdir = os.path.dirname(os.path.dirname(
-        os.path.abspath(os.path.join(__file__))))
-    srcdir = os.path.join(srcdir, 'src')
-
-    err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o test-rawrefcount-boehm"
-                    " test-rawrefcount-boehm.c" % (udir, srcdir))
-    assert err == 0
+    err = compile_test("test-rawrefcount-boehm")
+    if err != 0:
+        raise OSError("gcc failed")
     p = subprocess.Popen("./test-rawrefcount-boehm", stdout=subprocess.PIPE,
                          cwd=str(udir))
     stdout, _ = p.communicate()
@@ -243,3 +254,55 @@
             del links_p2g[p]
         else:
             assert False, repr(line)
+
+
+class TestBoehmTranslated(StandaloneTests):
+
+    def test_full_translation(self):
+
+        def make_ob():
+            p = W_Root(42)
+            ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+            rawrefcount.create_link_pypy(p, ob)
+            ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            assert rawrefcount.from_obj(PyObject, p) == ob
+            assert rawrefcount.to_obj(W_Root, ob) == p
+            return ob
+
+        prebuilt_p = W_Root(-42)
+        prebuilt_ob = lltype.malloc(PyObjectS, flavor='raw', zero=True,
+                                    immortal=True)
+
+        def entry_point(argv):
+            rawrefcount.create_link_pypy(prebuilt_p, prebuilt_ob)
+            prebuilt_ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            oblist = [make_ob() for i in range(50)]
+            rgc.collect()
+            deadlist = []
+            while True:
+                ob = rawrefcount.next_dead(PyObject)
+                if not ob: break
+                if ob.c_ob_refcnt != 1:
+                    print "next_dead().ob_refcnt != 1"
+                    return 1
+                deadlist.append(ob)
+            if len(deadlist) == 0:
+                print "no dead object"
+                return 1
+            if len(deadlist) < 30:
+                print "not enough dead objects"
+                return 1
+            for ob in deadlist:
+                if ob not in oblist:
+                    print "unexpected value for dead pointer"
+                    return 1
+                oblist.remove(ob)
+            print "OK!"
+            lltype.free(ob, flavor='raw')
+            return 0
+
+        self.config = get_combined_translation_config(translating=True)
+        self.config.translation.gc = "boehm"
+        t, cbuilder = self.compile(entry_point)
+        data = cbuilder.cmdexec('hi there')
+        assert data.startswith('OK!\n')

From pypy.commits at gmail.com  Tue Jan  3 07:25:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 04:25:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: py3 fix
Message-ID: <586b983d.aaa3c20a.75e9a.3234@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89332:776ecdbe8e0b
Date: 2017-01-03 13:24 +0100
http://bitbucket.org/pypy/pypy/changeset/776ecdbe8e0b/

Log:	py3 fix

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -468,10 +468,8 @@
             swapped = func(mask)
             assert swapped & mask == mask
             raises(OverflowError, func, -1)
-            raises(OverflowError, func, -1L)
             if size > 16:    # else, values too large are ignored
                 raises(OverflowError, func, 2 ** size)
-                raises(OverflowError, func, 2L ** size)
 
     def test_newsocket(self):
         import socket

From pypy.commits at gmail.com  Tue Jan  3 07:53:44 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 04:53:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-time: extend structseqtype to allow
 invisible fields (see struct_time, tm_zone, tm_gmtoff)
Message-ID: <586b9ed8.d5091c0a.6942f.aad4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-time
Changeset: r89333:08e4f8a740d5
Date: 2017-01-03 13:52 +0100
http://bitbucket.org/pypy/pypy/changeset/08e4f8a740d5/

Log:	extend structseqtype to allow invisible fields (see struct_time,
	tm_zone, tm_gmtoff)

diff --git a/lib_pypy/_structseq.py b/lib_pypy/_structseq.py
--- a/lib_pypy/_structseq.py
+++ b/lib_pypy/_structseq.py
@@ -41,22 +41,31 @@
                 assert field._index not in fields_by_index
                 fields_by_index[field._index] = field
                 field.__name__ = name
-        dict['n_fields'] = len(fields_by_index)
+        n_fields = len(fields_by_index)
+        dict['n_fields'] = n_fields
 
         extra_fields = sorted(fields_by_index.items())
         n_sequence_fields = 0
-        while extra_fields and extra_fields[0][0] == n_sequence_fields:
-            extra_fields.pop(0)
-            n_sequence_fields += 1
+        extra_off = 0
+        if 'n_sequence_fields' in dict:
+            n_sequence_fields = dict['n_sequence_fields']
+            extra_fields = extra_fields[:n_sequence_fields]
+            extra_off = n_fields - n_sequence_fields
+        else:
+            while extra_fields and extra_fields[0][0] == n_sequence_fields:
+                extra_fields.pop(0)
+                n_sequence_fields += 1
+
         dict['n_sequence_fields'] = n_sequence_fields
         dict['n_unnamed_fields'] = 0     # no fully anonymous fields in PyPy
 
-        extra_fields = [field for index, field in extra_fields]
-        for field in extra_fields:
+        extra_fields = [field for index, field in extra_fields[extra_off:]]
+        for i,field in enumerate(extra_fields):
             field.index = None     # no longer relevant
 
         assert '__new__' not in dict
         dict['_extra_fields'] = tuple(extra_fields)
+        dict['_extra_off'] = extra_off
         dict['__new__'] = structseq_new
         dict['__reduce__'] = structseq_reduce
         dict['__setattr__'] = structseq_setattr
@@ -70,34 +79,40 @@
 def structseq_new(cls, sequence, dict={}):
     sequence = tuple(sequence)
     dict = builtin_dict(dict)
-    N = cls.n_sequence_fields
-    if len(sequence) < N:
-        if N < cls.n_fields:
+    # visible fields
+    visible_count = cls.n_sequence_fields
+    # total fields (unnamed are not yet supported, extra fields not included)
+    real_count = cls.n_fields
+    length = len(sequence)
+    if length < visible_count:
+        if visible_count < real_count:
             msg = "at least"
         else:
             msg = "exactly"
-        raise TypeError("expected a sequence with %s %d items" % (
-            msg, N))
-    if len(sequence) > N:
-        if len(sequence) > cls.n_fields:
-            if N < cls.n_fields:
+        raise TypeError("expected a sequence with %s %d items. has %d 1" % (
+            msg, visible_count, length))
+    if length > visible_count:
+        if length > real_count:
+            if visible_count < real_count:
                 msg = "at most"
             else:
                 msg = "exactly"
-            raise TypeError("expected a sequence with %s %d items" % (
-                msg, cls.n_fields))
-        for field, value in zip(cls._extra_fields, sequence[N:]):
+            raise TypeError("expected a sequence with %s %d items. has %d 2" \
+                            % (msg, real_count, length))
+        for field, value in zip(cls._extra_fields, sequence[visible_count:]):
             name = field.__name__
             if name in dict:
                 raise TypeError("duplicate value for %r" % (name,))
             dict[name] = value
-        sequence = sequence[:N]
+        extra_off = cls._extra_off
+        sequence = sequence[:visible_count+extra_off]
     result = tuple.__new__(cls, sequence)
     object.__setattr__(result, '__dict__', dict)
     for field in cls._extra_fields:
         name = field.__name__
         if name not in dict:
             dict[name] = field._default(result)
+
     return result
 
 def structseq_reduce(self):
diff --git a/pypy/module/time/app_time.py b/pypy/module/time/app_time.py
--- a/pypy/module/time/app_time.py
+++ b/pypy/module/time/app_time.py
@@ -8,17 +8,21 @@
     __module__ = 'time'
     name = 'time.struct_time'
 
-    tm_year   = structseqfield(0)
-    tm_mon    = structseqfield(1)
-    tm_mday   = structseqfield(2)
-    tm_hour   = structseqfield(3)
-    tm_min    = structseqfield(4)
-    tm_sec    = structseqfield(5)
-    tm_wday   = structseqfield(6)
-    tm_yday   = structseqfield(7)
-    tm_isdst  = structseqfield(8)
-    tm_gmtoff = structseqfield(9)
-    tm_zone   = structseqfield(10)
+    n_sequence_fields = 9
+
+    tm_year   = structseqfield(0, "year, for example, 1993")
+    tm_mon    = structseqfield(1, "month of year, range [1, 12]")
+    tm_mday   = structseqfield(2, "day of month, range [1, 31]")
+    tm_hour   = structseqfield(3, "hours, range [0, 23]")
+    tm_min    = structseqfield(4, "minutes, range [0, 59]")
+    tm_sec    = structseqfield(5, "seconds, range [0, 61])")
+    tm_wday   = structseqfield(6, "day of week, range [0, 6], Monday is 0")
+    tm_yday   = structseqfield(7, "day of year, range [1, 366]")
+    tm_isdst  = structseqfield(8, "1 if summer time is in effect, 0 if not"
+                                  ", and -1 if unknown")
+    tm_zone   = structseqfield(9, "abbreviation of timezone name")
+    tm_gmtoff = structseqfield(10,"offset from UTC in seconds")
+
 
 def strptime(string, format="%a %b %d %H:%M:%S %Y"):
     """strptime(string, format) -> struct_time
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -1,8 +1,10 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror, exception_from_saved_errno
+from pypy.interpreter.error import (OperationError, oefmt,
+        strerror as _strerror, exception_from_saved_errno)
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter import timeutils
+from pypy.interpreter.unicodehelper import decode_utf8, encode_utf8
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat, widen
 from rpython.rlib.rtime import (GETTIMEOFDAY_NO_TZ, TIMEVAL,
@@ -200,6 +202,8 @@
 
 # XXX: optionally support the 2 additional tz fields
 _STRUCT_TM_ITEMS = 9
+if HAS_TM_ZONE:
+    _STRUCT_TM_ITEMS = 11
 
 class cConfig:
     pass
@@ -542,12 +546,14 @@
         space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
         space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
 
+    if HAS_TM_ZONE:
+        # CPython calls PyUnicode_DecodeLocale here should we do the same?
+        tm_zone = decode_utf8(space, rffi.charp2str(t.c_tm_zone), allow_surrogates=True)
+        time_tuple.append(space.newunicode(tm_zone))
+        time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_gmtoff')))
     w_struct_time = _get_module_object(space, 'struct_time')
     w_time_tuple = space.newtuple(time_tuple)
     w_obj = space.call_function(w_struct_time, w_time_tuple)
-    if HAS_TM_ZONE:
-        space.setattr(w_obj, space.wrap("tm_gmoff"), space.wrap(rffi.getintfield(t, 'c_tm_gmtoff')))
-        space.setattr(w_obj, space.wrap("tm_zone"), space.wrap(rffi.getintfield(t, 'c_tm_zone')))
     return w_obj
 
 def _gettmarg(space, w_tup, allowNone=True):
@@ -568,9 +574,9 @@
         return pbuf
 
     tup_w = space.fixedview(w_tup)
-    if len(tup_w) != 9:
+    if len(tup_w) < 9:
         raise oefmt(space.w_TypeError,
-                    "argument must be sequence of length 9, not %d",
+                    "argument must be sequence of at least length 9, not %d",
                     len(tup_w))
 
     y = space.c_int_w(tup_w[0])
@@ -591,13 +597,16 @@
     rffi.setintfield(glob_buf, 'c_tm_wday', space.c_int_w(tup_w[6]))
     rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
     rffi.setintfield(glob_buf, 'c_tm_isdst', space.c_int_w(tup_w[8]))
-    if _POSIX:
-        if _CYGWIN:
-            pass
-        else:
-            # actually never happens, but makes annotator happy
-            glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
-            rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
+    if HAS_TM_ZONE:
+        #tm_zone = encode_utf8(space, space.unicode_w(tup_w[9]), allow_surrogates=True)
+        #glob_buf.c_tm_zone = rffi.str2charp(tm_zone)
+        # TODO using str2charp allocates an object that is never freed
+        # find a solution for this memory leak
+        glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
+        rffi.setintfield(glob_buf, 'c_tm_gmtoff', space.c_int_w(tup_w[10]))
+    else:
+        glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
+        rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
 
     # tm_wday does not need checking of its upper-bound since taking "%
     #  7" in _gettmarg() automatically restricts the range.

From pypy.commits at gmail.com  Tue Jan  3 09:46:16 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 06:46:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-time: better fit the visible field concept
 with extra_fields in _structseq.py
Message-ID: <586bb938.c64bc20a.84589.7fc4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-time
Changeset: r89334:048cc5e4d429
Date: 2017-01-03 15:31 +0100
http://bitbucket.org/pypy/pypy/changeset/048cc5e4d429/

Log:	better fit the visible field concept with extra_fields in
	_structseq.py

diff --git a/lib_pypy/_structseq.py b/lib_pypy/_structseq.py
--- a/lib_pypy/_structseq.py
+++ b/lib_pypy/_structseq.py
@@ -46,11 +46,14 @@
 
         extra_fields = sorted(fields_by_index.items())
         n_sequence_fields = 0
-        extra_off = 0
         if 'n_sequence_fields' in dict:
             n_sequence_fields = dict['n_sequence_fields']
-            extra_fields = extra_fields[:n_sequence_fields]
-            extra_off = n_fields - n_sequence_fields
+            extra_fields = extra_fields[n_sequence_fields:]
+            seq = n_sequence_fields
+            # pop all fields that are still in sequence!
+            while extra_fields and extra_fields[0][0] == seq:
+                extra_fields.pop(0)
+                seq += 1
         else:
             while extra_fields and extra_fields[0][0] == n_sequence_fields:
                 extra_fields.pop(0)
@@ -59,13 +62,12 @@
         dict['n_sequence_fields'] = n_sequence_fields
         dict['n_unnamed_fields'] = 0     # no fully anonymous fields in PyPy
 
-        extra_fields = [field for index, field in extra_fields[extra_off:]]
+        extra_fields = [field for index, field in extra_fields]
         for i,field in enumerate(extra_fields):
             field.index = None     # no longer relevant
 
         assert '__new__' not in dict
         dict['_extra_fields'] = tuple(extra_fields)
-        dict['_extra_off'] = extra_off
         dict['__new__'] = structseq_new
         dict['__reduce__'] = structseq_reduce
         dict['__setattr__'] = structseq_setattr
@@ -89,7 +91,7 @@
             msg = "at least"
         else:
             msg = "exactly"
-        raise TypeError("expected a sequence with %s %d items. has %d 1" % (
+        raise TypeError("expected a sequence with %s %d items. has %d" % (
             msg, visible_count, length))
     if length > visible_count:
         if length > real_count:
@@ -97,15 +99,14 @@
                 msg = "at most"
             else:
                 msg = "exactly"
-            raise TypeError("expected a sequence with %s %d items. has %d 2" \
+            raise TypeError("expected a sequence with %s %d items. has %d" \
                             % (msg, real_count, length))
         for field, value in zip(cls._extra_fields, sequence[visible_count:]):
             name = field.__name__
             if name in dict:
                 raise TypeError("duplicate value for %r" % (name,))
             dict[name] = value
-        extra_off = cls._extra_off
-        sequence = sequence[:visible_count+extra_off]
+        sequence = sequence[:visible_count]
     result = tuple.__new__(cls, sequence)
     object.__setattr__(result, '__dict__', dict)
     for field in cls._extra_fields:
@@ -124,9 +125,11 @@
 
 def structseq_repr(self):
     fields = {}
+    visible_count = self.n_sequence_fields
     for field in type(self).__dict__.values():
-        if isinstance(field, structseqfield):
+        if isinstance(field, structseqfield) and \
+           field._index <= visible_count:
             fields[field._index] = field
     parts = ["%s=%r" % (fields[index].__name__, value)
-             for index, value in enumerate(self)]
+            for index, value in enumerate(self[:visible_count])]
     return "%s(%s)" % (self._name, ", ".join(parts))
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -597,16 +597,16 @@
     rffi.setintfield(glob_buf, 'c_tm_wday', space.c_int_w(tup_w[6]))
     rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
     rffi.setintfield(glob_buf, 'c_tm_isdst', space.c_int_w(tup_w[8]))
-    if HAS_TM_ZONE:
-        #tm_zone = encode_utf8(space, space.unicode_w(tup_w[9]), allow_surrogates=True)
-        #glob_buf.c_tm_zone = rffi.str2charp(tm_zone)
-        # TODO using str2charp allocates an object that is never freed
-        # find a solution for this memory leak
-        glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
-        rffi.setintfield(glob_buf, 'c_tm_gmtoff', space.c_int_w(tup_w[10]))
-    else:
-        glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
-        rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
+    #
+    glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
+    rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
+    if HAS_TM_ZONE :
+        if len(tup_w) >= 10:
+            # XXX str2charp leaks the object
+            tm_zone = encode_utf8(space, space.unicode_w(tup_w[9]), allow_surrogates=True)
+            glob_buf.c_tm_zone = rffi.str2charp(tm_zone, track_allocation=False)
+        if len(tup_w) >= 11:
+            rffi.setintfield(glob_buf, 'c_tm_gmtoff', space.c_int_w(tup_w[10]))
 
     # tm_wday does not need checking of its upper-bound since taking "%
     #  7" in _gettmarg() automatically restricts the range.

From pypy.commits at gmail.com  Tue Jan  3 09:46:18 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 06:46:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-time: introduce invisible fields,
 now the test_time.py suite passes
Message-ID: <586bb93a.581d1c0a.853b9.1b08@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-time
Changeset: r89335:290ed01b7833
Date: 2017-01-03 15:45 +0100
http://bitbucket.org/pypy/pypy/changeset/290ed01b7833/

Log:	introduce invisible fields, now the test_time.py suite passes

diff --git a/lib_pypy/_structseq.py b/lib_pypy/_structseq.py
--- a/lib_pypy/_structseq.py
+++ b/lib_pypy/_structseq.py
@@ -46,12 +46,16 @@
 
         extra_fields = sorted(fields_by_index.items())
         n_sequence_fields = 0
+        invis_fields = []
         if 'n_sequence_fields' in dict:
             n_sequence_fields = dict['n_sequence_fields']
             extra_fields = extra_fields[n_sequence_fields:]
             seq = n_sequence_fields
             # pop all fields that are still in sequence!
             while extra_fields and extra_fields[0][0] == seq:
+                field = extra_fields[0][1]
+                field.index = None
+                invis_fields.append(field)
                 extra_fields.pop(0)
                 seq += 1
         else:
@@ -68,6 +72,7 @@
 
         assert '__new__' not in dict
         dict['_extra_fields'] = tuple(extra_fields)
+        dict['_invis_fields'] = tuple(invis_fields)
         dict['__new__'] = structseq_new
         dict['__reduce__'] = structseq_reduce
         dict['__setattr__'] = structseq_setattr
@@ -101,7 +106,12 @@
                 msg = "exactly"
             raise TypeError("expected a sequence with %s %d items. has %d" \
                             % (msg, real_count, length))
-        for field, value in zip(cls._extra_fields, sequence[visible_count:]):
+        for field, value in zip(cls._invis_fields, sequence[visible_count:real_count]):
+            name = field.__name__
+            if name in dict:
+                raise TypeError("duplicate value for %r" % (name,))
+            dict[name] = value
+        for field, value in zip(cls._extra_fields, sequence[real_count:]):
             name = field.__name__
             if name in dict:
                 raise TypeError("duplicate value for %r" % (name,))

From pypy.commits at gmail.com  Tue Jan  3 09:57:56 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 06:57:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-time: removed leak,
 simplified (removed invisfields, not needed)
Message-ID: <586bbbf4.581d1c0a.853b9.1f6e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-time
Changeset: r89336:6498ac1ab56d
Date: 2017-01-03 15:57 +0100
http://bitbucket.org/pypy/pypy/changeset/6498ac1ab56d/

Log:	removed leak, simplified (removed invisfields, not needed)

diff --git a/lib_pypy/_structseq.py b/lib_pypy/_structseq.py
--- a/lib_pypy/_structseq.py
+++ b/lib_pypy/_structseq.py
@@ -50,14 +50,6 @@
         if 'n_sequence_fields' in dict:
             n_sequence_fields = dict['n_sequence_fields']
             extra_fields = extra_fields[n_sequence_fields:]
-            seq = n_sequence_fields
-            # pop all fields that are still in sequence!
-            while extra_fields and extra_fields[0][0] == seq:
-                field = extra_fields[0][1]
-                field.index = None
-                invis_fields.append(field)
-                extra_fields.pop(0)
-                seq += 1
         else:
             while extra_fields and extra_fields[0][0] == n_sequence_fields:
                 extra_fields.pop(0)
@@ -72,7 +64,6 @@
 
         assert '__new__' not in dict
         dict['_extra_fields'] = tuple(extra_fields)
-        dict['_invis_fields'] = tuple(invis_fields)
         dict['__new__'] = structseq_new
         dict['__reduce__'] = structseq_reduce
         dict['__setattr__'] = structseq_setattr
@@ -106,12 +97,7 @@
                 msg = "exactly"
             raise TypeError("expected a sequence with %s %d items. has %d" \
                             % (msg, real_count, length))
-        for field, value in zip(cls._invis_fields, sequence[visible_count:real_count]):
-            name = field.__name__
-            if name in dict:
-                raise TypeError("duplicate value for %r" % (name,))
-            dict[name] = value
-        for field, value in zip(cls._extra_fields, sequence[real_count:]):
+        for field, value in zip(cls._extra_fields, sequence[visible_count:]):
             name = field.__name__
             if name in dict:
                 raise TypeError("duplicate value for %r" % (name,))
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -237,6 +237,7 @@
                  (HAS_CLOCK_GETTIME and (HAS_CLOCK_HIGHRES or HAS_CLOCK_MONOTONIC)))
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
+glob_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
 
 if _WIN:
     _GetSystemTimeAsFileTime = rwin32.winexternal('GetSystemTimeAsFileTime',
@@ -602,9 +603,15 @@
     rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
     if HAS_TM_ZONE :
         if len(tup_w) >= 10:
-            # XXX str2charp leaks the object
+            # NOTE this is not cleanly solved, the global variable glob_tm_zone
+            # saves the string that is later deleted when this function is called again
+            # an refactoring of this module could remove this
             tm_zone = encode_utf8(space, space.unicode_w(tup_w[9]), allow_surrogates=True)
-            glob_buf.c_tm_zone = rffi.str2charp(tm_zone, track_allocation=False)
+            malloced_str = rffi.str2charp(tm_zone, track_allocation=False)
+            if glob_tm_zone != lltype.nullptr(rffi.CCHARP.TO):
+                rffi.freecharp(glob_tm_zone, track_allocation=False)
+            glob_tm_zone = malloced_str
+            glob_buf.c_tm_zone = malloced_str
         if len(tup_w) >= 11:
             rffi.setintfield(glob_buf, 'c_tm_gmtoff', space.c_int_w(tup_w[10]))
 

From pypy.commits at gmail.com  Tue Jan  3 10:10:36 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 03 Jan 2017 07:10:36 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-time: variable not visible
Message-ID: <586bbeec.c4251c0a.6f1d.14ca@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-time
Changeset: r89337:29578064d2db
Date: 2017-01-03 16:02 +0100
http://bitbucket.org/pypy/pypy/changeset/29578064d2db/

Log:	variable not visible

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -606,6 +606,7 @@
             # NOTE this is not cleanly solved, the global variable glob_tm_zone
             # saves the string that is later deleted when this function is called again
             # an refactoring of this module could remove this
+            global glob_tm_zone
             tm_zone = encode_utf8(space, space.unicode_w(tup_w[9]), allow_surrogates=True)
             malloced_str = rffi.str2charp(tm_zone, track_allocation=False)
             if glob_tm_zone != lltype.nullptr(rffi.CCHARP.TO):

From pypy.commits at gmail.com  Tue Jan  3 11:03:01 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 03 Jan 2017 08:03:01 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: fix tests
Message-ID: <586bcb35.0f341c0a.add22.0368@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89338:ecfe2f400573
Date: 2017-01-03 16:02 +0000
http://bitbucket.org/pypy/pypy/changeset/ecfe2f400573/

Log:	fix tests

diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,9 +89,9 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest1'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
                 == ('Py_ssize_t', 'Py_ssize_t arg0'))
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest2'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
                 == ('Py_ssize_t *', 'Py_ssize_t *arg0'))
 
         PyPy_TypedefTest1(space, 0)
@@ -100,7 +100,7 @@
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
 
- at pytest.mark.skipif(os.environ.get('USER')=='root', 
+ at pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
     api.copy_header_files(tmpdir, True)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -27,8 +27,9 @@
 
 class TestApi:
     def test_signature(self):
-        assert 'PyModule_Check' in api.FUNCTIONS
-        assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
+        common_functions = api.FUNCTIONS_BY_HEADER[api.pypy_decl]
+        assert 'PyModule_Check' in common_functions
+        assert common_functions['PyModule_Check'].argtypes == [api.PyObject]
 
 
 class SpaceCompiler(SystemCompilationInfo):

From pypy.commits at gmail.com  Tue Jan  3 11:45:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 03 Jan 2017 08:45:26 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Simplify declaration of
 _PyExc_xxx objects
Message-ID: <586bd526.45f6c20a.23a87.83cc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89339:b0f085b855a5
Date: 2017-01-03 16:44 +0000
http://bitbucket.org/pypy/pypy/changeset/b0f085b855a5/

Log:	Simplify declaration of _PyExc_xxx objects

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -572,7 +572,9 @@
     # PyExc_AttributeError, PyExc_OverflowError, PyExc_ImportError,
     # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError,
     # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ...
-    for exc_name in exceptions.Module.interpleveldefs.keys():
+    global all_exceptions
+    all_exceptions = list(exceptions.Module.interpleveldefs)
+    for exc_name in all_exceptions:
         register_global('PyExc_' + exc_name,
             'PyTypeObject*',
             'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, ))
@@ -1055,15 +1057,8 @@
     """ % dict(members=structmembers)
 
     global_objects = []
-    for name, (typ, expr) in GLOBALS.iteritems():
-        if '#' in name:
-            continue
-        if typ == 'PyDateTime_CAPI*':
-            continue
-        elif name.startswith('PyExc_'):
-            global_objects.append('%s _%s;' % (typ[:-1], name))
-        else:
-            global_objects.append('%s %s = NULL;' % (typ, name))
+    for name in all_exceptions:
+        global_objects.append('PyTypeObject _PyExc_%s;' % name)
     global_code = '\n'.join(global_objects)
 
     prologue = ("#include <Python.h>\n"

From pypy.commits at gmail.com  Tue Jan  3 11:58:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 08:58:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Support bytearrays in unix domain socket
 names
Message-ID: <586bd82e.6a5cc20a.d0b08.8d9e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89340:d1e27385329f
Date: 2017-01-03 17:57 +0100
http://bitbucket.org/pypy/pypy/changeset/d1e27385329f/

Log:	Support bytearrays in unix domain socket names

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -136,6 +136,8 @@
         # Not using space.fsencode_w since Linux allows embedded NULs.
         if space.isinstance_w(w_address, space.w_unicode):
             w_address = space.fsencode(w_address)
+        elif space.isinstance_w(w_address, space.w_bytearray):
+            w_address = space.newbytes(space.charbuf_w(w_address))
         bytelike = space.bytes_w(w_address) # getarg_w('y*', w_address)
         return rsocket.UNIXAddress(bytelike)
     if rsocket.HAS_AF_NETLINK and family == rsocket.AF_NETLINK:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -577,7 +577,9 @@
         oldcwd = os.getcwd()
         os.chdir(self.udir)
         try:
-            sockpath = 'app_test_unix_socket_connect'
+          for sockpath in ['app_test_unix_socket_connect',
+                           b'b_app_test_unix_socket_connect',
+                           bytearray(b'ba_app_test_unix_socket_connect')]:
 
             serversock = _socket.socket(_socket.AF_UNIX)
             serversock.bind(sockpath)

From pypy.commits at gmail.com  Tue Jan  3 12:08:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 09:08:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: generalize the error messages expected by
 this test
Message-ID: <586bda86.4673c20a.fafa7.9b44@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89341:33162b5a4231
Date: 2017-01-03 18:07 +0100
http://bitbucket.org/pypy/pypy/changeset/33162b5a4231/

Log:	generalize the error messages expected by this test

diff --git a/lib-python/3/test/test_socket.py b/lib-python/3/test/test_socket.py
--- a/lib-python/3/test/test_socket.py
+++ b/lib-python/3/test/test_socket.py
@@ -711,6 +711,8 @@
 
     def testSendtoErrors(self):
         # Testing that sendto doesn't masks failures. See #10169.
+        # PyPy note: made the test accept broader messages: PyPy's
+        # messages are equivalent but worded differently.
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         self.addCleanup(s.close)
         s.bind(('', 0))
@@ -718,33 +720,37 @@
         # 2 args
         with self.assertRaises(TypeError) as cm:
             s.sendto('\u2620', sockname)
-        self.assertEqual(str(cm.exception),
-                         "a bytes-like object is required, not 'str'")
+        self.assertIn(str(cm.exception),
+                      ["a bytes-like object is required, not 'str'", # cpython
+                       "'str' does not support the buffer interface"]) # pypy
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, sockname)
-        self.assertEqual(str(cm.exception),
-                         "a bytes-like object is required, not 'complex'")
+        self.assertIn(str(cm.exception),
+                      ["a bytes-like object is required, not 'complex'",
+                       "'complex' does not support the buffer interface"])
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', None)
-        self.assertIn('not NoneType',str(cm.exception))
+        self.assertIn('NoneType', str(cm.exception))
         # 3 args
         with self.assertRaises(TypeError) as cm:
             s.sendto('\u2620', 0, sockname)
-        self.assertEqual(str(cm.exception),
-                         "a bytes-like object is required, not 'str'")
+        self.assertIn(str(cm.exception),
+                      ["a bytes-like object is required, not 'str'",
+                       "'str' does not support the buffer interface"])
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, 0, sockname)
-        self.assertEqual(str(cm.exception),
-                         "a bytes-like object is required, not 'complex'")
+        self.assertIn(str(cm.exception),
+                      ["a bytes-like object is required, not 'complex'",
+                       "'complex' does not support the buffer interface"])
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', 0, None)
-        self.assertIn('not NoneType', str(cm.exception))
+        self.assertIn('NoneType', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', 'bar', sockname)
-        self.assertIn('an integer is required', str(cm.exception))
+        self.assertIn('integer', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', None, None)
-        self.assertIn('an integer is required', str(cm.exception))
+        self.assertIn('integer', str(cm.exception))
         # wrong number of args
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo')

From pypy.commits at gmail.com  Tue Jan  3 12:16:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 09:16:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix two tests,
 remove two other old-style class tests
Message-ID: <586bdc81.6737c20a.6877a.b20f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89342:e3a207b54c14
Date: 2017-01-03 18:16 +0100
http://bitbucket.org/pypy/pypy/changeset/e3a207b54c14/

Log:	Fix two tests, remove two other old-style class tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -105,7 +105,11 @@
         # -------------------------------
         entry_bridge, = log.loops_by_filename(self.filepath, is_entry_bridge=True)
         ops = entry_bridge.ops_by_id('mutate', opcode='LOAD_ATTR')
-        assert log.opnames(ops) == ['guard_value', 'guard_not_invalidated',
+        # in PyPy3 we get a dummy getfield_gc_r (*) for
+        # W_UnicodeObject._utf8, which is usually removed by the backend
+        assert log.opnames(ops) == ['guard_value',
+                                    'getfield_gc_r',  # <= (*)
+                                    'guard_not_invalidated',
                                     'getfield_gc_i']
         # the STORE_ATTR is folded away
         assert list(entry_bridge.ops_by_id('meth1', opcode='STORE_ATTR')) == []
@@ -153,7 +157,11 @@
         # -------------------------------
         entry_bridge, = log.loops_by_filename(self.filepath, is_entry_bridge=True)
         ops = entry_bridge.ops_by_id('mutate', opcode='LOAD_ATTR')
-        assert log.opnames(ops) == ['guard_value', 'guard_not_invalidated',
+        # in PyPy3 we get a dummy getfield_gc_r (*) for
+        # W_UnicodeObject._utf8, which is usually removed by the backend
+        assert log.opnames(ops) == ['guard_value',
+                                    'getfield_gc_r',   # <= (*)
+                                    'guard_not_invalidated',
                                     'getfield_gc_r', 'guard_nonnull_class',
                                     'getfield_gc_r', 'guard_value', # type check on the attribute
                                     ]
@@ -187,76 +195,6 @@
 
         """)
 
-    def test_oldstyle_methcall(self):
-        def main():
-            def g(): pass
-            class A:
-                def f(self):
-                    return self.x + 1
-            class I(A):
-                pass
-            class J(I):
-                pass
-
-
-            class B(J):
-                def __init__(self, x):
-                    self.x = x
-
-            i = 0
-            b = B(1)
-            while i < 1000:
-                g()
-                v = b.f() # ID: meth
-                i += v
-            return i
-
-        log = self.run(main, [], threshold=80)
-        loop, = log.loops_by_filename(self.filepath, is_entry_bridge=True)
-        assert loop.match_by_id('meth',
-        '''
-    guard_nonnull_class(p18, ..., descr=...)
-    p52 = getfield_gc_r(p18, descr=...) # read map
-    guard_value(p52, ConstPtr(ptr53), descr=...)
-    p54 = getfield_gc_r(p18, descr=...) # read class
-    guard_value(p54, ConstPtr(ptr55), descr=...)
-    p56 = force_token() # done
-        ''')
-
-
-    def test_oldstyle_newstyle_mix(self):
-        def main():
-            class A:
-                pass
-
-            class B(object, A):
-                def __init__(self, x):
-                    self.x = x
-
-            i = 0
-            b = B(1)
-            while i < 100:
-                v = b.x # ID: loadattr1
-                v = b.x # ID: loadattr2
-                i += v
-            return i
-
-        log = self.run(main, [], threshold=80)
-        loop, = log.loops_by_filename(self.filepath)
-        assert loop.match_by_id('loadattr1',
-        '''
-        guard_not_invalidated(descr=...)
-        i19 = call_i(ConstClass(ll_call_lookup_function), _, _, _, 0, descr=...)
-        guard_no_exception(descr=...)
-        i22 = int_lt(i19, 0)
-        guard_true(i22, descr=...)
-        i26 = call_i(ConstClass(ll_call_lookup_function), _, _, _, 0, descr=...)
-        guard_no_exception(descr=...)
-        i29 = int_lt(i26, 0)
-        guard_true(i29, descr=...)
-        ''')
-        assert loop.match_by_id('loadattr2', "")   # completely folded away
-
     def test_python_contains(self):
         def main():
             class A(object):

From pypy.commits at gmail.com  Tue Jan  3 12:30:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 09:30:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: A failing test that is the root cause of
 the failures of tests checking
Message-ID: <586bdfb2.8a281c0a.b5464.2aff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89343:ddae4777286a
Date: 2017-01-03 18:29 +0100
http://bitbucket.org/pypy/pypy/changeset/ddae4777286a/

Log:	A failing test that is the root cause of the failures of tests
	checking Warnings

diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -18,6 +18,16 @@
         _warnings.warn("some message", Warning)
         _warnings.warn(("some message",1), Warning)
 
+    def test_use_builtin__warnings(self):
+        """Check that the stdlib warnings.py module manages to import our
+        _warnings module.  If something is missing, it doesn't, and silently
+        continues.  Then things don't reliably work: either the
+        functionality of the pure Python version is subtly different, or
+        more likely we get confusion because of a half-imported _warnings.
+        """
+        import warnings
+        assert not hasattr(warnings, '_filters_version')
+
     def test_lineno(self):
         import warnings, _warnings, sys
         with warnings.catch_warnings(record=True) as w:

From pypy.commits at gmail.com  Tue Jan  3 12:50:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 09:50:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Use a simpler,
 direct method of implementing space.warn()
Message-ID: <586be461.448e1c0a.328c8.3b74@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89344:4be96d48f937
Date: 2017-01-03 18:49 +0100
http://bitbucket.org/pypy/pypy/changeset/4be96d48f937/

Log:	Use a simpler, direct method of implementing space.warn()

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1836,11 +1836,8 @@
         return fd
 
     def warn(self, w_msg, w_warningcls, stacklevel=2):
-        self.appexec([w_msg, w_warningcls, self.wrap(stacklevel)],
-                     """(msg, warningcls, stacklevel):
-            import _warnings
-            _warnings.warn(msg, warningcls, stacklevel=stacklevel)
-        """)
+        from pypy.module._warnings.interp_warnings import warn
+        warn(self, w_msg, w_warningcls, self.newint(stacklevel - 1))
 
 
 class AppExecCache(SpaceCache):

From pypy.commits at gmail.com  Tue Jan  3 13:27:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 10:27:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Update the _warnings module to CPython 3.5
Message-ID: <586bed1f.c5371c0a.f6936.5f42@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89345:018a60dc3997
Date: 2017-01-03 19:26 +0100
http://bitbucket.org/pypy/pypy/changeset/018a60dc3997/

Log:	Update the _warnings module to CPython 3.5

diff --git a/pypy/module/_warnings/__init__.py b/pypy/module/_warnings/__init__.py
--- a/pypy/module/_warnings/__init__.py
+++ b/pypy/module/_warnings/__init__.py
@@ -7,6 +7,7 @@
     interpleveldefs = {
         'warn'         : 'interp_warnings.warn',
         'warn_explicit': 'interp_warnings.warn_explicit',
+        '_filters_mutated': 'interp_warnings.filters_mutated',
     }
 
     appleveldefs = {
diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -11,6 +11,10 @@
         self.init_filters(space)
         self.w_once_registry = space.newdict()
         self.w_default_action = space.wrap("default")
+        self.filters_mutated(space)
+
+    def filters_mutated(self, space):
+        self.w_filters_version = space.call_function(space.w_object)
 
     def init_filters(self, space):
         filters_w = []
@@ -32,6 +36,8 @@
         filters_w.append(create_filter(
             space, space.w_BytesWarning, action))
 
+        # note: in CPython, resource usage warnings are enabled by default
+        # in pydebug mode
         filters_w.append(create_filter(
             space, space.w_ResourceWarning, "ignore"))
 
@@ -61,19 +67,52 @@
         w_category = space.w_UserWarning
 
     # Validate category
-    if not space.abstract_issubclass_w(w_category, space.w_Warning):
+    try:
+        if not space.abstract_issubclass_w(w_category, space.w_Warning):
+            raise oefmt(space.w_ValueError,
+                        "category is not a subclass of Warning")
+    except OperationError as e:
+        if e.async(space):
+            raise
         raise oefmt(space.w_ValueError,
-                    "category is not a subclass of Warning")
+                    "category must be a Warning subclass, not '%T'",
+                    w_category)
 
     return w_category
 
+def is_internal_frame(space, frame):
+    if frame is None:
+        return False
+    code = frame.getcode()
+    if code is None or code.co_filename is None:
+        return False
+    # XXX XXX HAAAACK copied directly from CPython, which I'm particularly
+    # unhappy about, but I can't do anything more than say "bah"
+    return "importlib" in code.co_filename and "_bootstrap" in code.co_filename
+
+def next_external_frame(space, frame):
+    ec = space.getexecutioncontext()
+    while True:
+        frame = ec.getnextframe_nohidden(frame)
+        if frame is None or not is_internal_frame(space, frame):
+            return frame
+
 def setup_context(space, stacklevel):
     # Setup globals and lineno
     ec = space.getexecutioncontext()
+
+    # Direct copy of CPython's logic, which has grown its own notion of
+    # "internal frames".  xxx not sure I understand this logic.
     frame = ec.gettopframe_nohidden()
-    while frame and stacklevel > 1:
-        frame = ec.getnextframe_nohidden(frame)
-        stacklevel -= 1
+    if stacklevel <= 0 or is_internal_frame(space, frame):
+        while stacklevel > 1 and frame:
+            frame = ec.getnextframe_nohidden(frame)
+            stacklevel -= 1
+    else:
+        while stacklevel > 1 and frame:
+            frame = next_external_frame(space, frame)
+            stacklevel -= 1
+
     if frame:
         w_globals = frame.get_w_globals()
         lineno = frame.get_last_lineno()
@@ -116,8 +155,7 @@
             w_filename = w_module
     else:
         lc_filename = filename.lower()
-        if (lc_filename.endswith(".pyc") or 
-            lc_filename.endswith(".pyo")):
+        if lc_filename.endswith(".pyc"):
             # strip last character
             w_filename = space.wrap(filename[:-1])
 
@@ -173,27 +211,30 @@
     return already_warned(space, w_registry, w_key, should_set=True)
 
 def already_warned(space, w_registry, w_key, should_set=False):
-    try:
-        w_warned = space.getitem(w_registry, w_key)
-    except OperationError as e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if should_set:
-            space.setitem(w_registry, w_key, space.w_True)
-        return False
+    w_version_obj = space.finditem_str(w_registry, "version")
+    state = space.fromcache(State)
+    if w_version_obj is not state.w_filters_version:
+        space.call_method(w_registry, "clear")
+        space.setitem_str(w_registry, "version", state.w_filters_version)
     else:
-        return space.is_true(w_warned)
+        w_already_warned = space.finditem(w_registry, w_key)
+        if w_already_warned is not None and space.is_true(w_already_warned):
+            return True
+    # This warning wasn't found in the registry, set it.
+    if should_set:
+        space.setitem(w_registry, w_key, space.w_True)
+    return False
 
 def normalize_module(space, w_filename):
-    if not space.is_true(w_filename):
+    filename = space.identifier_w(w_filename)
+    if len(filename) == 0:
         return space.wrap("<unknown>")
-
-    filename = space.str_w(w_filename)
     if filename.endswith(".py"):
         n = len(filename) - 3
         assert n >= 0
         filename = filename[:n]
-    return space.wrap(filename)
+        return space.newutf8(filename)
+    return w_filename
 
 def show_warning(space, w_filename, lineno, w_text, w_category,
                  w_sourceline=None):
@@ -308,6 +349,7 @@
 
 @unwrap_spec(stacklevel=int)
 def warn(space, w_message, w_category=None, stacklevel=1):
+    "Issue a warning, or maybe ignore it or raise an exception."
     w_category = get_category(space, w_message, w_category);
     do_warn(space, w_message, w_category, stacklevel)
 
@@ -350,9 +392,13 @@
              w_module_globals = WrappedDefault(None))
 def warn_explicit(space, w_message, w_category, w_filename, lineno,
                   w_module=None, w_registry=None, w_module_globals=None):
+    "Low-level inferface to warnings functionality."
 
     w_source_line = get_source_line(space, w_module_globals, lineno)
 
     do_warn_explicit(space, w_category, w_message,
                      (w_filename, lineno, w_module, w_registry),
                      w_source_line)
+
+def filters_mutated(space):
+    space.fromcache(State).filters_mutated(space)

From pypy.commits at gmail.com  Tue Jan  3 13:43:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 10:43:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Need to remove frames from both
 importlib._bootstrap and
Message-ID: <586bf0cd.a285c20a.d98b9.dda0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89346:05c795ec40f6
Date: 2017-01-03 19:42 +0100
http://bitbucket.org/pypy/pypy/changeset/05c795ec40f6/

Log:	Need to remove frames from both importlib._bootstrap and
	importlib._bootstrap_external

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -327,11 +327,11 @@
         """Set the current traceback."""
         self._application_traceback = traceback
 
-    def remove_traceback_module_frames(self, module_name):
+    def remove_traceback_module_frames(self, *module_names):
         from pypy.interpreter.pytraceback import PyTraceback
         tb = self._application_traceback
         while tb is not None and isinstance(tb, PyTraceback):
-            if tb.frame.pycode.co_filename != module_name:
+            if tb.frame.pycode.co_filename not in module_names:
                 break
             tb = tb.next
         self._application_traceback = tb
diff --git a/pypy/module/_frozen_importlib/interp_import.py b/pypy/module/_frozen_importlib/interp_import.py
--- a/pypy/module/_frozen_importlib/interp_import.py
+++ b/pypy/module/_frozen_importlib/interp_import.py
@@ -15,7 +15,9 @@
         return space.call_args(
             space.fromcache(FrozenCache).w_frozen_import, __args__)
     except OperationError as e:
-        e.remove_traceback_module_frames('<frozen importlib._bootstrap>')
+        e.remove_traceback_module_frames(
+              '<frozen importlib._bootstrap>',
+              '<frozen importlib._bootstrap_external>')
         raise
 import_with_frames_removed = interp2app(import_with_frames_removed,
                                         app_name='__import__')

From pypy.commits at gmail.com  Tue Jan  3 13:51:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 10:51:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <586bf299.46bb1c0a.a84b0.7bc2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89347:c5679870cc23
Date: 2017-01-03 19:50 +0100
http://bitbucket.org/pypy/pypy/changeset/c5679870cc23/

Log:	fix

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1836,8 +1836,12 @@
         return fd
 
     def warn(self, w_msg, w_warningcls, stacklevel=2):
-        from pypy.module._warnings.interp_warnings import warn
-        warn(self, w_msg, w_warningcls, self.newint(stacklevel - 1))
+        from pypy.module._warnings.interp_warnings import do_warn
+
+        # 'w_warningcls' must a Warning subclass
+        if not we_are_translated():
+            assert self.issubtype_w(w_warningcls, self.w_Warning)
+        do_warn(self, w_msg, w_warningcls, stacklevel - 1)
 
 
 class AppExecCache(SpaceCache):

From pypy.commits at gmail.com  Tue Jan  3 15:50:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jan 2017 12:50:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Not exactly sure why,
 but in a translated pypy we also see this
Message-ID: <586c0e90.0d1a1c0a.8731c.b719@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89348:88b6bdd9f8d3
Date: 2017-01-03 20:42 +0100
http://bitbucket.org/pypy/pypy/changeset/88b6bdd9f8d3/

Log:	Not exactly sure why, but in a translated pypy we also see this
	alternative co_filename

diff --git a/pypy/module/_frozen_importlib/interp_import.py b/pypy/module/_frozen_importlib/interp_import.py
--- a/pypy/module/_frozen_importlib/interp_import.py
+++ b/pypy/module/_frozen_importlib/interp_import.py
@@ -17,7 +17,8 @@
     except OperationError as e:
         e.remove_traceback_module_frames(
               '<frozen importlib._bootstrap>',
-              '<frozen importlib._bootstrap_external>')
+              '<frozen importlib._bootstrap_external>',
+              '<builtin>/frozen importlib._bootstrap_external')
         raise
 import_with_frames_removed = interp2app(import_with_frames_removed,
                                         app_name='__import__')

From pypy.commits at gmail.com  Tue Jan  3 19:08:21 2017
From: pypy.commits at gmail.com (mjacob)
Date: Tue, 03 Jan 2017 16:08:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Update test to check for ValueError
 instead of TypeError.
Message-ID: <586c3cf5.2919c20a.cae20.b07f@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r89349:6ca32c9a5a9c
Date: 2017-01-04 01:07 +0100
http://bitbucket.org/pypy/pypy/changeset/6ca32c9a5a9c/

Log:	Update test to check for ValueError instead of TypeError.

	This is also what CPython 3.5.2 raises in this case.

diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -150,7 +150,7 @@
         v = f1.descr_pow(self.space, f2, f3)
         assert v.intval == pow(x, y, z)
         f1, f2, f3 = [iobj.W_IntObject(i) for i in (10, -1, 42)]
-        self.space.raises_w(self.space.w_TypeError,
+        self.space.raises_w(self.space.w_ValueError,
                             f1.descr_pow, self.space, f2, f3)
         f1, f2, f3 = [iobj.W_IntObject(i) for i in (10, 5, 0)]
         self.space.raises_w(self.space.w_ValueError,

From pypy.commits at gmail.com  Tue Jan  3 19:25:22 2017
From: pypy.commits at gmail.com (mjacob)
Date: Tue, 03 Jan 2017 16:25:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add 'select' to list of used modules in
 test.
Message-ID: <586c40f2.832cc20a.e7ec1.00e2@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r89350:3c9879434b91
Date: 2017-01-04 01:20 +0100
http://bitbucket.org/pypy/pypy/changeset/3c9879434b91/

Log:	Add 'select' to list of used modules in test.

diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -213,7 +213,8 @@
 
 class AppTestPyexpat2:
     spaceconfig = dict(usemodules=['_rawffi', 'pyexpat', 'itertools',
-                                   '_socket', 'time', 'struct', 'binascii'])
+                                   '_socket', 'time', 'struct', 'binascii',
+                                   'select'])
 
     def test_django_bug(self):
         xml_str = '<?xml version="1.0" standalone="no"?><!DOCTYPE example SYSTEM "http://example.com/example.dtd"><root/>'

From pypy.commits at gmail.com  Tue Jan  3 19:25:24 2017
From: pypy.commits at gmail.com (mjacob)
Date: Tue, 03 Jan 2017 16:25:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: 2to3: 'returns_unicode' attribute isn't
 present on Python 3.
Message-ID: <586c40f4.0e0a1c0a.5228b.cb97@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r89351:5c8d5e46cf84
Date: 2017-01-04 01:23 +0100
http://bitbucket.org/pypy/pypy/changeset/5c8d5e46cf84/

Log:	2to3: 'returns_unicode' attribute isn't present on Python 3.

diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -31,7 +31,6 @@
         for attr in ('buffer_text', 'namespace_prefixes', 'ordered_attributes',
                      'specified_attributes'):
             test_setget(p, attr)
-        test_setget(p, 'returns_unicode', True)
 
     def test_version(self):
         import pyexpat

From pypy.commits at gmail.com  Tue Jan  3 19:39:25 2017
From: pypy.commits at gmail.com (mjacob)
Date: Tue, 03 Jan 2017 16:39:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: 2to3
Message-ID: <586c443d.ce181c0a.bb41f.ccb8@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r89352:a222998f6e68
Date: 2017-01-04 01:38 +0100
http://bitbucket.org/pypy/pypy/changeset/a222998f6e68/

Log:	2to3

diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -857,11 +857,11 @@
         except SystemError:
             skip("resizing not supported")
         assert m.tell() == 5000
-        assert m.read(14) == ''
-        assert m.read(-1) == ''
+        assert m.read(14) == b''
+        assert m.read(-1) == b''
         raises(ValueError, m.read_byte)
-        assert m.readline() == ''
-        raises(ValueError, m.write_byte, 'b')
-        raises(ValueError, m.write, 'abc')
+        assert m.readline() == b''
+        raises(ValueError, m.write_byte, ord(b'b'))
+        raises(ValueError, m.write, b'abc')
         assert m.tell() == 5000
         m.close()

From pypy.commits at gmail.com  Tue Jan  3 20:02:06 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 03 Jan 2017 17:02:06 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Put the defines for global
 symbols in the decl file in all cases
Message-ID: <586c498e.8c1f1c0a.f8a7f.dbc4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89353:93a816b0600f
Date: 2017-01-04 00:34 +0000
http://bitbucket.org/pypy/pypy/changeset/93a816b0600f/

Log:	Put the defines for global symbols in the decl file in all cases

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1213,7 +1213,7 @@
 def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
-    export_symbols = sorted(SYMBOLS_C) + sorted(GLOBALS)
+    export_symbols = sorted(SYMBOLS_C)
     for name in export_symbols:
         if '#' in name:
             name, header = name.split('#')
@@ -1292,8 +1292,7 @@
         elif name.startswith('PyExc_'):
             typ = 'PyObject*'
             header = pypy_decl
-        if header != pypy_decl:
-            decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
+        decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
         decls[header].append('PyAPI_DATA(%s) %s;' % (typ, name))
 
     for header_name in FUNCTIONS_BY_HEADER.keys():
@@ -1433,10 +1432,8 @@
             name, header = name.split('#')
             assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
             typ = typ[:-1]
-            if header != pypy_decl:
-                # since the #define is not in pypy_macros, do it here
-                mname = mangle_name(prefix, name)
-                include_lines.append('#define %s %s\n' % (name, mname))
+            mname = mangle_name(prefix, name)
+            include_lines.append('#define %s %s\n' % (name, mname))
         elif name.startswith('PyExc_'):
             typ = 'PyTypeObject'
             name = '_' + name

From pypy.commits at gmail.com  Tue Jan  3 20:02:07 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 03 Jan 2017 17:02:07 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Simplify creation of remaining
 #defines in pypy_macros.h
Message-ID: <586c498f.8f95c20a.1cc9c.4c19@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89354:26767850c0c5
Date: 2017-01-04 01:01 +0000
http://bitbucket.org/pypy/pypy/changeset/26767850c0c5/

Log:	Simplify creation of remaining #defines in pypy_macros.h

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1213,18 +1213,10 @@
 def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
-    export_symbols = sorted(SYMBOLS_C)
-    for name in export_symbols:
-        if '#' in name:
-            name, header = name.split('#')
-        else:
-            header = pypy_decl
+    for name in SYMBOLS_C:
         newname = mangle_name(prefix, name)
         assert newname, name
-        if header == pypy_decl:
-            pypy_macros.append('#define %s %s' % (name, newname))
-        if name.startswith("PyExc_"):
-            pypy_macros.append('#define _%s _%s' % (name, newname))
+        pypy_macros.append('#define %s %s' % (name, newname))
 
     # Generate defines
     for macro_name, size in [

From pypy.commits at gmail.com  Tue Jan  3 20:06:34 2017
From: pypy.commits at gmail.com (mjacob)
Date: Tue, 03 Jan 2017 17:06:34 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add newutf8 to fake objspace.
Message-ID: <586c4a9a.06891c0a.4d4b.df1f@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r89355:42f98dcdd48f
Date: 2017-01-04 02:02 +0100
http://bitbucket.org/pypy/pypy/changeset/42f98dcdd48f/

Log:	Add newutf8 to fake objspace.

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -209,6 +209,9 @@
     def newunicode(self, x):
         return w_some_obj()
 
+    def newutf8(self, x):
+        return w_some_obj()
+
     @specialize.argtype(1)
     def wrap(self, x):
         if not we_are_translated():

From pypy.commits at gmail.com  Wed Jan  4 05:05:54 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 04 Jan 2017 02:05:54 -0800 (PST)
Subject: [pypy-commit] pypy default: merge strbuf-as-buffer
Message-ID: <586cc902.c19d1c0a.e9a01.7ca4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r89356:8794d97c98d1
Date: 2017-01-04 10:53 +0100
http://bitbucket.org/pypy/pypy/changeset/8794d97c98d1/

Log:	merge strbuf-as-buffer

diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -32,7 +32,7 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises((TypeError, ValueError),
+        self.assertRaises(TypeError,
                           (c_char * 16).from_buffer, "a" * 16)
 
     def test_fom_buffer_with_offset(self):
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -85,7 +85,17 @@
 
     def from_buffer(self, obj, offset=0):
         size = self._sizeofinstances()
+        if isinstance(obj, (str, unicode)):
+            # hack, buffer(str) will always return a readonly buffer.
+            # CPython calls PyObject_AsWriteBuffer(...) here!
+            # str cannot be modified, thus raise a type error in this case
+            raise TypeError("Cannot use %s as modifiable buffer" % str(type(obj)))
+
+        # why not just call memoryview(obj)[offset:]?
+        # array in Python 2.7 does not support the buffer protocol and will
+        # fail, even though buffer is supported
         buf = buffer(obj, offset, size)
+
         if len(buf) < size:
             raise ValueError(
                 "Buffer size too small (%d instead of at least %d bytes)"
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -4,6 +4,7 @@
 
 from rpython.rlib.buffer import Buffer
 from pypy.interpreter.gateway import unwrap_spec
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
 
 
 class ByteBuffer(Buffer):
@@ -22,6 +23,8 @@
     def setitem(self, index, char):
         self.data[index] = char
 
+    def get_raw_address(self):
+        return nonmoving_raw_ptr_for_resizable_list(self.data)
 
 @unwrap_spec(length=int)
 def bytebuffer(space, length):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -136,6 +136,9 @@
     return _from_buffer(space, w_ctype, w_x)
 
 def _from_buffer(space, w_ctype, w_x):
+    if space.isinstance_w(w_x, space.w_unicode):
+        raise oefmt(space.w_TypeError,
+                        "from_buffer() cannot return the address a unicode")
     buf = _fetch_as_read_buffer(space, w_x)
     if space.isinstance_w(w_x, space.w_str):
         _cdata = get_raw_address_of_string(space, w_x)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3419,24 +3419,28 @@
     BCharA = new_array_type(BCharP, None)
     p1 = from_buffer(BCharA, b"foo")
     assert p1 == from_buffer(BCharA, b"foo")
-    import gc; gc.collect()
-    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
-        # from_buffer(buffer(b"foo")) does not work, because it's not
-        # implemented on pypy; only from_buffer(b"foo") works.
-        py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
-        py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
+        contents = from_buffer(BCharA, buffer(b"foo"))
+        for i in range(len(contents)):
+            assert contents[i] == p1[i]
+        p4 = from_buffer(BCharA, b"f\x00\x00\x00o\x00\x00\x00o\x00\x00\x00")
+        contents = from_buffer(BCharA, buffer(u+"foo"))
+        for i in range(len(contents)):
+            assert contents[i] == p4[i]
     try:
         from __builtin__ import memoryview
     except ImportError:
         pass
     else:
-        py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo"))
+        contents = from_buffer(BCharA, memoryview(b"foo"))
+        for i in range(len(contents)):
+            assert contents[i] == p1[i]
+
 
 def test_from_buffer_bytearray():
     a = bytearray(b"xyz")
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import (
     TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rarithmetic import r_longlong, intmask
@@ -120,6 +121,9 @@
     def setitem(self, index, char):
         self.buf[self.start + index] = char
 
+    def get_raw_address(self):
+        return nonmoving_raw_ptr_for_resizable_list(self.buf)
+
 class BufferedMixin:
     _mixin_ = True
 
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -424,7 +424,7 @@
             w_addr = w_param3
         try:
             addr = self.addr_from_object(space, w_addr)
-            count = self.sock.sendto(data, flags, addr)
+            count = self.sock.sendto(data, len(data), flags, addr)
         except SocketError as e:
             raise converted_error(space, e)
         return space.wrap(count)
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -199,7 +199,9 @@
         raises(TypeError, "buf[MyInt(0):MyInt(5)]")
 
     def test_pypy_raw_address_base(self):
-        raises(ValueError, buffer("foobar")._pypy_raw_address)
-        raises(ValueError, buffer(u"foobar")._pypy_raw_address)
-        a = buffer(bytearray("foobar"))._pypy_raw_address()
+        a = buffer("foobar")._pypy_raw_address()
         assert a != 0
+        b = buffer(u"foobar")._pypy_raw_address()
+        assert b != 0
+        c = buffer(bytearray("foobar"))._pypy_raw_address()
+        assert c != 0
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -56,6 +56,7 @@
         assert u"abc" != memoryview("abc")
 
     def test_pypy_raw_address_base(self):
-        raises(ValueError, memoryview("foobar")._pypy_raw_address)
-        a = memoryview(bytearray("foobar"))._pypy_raw_address()
+        a = memoryview("foobar")._pypy_raw_address()
         assert a != 0
+        b = memoryview(bytearray("foobar"))._pypy_raw_address()
+        assert b != 0
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -158,7 +158,11 @@
 # record that ignore_finalizer() has been called
 GCFLAG_IGNORE_FINALIZER = first_gcflag << 10
 
-_GCFLAG_FIRST_UNUSED = first_gcflag << 11    # the first unused bit
+# shadow objects can have its memory initialized when it is created.
+# It does not need an additional copy in trace out
+GCFLAG_SHADOW_INITIALIZED   = first_gcflag << 11
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 12    # the first unused bit
 
 
 # States for the incremental GC
@@ -729,6 +733,16 @@
         obj = self.external_malloc(typeid, length, alloc_young=True)
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
+    def move_out_of_nursery(self, obj):
+        # called twice, it should return the same shadow object,
+        # and not creating another shadow object
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW:
+            shadow = self.nursery_objects_shadows.get(obj)
+            ll_assert(shadow != llmemory.NULL,
+                      "GCFLAG_HAS_SHADOW but no shadow found")
+            return shadow
+
+        return self._allocate_shadow(obj, copy=True)
 
     def collect(self, gen=2):
         """Do a minor (gen=0), start a major (gen=1), or do a full
@@ -1982,6 +1996,9 @@
                 and self.young_rawmalloced_objects.contains(obj)):
                 self._visit_young_rawmalloced_object(obj)
             return
+        # copy the contents of the object? usually yes, but not for some
+        # shadow objects
+        copy = True
         #
         size_gc_header = self.gcheaderbuilder.size_gc_header
         if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
@@ -2037,13 +2054,18 @@
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
             # copied to the shadow itself.
             self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+            tid = self.header(obj).tid
+            if (tid & GCFLAG_SHADOW_INITIALIZED) != 0:
+                copy = False
+                self.header(obj).tid &= ~GCFLAG_SHADOW_INITIALIZED
             #
             totalsize = size_gc_header + self.get_size(obj)
             self.nursery_surviving_size += raw_malloc_usage(totalsize)
         #
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.
-        llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
+        if copy:
+            llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
         #
         # Set the old object's tid to -42 (containing all flags) and
         # replace the old object's content with the target address.
@@ -2570,7 +2592,8 @@
     # ----------
     # id() and identityhash() support
 
-    def _allocate_shadow(self, obj):
+    @specialize.arg(2)
+    def _allocate_shadow(self, obj, copy=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         size = self.get_size(obj)
         shadowhdr = self._malloc_out_of_nursery(size_gc_header +
@@ -2592,6 +2615,12 @@
         #
         self.header(obj).tid |= GCFLAG_HAS_SHADOW
         self.nursery_objects_shadows.setitem(obj, shadow)
+
+        if copy:
+            self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED
+            totalsize = size_gc_header + self.get_size(obj)
+            llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize)
+
         return shadow
 
     def _find_shadow(self, obj):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -551,6 +551,13 @@
                                               [s_gc, SomeAddress()],
                                               annmodel.s_None)
 
+        self.move_out_of_nursery_ptr = None
+        if hasattr(GCClass, 'move_out_of_nursery'):
+            self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
+                                              [s_gc, SomeAddress()],
+                                              SomeAddress())
+
+
     def create_custom_trace_funcs(self, gc, rtyper):
         custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
         rtyper.custom_trace_funcs = custom_trace_funcs
@@ -1585,6 +1592,17 @@
             hop.genop("direct_call", [self.ignore_finalizer_ptr,
                                       self.c_const_gc, v_adr])
 
+    def gct_gc_move_out_of_nursery(self, hop):
+        if self.move_out_of_nursery_ptr is not None:
+            v_adr = hop.genop("cast_ptr_to_adr", [hop.spaceop.args[0]],
+                              resulttype=llmemory.Address)
+            v_ret = hop.genop("direct_call", [self.move_out_of_nursery_ptr,
+                                      self.c_const_gc, v_adr],
+                                      resulttype=llmemory.Address)
+            hop.genop("cast_adr_to_ptr", [v_ret],
+                      resultvar = hop.spaceop.result)
+
+
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -2,6 +2,8 @@
 Buffer protocol support.
 """
 from rpython.rlib import jit
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+        nonmoving_raw_ptr_for_resizable_list)
 
 
 class Buffer(object):
@@ -84,7 +86,7 @@
 
     def __init__(self, value):
         self.value = value
-        self.readonly = True
+        self.readonly = 1
 
     def getlength(self):
         return len(self.value)
@@ -108,6 +110,9 @@
             return self.value[start:stop]
         return Buffer.getslice(self, start, stop, step, size)
 
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        return rffi.get_raw_address_of_string(self.value)
 
 class SubBuffer(Buffer):
     _attrs_ = ['buffer', 'offset', 'size', 'readonly']
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -535,6 +535,25 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     llop.gc_ignore_finalizer(lltype.Void, obj)
 
+ at jit.dont_look_inside
+def move_out_of_nursery(obj):
+    """ Returns another object which is a copy of obj; but at any point
+        (either now or in the future) the returned object might suddenly
+        become identical to the one returned.
+
+        NOTE: Only use for immutable objects!
+    """
+    pass
+
+class MoveOutOfNurseryEntry(ExtRegistryEntry):
+    _about_ = move_out_of_nursery
+
+    def compute_result_annotation(self, s_obj):
+        return s_obj
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -997,12 +997,12 @@
                 if signal_checker is not None:
                     signal_checker()
 
-    def sendto(self, data, flags, address):
+    def sendto(self, data, length, flags, address):
         """Like send(data, flags) but allows specifying the destination
         address.  (Note that 'flags' is mandatory here.)"""
         self.wait_for_data(True)
         addr = address.lock()
-        res = _c.sendto(self.fd, data, len(data), flags,
+        res = _c.sendto(self.fd, data, length, flags,
                         addr, address.addrlen)
         address.unlock()
         if res < 0:
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -1,4 +1,4 @@
-from rpython.rlib.buffer import *
+from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer
 from rpython.annotator.annrpython import RPythonAnnotator
 from rpython.annotator.model import SomeInteger
 
@@ -64,3 +64,10 @@
     for i in range(9999, 9, -1):
         buf = SubBuffer(buf, 1, i)
     assert buf.getlength() == 10
+
+def test_string_buffer_as_buffer():
+    buf = StringBuffer(b'hello world')
+    addr = buf.get_raw_address()
+    assert addr[0] == b'h'
+    assert addr[4] == b'o'
+    assert addr[6] == b'w'
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -320,7 +320,7 @@
     s2.bind(INETAddress('127.0.0.1', INADDR_ANY))
     addr2 = s2.getsockname()
 
-    s1.sendto('?', 0, addr2)
+    s1.sendto('?', 1, 0, addr2)
     buf = s2.recv(100)
     assert buf == '?'
     s2.connect(addr)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1138,6 +1138,9 @@
         exc_data.exc_value = lltype.typeOf(evalue)._defl()
         return bool(etype)
 
+    def op_gc_move_out_of_nursery(self, obj):
+        raise NotImplementedError("gc_move_out_of_nursery")
+
 
 class Tracer(object):
     Counter = 0
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -496,6 +496,8 @@
     'gc_rawrefcount_to_obj':            LLOp(sideeffects=False),
     'gc_rawrefcount_next_dead':         LLOp(),
 
+    'gc_move_out_of_nursery':           LLOp(),
+
     # ------- JIT & GC interaction, only for some GCs ----------
 
     'gc_adr_of_nursery_free' : LLOp(),
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -739,6 +739,9 @@
 def op_gc_ignore_finalizer(obj):
     pass
 
+def op_gc_move_out_of_nursery(obj):
+    return obj
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -754,6 +754,7 @@
 SIGNED = lltype.Signed
 SIGNEDP = lltype.Ptr(lltype.Array(SIGNED, hints={'nolength': True}))
 
+
 # various type mapping
 
 # conversions between str and char*
@@ -876,6 +877,7 @@
     get_nonmovingbuffer._always_inline_ = 'try' # get rid of the returned tuple
     get_nonmovingbuffer._annenforceargs_ = [strtype]
 
+
     @jit.dont_look_inside
     def get_nonmovingbuffer_final_null(data):
         tup = get_nonmovingbuffer(data)
@@ -1310,3 +1312,51 @@
             releasegil=False,
             calling_conv='c',
         )
+
+
+if not we_are_translated():
+    class RawBytes(object):
+        # literal copy of _cffi_backend/func.py
+        def __init__(self, string):
+            self.ptr = str2charp(string, track_allocation=False)
+        def __del__(self):
+            free_charp(self.ptr, track_allocation=False)
+
+    TEST_RAW_ADDR_KEEP_ALIVE = {}
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(string):
+    """Returns a 'char *' that is valid as long as the rpython string object is alive.
+    Two calls to to this function, given the same string parameter,
+    are guaranteed to return the same pointer.
+
+    The extra parameter key is necessary to create a weak reference.
+    The buffer of the returned pointer (if object is young) lives as long
+    as key is alive. If key goes out of scope, the buffer will eventually
+    be freed. `string` cannot go out of scope until the RawBytes object
+    referencing it goes out of scope.
+    """
+    assert isinstance(string, str)
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rtyper.lltypesystem.rstr import STR
+    from rpython.rtyper.lltypesystem import llmemory
+    from rpython.rlib import rgc
+
+    if we_are_translated():
+        if rgc.can_move(string):
+            string = rgc.move_out_of_nursery(string)
+
+        # string cannot move now! return the address
+        lldata = llstr(string)
+        data_start = (llmemory.cast_ptr_to_adr(lldata) +
+                      offsetof(STR, 'chars') +
+                      llmemory.itemoffsetof(STR.chars, 0))
+        data_start = cast(CCHARP, data_start)
+        data_start[len(string)] = '\x00'   # write the final extra null
+        return data_start
+    else:
+        global TEST_RAW_ADDR_KEEP_ALIVE
+        if string in TEST_RAW_ADDR_KEEP_ALIVE:
+            return TEST_RAW_ADDR_KEEP_ALIVE[string].ptr
+        TEST_RAW_ADDR_KEEP_ALIVE[string] = rb = RawBytes(string)
+        return rb.ptr
diff --git a/rpython/rtyper/lltypesystem/test/test_ztranslated.py b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
@@ -0,0 +1,59 @@
+import gc
+from rpython.translator.c.test.test_genc import compile
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rlib import rgc
+
+def debug_assert(boolresult, msg):
+    if not boolresult:
+        llop.debug_print(lltype.Void, "\n\nassert failed: %s\n\n" % msg)
+        assert boolresult
+
+def use_str():
+    mystr = b'abc'
+    #debug_assert(rgc.can_move(mystr), "short string cannot move... why?")
+    ptr = rffi.get_raw_address_of_string(mystr)
+    ptr2 = rffi.get_raw_address_of_string(mystr)
+    debug_assert(ptr == ptr2, "ptr != ptr2")
+    debug_assert(ptr[0] == b'a', "notnurseryadr[0] == b'a' is is %s" % ptr[0])
+    ptr[0] = b'x' # oh no no, in real programs nobody is allowed to modify that
+    debug_assert(mystr[0] == b'a', "mystr[0] != b'a'")
+    debug_assert(ptr[0] == b'x', "notnurseryadr[0] == b'x'")
+    gc.collect()
+    nptr = rffi.get_raw_address_of_string(mystr)
+    debug_assert(nptr == ptr, "second call to mystr must return the same ptr")
+    debug_assert(ptr[0] == b'x', "failure a")
+    debug_assert(nptr[0] == b'x', "failure b")
+    mystr = None
+
+def long_str(lstr):
+    ptr = rffi.get_raw_address_of_string(lstr)
+    for i,c in enumerate(lstr):
+        debug_assert(ptr[i] == c, "failure c")
+    gc.collect()
+    ptr2 = rffi.get_raw_address_of_string(lstr)
+    debug_assert(ptr == ptr2, "ptr != ptr2!!!")
+    return ptr
+
+def main(argv=[]):
+    use_str()
+    gc.collect()
+    mystr = b"12341234aa"*4096*10
+    #debug_assert(not rgc.can_move(mystr), "long string can move... why?")
+    p1 = long_str(mystr)
+    gc.collect()
+    copystr = mystr[:]
+    copystr += 'a'
+    p2 = long_str(copystr)
+    debug_assert(p1 != p2, "p1 == p2")
+    return 0
+
+# ____________________________________________________________
+
+def target(driver, args):
+    return main
+
+def test_compiled_incminimark():
+    fn = compile(main, [], gcpolicy="incminimark")
+    fn()

From pypy.commits at gmail.com  Wed Jan  4 05:05:57 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 04 Jan 2017 02:05:57 -0800 (PST)
Subject: [pypy-commit] pypy default: document strbuf-as-buffer branch
Message-ID: <586cc905.d32f1c0a.9e8ad.3dcd@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r89357:30d4a493e334
Date: 2017-01-04 11:03 +0100
http://bitbucket.org/pypy/pypy/changeset/30d4a493e334/

Log:	document strbuf-as-buffer branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -81,3 +81,9 @@
 
 Support translations of cpyext with the Boehm GC (for special cases like
 revdb).
+
+.. branch: strbuf-as-buffer
+
+Implement StringBuffer.get_raw_address (missing feature for the buffer protocol).
+More generally it is now possible to obtain the address of any object (if it
+is readonly) without pinning it.

From pypy.commits at gmail.com  Wed Jan  4 05:06:09 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 04 Jan 2017 02:06:09 -0800 (PST)
Subject: [pypy-commit] pypy strbuf-as-buffer: close branch
Message-ID: <586cc911.0209c20a.3e8d9.0f8e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: strbuf-as-buffer
Changeset: r89358:3b422e4d09f4
Date: 2017-01-04 11:05 +0100
http://bitbucket.org/pypy/pypy/changeset/3b422e4d09f4/

Log:	close branch


From pypy.commits at gmail.com  Wed Jan  4 05:22:06 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 04 Jan 2017 02:22:06 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-time: several translation issues resolved
Message-ID: <586cccce.05371c0a.31ce.7604@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-time
Changeset: r89359:0cc981ee46c7
Date: 2017-01-04 11:12 +0100
http://bitbucket.org/pypy/pypy/changeset/0cc981ee46c7/

Log:	several translation issues resolved

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -237,7 +237,6 @@
                  (HAS_CLOCK_GETTIME and (HAS_CLOCK_HIGHRES or HAS_CLOCK_MONOTONIC)))
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
-glob_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
 
 if _WIN:
     _GetSystemTimeAsFileTime = rwin32.winexternal('GetSystemTimeAsFileTime',
@@ -549,11 +548,14 @@
 
     if HAS_TM_ZONE:
         # CPython calls PyUnicode_DecodeLocale here should we do the same?
-        tm_zone = decode_utf8(space, rffi.charp2str(t.c_tm_zone), allow_surrogates=True)
-        time_tuple.append(space.newunicode(tm_zone))
-        time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_gmtoff')))
+        tm_zone = decode_utf8(space, rffi.charp2str(t.c_tm_zone),
+                              allow_surrogates=True)
+        extra = [space.newunicode(tm_zone),
+                 space.wrap(rffi.getintfield(t, 'c_tm_gmtoff'))]
+        w_time_tuple = space.newtuple(time_tuple + extra)
+    else:
+        w_time_tuple = space.newtuple(time_tuple)
     w_struct_time = _get_module_object(space, 'struct_time')
-    w_time_tuple = space.newtuple(time_tuple)
     w_obj = space.call_function(w_struct_time, w_time_tuple)
     return w_obj
 
@@ -599,19 +601,19 @@
     rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
     rffi.setintfield(glob_buf, 'c_tm_isdst', space.c_int_w(tup_w[8]))
     #
+    old_tm_zone = glob_buf.c_tm_zone
     glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
     rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
     if HAS_TM_ZONE :
         if len(tup_w) >= 10:
-            # NOTE this is not cleanly solved, the global variable glob_tm_zone
-            # saves the string that is later deleted when this function is called again
-            # an refactoring of this module could remove this
-            global glob_tm_zone
+            # NOTE this is not cleanly solved!
+            # it saves the string that is later deleted when this
+            # function is called again. A refactoring of this module
+            # could remove this
             tm_zone = encode_utf8(space, space.unicode_w(tup_w[9]), allow_surrogates=True)
             malloced_str = rffi.str2charp(tm_zone, track_allocation=False)
-            if glob_tm_zone != lltype.nullptr(rffi.CCHARP.TO):
-                rffi.freecharp(glob_tm_zone, track_allocation=False)
-            glob_tm_zone = malloced_str
+            if old_tm_zone != lltype.nullptr(rffi.CCHARP.TO):
+                rffi.free_charp(old_tm_zone, track_allocation=False)
             glob_buf.c_tm_zone = malloced_str
         if len(tup_w) >= 11:
             rffi.setintfield(glob_buf, 'c_tm_gmtoff', space.c_int_w(tup_w[10]))

From pypy.commits at gmail.com  Wed Jan  4 08:48:46 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 04 Jan 2017 05:48:46 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Split StaticObjectBuilder into 2
 subclasses: one for translation and one for tests
Message-ID: <586cfd3e.d5091c0a.6942f.be26@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89360:66f4e5e9c8f3
Date: 2017-01-04 13:48 +0000
http://bitbucket.org/pypy/pypy/changeset/66f4e5e9c8f3/

Log:	Split StaticObjectBuilder into 2 subclasses: one for translation and
	one for tests

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1101,7 +1101,7 @@
     space.fromcache(State).install_dll(eci)
 
     # populate static data
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder = TestingObjBuilder()
     for name, (typ, expr) in GLOBALS.iteritems():
         from pypy.module import cpyext    # for the eval() below
         w_obj = eval(expr)
@@ -1138,7 +1138,7 @@
             builder.prepare(py_obj, w_obj)
         else:
             assert False, "Unknown static object: %s %s" % (typ, name)
-    builder.attach_all()
+    builder.attach_all(space)
 
     pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
 
@@ -1156,9 +1156,8 @@
     return modulename.new(ext='')
 
 
-class StaticObjectBuilder:
-    def __init__(self, space):
-        self.space = space
+class StaticObjectBuilder(object):
+    def __init__(self):
         self.static_pyobjs = []
         self.static_objs_w = []
         self.cpyext_type_init = None
@@ -1174,13 +1173,12 @@
         self.static_pyobjs.append(py_obj)
         self.static_objs_w.append(w_obj)
 
-    def attach_all(self):
+    def attach_all(self, space):
         # this is RPython, called once in pypy-c when it imports cpyext
         from pypy.module.cpyext.pyobject import get_typedescr, make_ref
         from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
         from pypy.module.cpyext.pyobject import track_reference
         #
-        space = self.space
         static_pyobjs = self.get_static_pyobjs()
         static_objs_w = self.static_objs_w
         for i in range(len(static_objs_w)):
@@ -1201,6 +1199,12 @@
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
+class TestingObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used in tests."""
+
+class TranslationObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used during translation."""
+
 
 def mangle_name(prefix, name):
     if name.startswith('Py'):
@@ -1416,7 +1420,7 @@
     setup_va_functions(eci)
 
     # emit uninitialized static data
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder = TranslationObjBuilder()
     lines = ['PyObject *pypy_static_pyobjs[] = {\n']
     include_lines = ['RPY_EXTERN PyObject *pypy_static_pyobjs[];\n']
     for name, (typ, expr) in sorted(GLOBALS.items()):
@@ -1463,9 +1467,6 @@
     trunk_include = pypydir.dirpath() / 'include'
     copy_header_files(trunk_include, use_micronumpy)
 
-def init_static_data_translated(space):
-    builder = space.fromcache(StaticObjectBuilder)
-    builder.attach_all()
 
 def _load_from_cffi(space, name, path, initptr):
     from pypy.module._cffi_backend import cffi1_module
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -14,6 +14,7 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
+        self.builder = None
         if space.config.translation.gc != "boehm":
             pyobj_dealloc_action = PyObjDeallocAction(space)
             self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
@@ -84,17 +85,15 @@
 
     def startup(self, space):
         "This function is called when the program really starts"
-
         from pypy.module.cpyext.typeobject import setup_new_method_def
         from pypy.module.cpyext.api import INIT_FUNCTIONS
-        from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
             if space.config.translation.gc != "boehm":
                 rawrefcount.init(
                     llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
                     self.dealloc_trigger))
-            init_static_data_translated(space)
+            self.builder.attach_all(space)
 
         setup_new_method_def(space)
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -87,7 +87,7 @@
     py_getsetdef.c_set = rffi.cast(setter, 0)
     py_getsetdef.c_closure = rffi.cast(rffi.VOIDP, 0)
     return py_getsetdef
-    
+
 
 class W_MemberDescr(GetSetProperty):
     name = 'member_descriptor'
@@ -711,7 +711,7 @@
 
     pto.c_tp_free = llslot(space, PyObject_Free)
     pto.c_tp_alloc = llslot(space, PyType_GenericAlloc)
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder
     if ((pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE) != 0
             and builder.cpyext_type_init is None):
             # this ^^^ is not None only during startup of cpyext.  At that

From pypy.commits at gmail.com  Wed Jan  4 09:26:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 04 Jan 2017 06:26:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip warn() in the FakeObjSpace
Message-ID: <586d05f9.8c1f1c0a.f8a7f.ea70@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89361:8b953c5f7f48
Date: 2017-01-04 15:25 +0100
http://bitbucket.org/pypy/pypy/changeset/8b953c5f7f48/

Log:	skip warn() in the FakeObjSpace

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -358,6 +358,9 @@
     def lookup_in_type(self, w_type, name):
         return w_some_obj()
 
+    def warn(self, w_msg, w_warningcls, stacklevel=2):
+        pass
+
     # ----------
 
     def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):

From pypy.commits at gmail.com  Wed Jan  4 09:49:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 04 Jan 2017 06:49:54 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove outdated info
Message-ID: <586d0b92.52301c0a.a3729.e7ad@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89362:1c1e0c3b2f27
Date: 2017-01-04 15:49 +0100
http://bitbucket.org/pypy/pypy/changeset/1c1e0c3b2f27/

Log:	Remove outdated info

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -428,11 +428,6 @@
   ``datetime.date`` is the superclass of ``datetime.datetime``).
   Anyway, the proper fix is arguably to use a regular method call in
   the first place: ``datetime.date.today().strftime(...)``
-
-* the ``__dict__`` attribute of new-style classes returns a normal dict, as
-  opposed to a dict proxy like in CPython. Mutating the dict will change the
-  type and vice versa. For builtin types, a dictionary will be returned that
-  cannot be changed (but still looks and behaves like a normal dictionary).
   
 * some functions and attributes of the ``gc`` module behave in a
   slightly different way: for example, ``gc.enable`` and

From pypy.commits at gmail.com  Wed Jan  4 09:53:26 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 04 Jan 2017 06:53:26 -0800 (PST)
Subject: [pypy-commit] cffi default: merge strbuf-as-buffer
Message-ID: <586d0c66.8838c20a.ed9d4.125c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r2854:9af655eac351
Date: 2017-01-04 15:52 +0100
http://bitbucket.org/cffi/cffi/changeset/9af655eac351/

Log:	merge strbuf-as-buffer

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6134,54 +6134,18 @@
     return 0;
 }
 
-static int invalid_input_buffer_type(PyObject *x)
-{
-    /* From PyPy 5.4, from_buffer() accepts strings (but still not buffers
-       or memoryviews on strings). */
-    if (PyBytes_Check(x))
-        return 0;
-
-#if PY_MAJOR_VERSION < 3
-    if (PyBuffer_Check(x)) {
-        /* XXX fish fish fish in an inofficial way */
-        typedef struct {
-            PyObject_HEAD
-            PyObject *b_base;
-        } _my_PyBufferObject;
-
-        _my_PyBufferObject *b = (_my_PyBufferObject *)x;
-        x = b->b_base;
-        if (x == NULL)
-            return 0;
-    }
-    else
-#endif
-#if PY_MAJOR_VERSION > 2 || PY_MINOR_VERSION > 6
-    if (PyMemoryView_Check(x)) {
-        x = PyMemoryView_GET_BASE(x);
-        if (x == NULL)
-            return 0;
-    }
-    else
-#endif
-        ;
-
-    if (PyBytes_Check(x) || PyUnicode_Check(x))
-        return 1;
-    /* From PyPy 5.2, bytearray buffers can fetch a raw pointer, so
-       there is no reason any more to prevent from_buffer(bytearray()). */
-    return 0;
-}
-
 static PyObject *direct_from_buffer(CTypeDescrObject *ct, PyObject *x)
 {
     CDataObject *cd;
     Py_buffer *view;
 
-    if (invalid_input_buffer_type(x)) {
+    /* PyPy 5.7 can obtain buffers for string (python 2)
+       or bytes (python 3). from_buffer(u"foo") is disallowed.
+     */
+    if (PyUnicode_Check(x)) {
         PyErr_SetString(PyExc_TypeError,
-                        "from_buffer() cannot return the address of the "
-                        "raw string within a "STR_OR_BYTES" or unicode object");
+                        "from_buffer() cannot return the address "
+                        "of a unicode object");
         return NULL;
     }
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3438,16 +3438,26 @@
     except ImportError:
         pass
     else:
-        # from_buffer(buffer(b"foo")) does not work, because it's not
-        # implemented on pypy; only from_buffer(b"foo") works.
-        py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
-        py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
+        # Python 2 only
+        contents = from_buffer(BCharA, buffer(b"foo"))
+        assert len(contents) == len(p1)
+        for i in range(len(contents)):
+            assert contents[i] == p1[i]
+        p4 = buffer(u+"foo")
+        contents = from_buffer(BCharA, buffer(u+"foo"))
+        assert len(contents) == len(p4)
+        for i in range(len(contents)):
+            assert contents[i] == p4[i]
     try:
         from __builtin__ import memoryview
     except ImportError:
         pass
     else:
-        py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo"))
+        contents = from_buffer(BCharA, memoryview(b"foo"))
+        assert len(contents) == len(p1)
+        for i in range(len(contents)):
+            assert contents[i] == p1[i]
+
 
 def test_from_buffer_bytearray():
     a = bytearray(b"xyz")
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -175,9 +175,8 @@
 **ffi.from_buffer(python_buffer)**: return a ``<cdata 'char[]'>`` that
 points to the data of the given Python object, which must support the
 buffer interface.  This is the opposite of ``ffi.buffer()``.  It gives
-a reference to the existing data, not a copy; for this
-reason, and for PyPy compatibility, it does not work with the built-in
-type unicode; nor buffers/memoryviews to byte or unicode strings.
+a reference to the existing data, not a copy; It does not work with the built-in
+type unicode.
 It is meant to be used on objects
 containing large quantities of raw data, like bytearrays
 or ``array.array`` or numpy
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -23,6 +23,8 @@
 * Add support for some obscure compilers (non-msvc, non-gcc, non-icc,
   non-clang)
 
+* ``ffi.from_buffer`` allows Python 2 strings and Python 3 bytes to be
+  passed. Unicode is the only type disallowed.
 
 v1.9
 ====

From pypy.commits at gmail.com  Wed Jan  4 09:53:53 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 04 Jan 2017 06:53:53 -0800 (PST)
Subject: [pypy-commit] cffi strbuf-as-buffer: close branch
Message-ID: <586d0c81.e576c20a.37a92.4f3b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: strbuf-as-buffer
Changeset: r2855:379fc340f76f
Date: 2017-01-04 15:53 +0100
http://bitbucket.org/cffi/cffi/changeset/379fc340f76f/

Log:	close branch


From pypy.commits at gmail.com  Wed Jan  4 09:59:03 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 04 Jan 2017 06:59:03 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Reduce differences between
 build_bridge() and setup_library()
Message-ID: <586d0db7.ce941c0a.d731b.e761@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89363:ca30c209f6c1
Date: 2017-01-04 14:58 +0000
http://bitbucket.org/pypy/pypy/changeset/ca30c209f6c1/

Log:	Reduce differences between build_bridge() and setup_library()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1078,6 +1078,7 @@
     eci = build_eci(True, code, use_micronumpy)
     eci = eci.compile_shared_lib(
         outputfilename=str(udir / "module_cache" / "pypyapi"))
+    space.fromcache(State).install_dll(eci)
     modulename = py.path.local(eci.libraries[-1])
 
     def dealloc_trigger():
@@ -1098,46 +1099,42 @@
     # load the bridge, and init structure
     bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
 
-    space.fromcache(State).install_dll(eci)
 
     # populate static data
     builder = space.fromcache(State).builder = TestingObjBuilder()
     for name, (typ, expr) in GLOBALS.iteritems():
+        if '#' in name:
+            name, header = name.split('#')
+            assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
+            isptr = False
+        elif name.startswith('PyExc_'):
+            isptr = False
+        elif typ == 'PyDateTime_CAPI*':
+            isptr = True
+        else:
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
+
         from pypy.module import cpyext    # for the eval() below
         w_obj = eval(expr)
-        if '#' in name:
-            name = name.split('#')[0]
-            isptr = False
-        else:
-            isptr = True
-        if name.startswith('PyExc_'):
-            isptr = False
-
         INTERPLEVEL_API[name] = w_obj
 
-        name = name.replace('Py', prefix)
+        mname = mangle_name(prefix, name)
         if isptr:
-            ptr = ctypes.c_void_p.in_dll(bridge, name)
-            if typ == 'PyObject*':
-                value = make_ref(space, w_obj)
-            elif typ == 'PyDateTime_CAPI*':
-                value = w_obj
-            else:
-                assert False, "Unknown static pointer: %s %s" % (typ, name)
+            assert typ == 'PyDateTime_CAPI*'
+            value = w_obj
+            ptr = ctypes.c_void_p.in_dll(bridge, mname)
             ptr.value = ctypes.cast(ll2ctypes.lltype2ctypes(value),
                                     ctypes.c_void_p).value
-        elif typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*'):
-            if name.startswith('PyPyExc_') or name.startswith('cpyexttestExc_'):
+        else:
+            if name.startswith('PyExc_'):
                 # we already have the pointer
-                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, in_dll)
             else:
                 # we have a structure, get its address
-                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, ctypes.pointer(in_dll))
             builder.prepare(py_obj, w_obj)
-        else:
-            assert False, "Unknown static object: %s %s" % (typ, name)
     builder.attach_all(space)
 
     pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
@@ -1413,7 +1410,6 @@
     code  += "\n".join(functions)
 
     eci = build_eci(False, code, use_micronumpy)
-
     space.fromcache(State).install_dll(eci)
 
     run_bootstrap_functions(space)
@@ -1436,7 +1432,7 @@
         elif typ == 'PyDateTime_CAPI*':
             continue
         else:
-            assert False, "Unknown static data: %s %s" % (typ, name)
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
 
         from pypy.module import cpyext     # for the eval() below
         w_obj = eval(expr)

From pypy.commits at gmail.com  Wed Jan  4 10:30:39 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 04 Jan 2017 07:30:39 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Consistently use 'translating'
 flag for special-casing in functions called from
 setup_library()/build_bridge()
Message-ID: <586d151f.c4251c0a.6f1d.1b6a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89364:d305e8c57ac7
Date: 2017-01-04 15:30 +0000
http://bitbucket.org/pypy/pypy/changeset/d305e8c57ac7/

Log:	Consistently use 'translating' flag for special-casing in functions
	called from setup_library()/build_bridge()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -963,11 +963,7 @@
                                TP, compilation_info=eci)
         globals()['va_get_%s' % name_no_star] = func
 
-def setup_init_functions(eci, translating):
-    if translating:
-        prefix = 'PyPy'
-    else:
-        prefix = 'cpyexttest'
+def setup_init_functions(eci, prefix):
     # jump through hoops to avoid releasing the GIL during initialization
     # of the cpyext module.  The C functions are called with no wrapper,
     # but must not do anything like calling back PyType_Ready().  We
@@ -1029,13 +1025,13 @@
 # Do not call this more than once per process
 def build_bridge(space):
     "NOT_RPYTHON"
-    from pypy.module.cpyext.pyobject import make_ref
     from rpython.translator.c.database import LowLevelDatabase
     use_micronumpy = setup_micronumpy(space)
     db = LowLevelDatabase()
-    prefix ='cpyexttest'
+    prefix = 'cpyexttest'
 
-    functions = generate_decls_and_callbacks(db, prefix=prefix)
+    functions = generate_decls_and_callbacks(
+            db, prefix=prefix, translating=False)
 
     # Structure declaration code
     members = []
@@ -1075,7 +1071,7 @@
             '\n' +
             '\n'.join(functions))
 
-    eci = build_eci(True, code, use_micronumpy)
+    eci = build_eci(code, use_micronumpy, translating=False)
     eci = eci.compile_shared_lib(
         outputfilename=str(udir / "module_cache" / "pypyapi"))
     space.fromcache(State).install_dll(eci)
@@ -1149,7 +1145,7 @@
                 ctypes.c_void_p)
     setup_va_functions(eci)
 
-    setup_init_functions(eci, translating=False)
+    setup_init_functions(eci, prefix)
     return modulename.new(ext='')
 
 
@@ -1211,7 +1207,7 @@
     else:
         return None
 
-def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
+def generate_decls_and_callbacks(db, prefix='', translating=False):
     "NOT_RPYTHON"
     pypy_macros = []
     for name in SYMBOLS_C:
@@ -1263,7 +1259,7 @@
             header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
             header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, _name, args))
-            if api_struct:
+            if not translating:
                 callargs = ', '.join('arg%d' % (i,)
                                     for i in range(len(func.argtypes)))
                 if func.restype is lltype.Void:
@@ -1271,6 +1267,7 @@
                 else:
                     body = "{ return _pypyAPI.%s(%s); }" % (_name, callargs)
                 functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
+
     for name in VA_TP_LIST:
         name_no_star = process_va_name(name)
         header = ('%s pypy_va_get_%s(va_list* vp)' %
@@ -1318,14 +1315,16 @@
                          source_dir / "pymem.c",
                          ]
 
-def build_eci(building_bridge, code, use_micronumpy=False):
+def build_eci(code, use_micronumpy=False, translating=False):
     "NOT_RPYTHON"
     # Build code and get pointer to the structure
     kwds = {}
 
     compile_extra=['-DPy_BUILD_CORE']
 
-    if building_bridge:
+    if translating:
+        kwds["includes"] = ['Python.h'] # this is our Python.h
+    else:
         if sys.platform == "win32":
             # '%s' undefined; assuming extern returning int
             compile_extra.append("/we4013")
@@ -1335,8 +1334,6 @@
         elif sys.platform.startswith('linux'):
             compile_extra.append("-Werror=implicit-function-declaration")
             compile_extra.append('-g')
-    else:
-        kwds["includes"] = ['Python.h'] # this is our Python.h
 
     # Generate definitions for global structures
     structs = ["#include <Python.h>"]
@@ -1403,13 +1400,15 @@
     db = LowLevelDatabase()
     prefix = 'PyPy'
 
-    functions = generate_decls_and_callbacks(db, api_struct=False, prefix=prefix)
+    functions = generate_decls_and_callbacks(
+            db, prefix=prefix, translating=True)
+
     code = "#include <Python.h>\n"
     if use_micronumpy:
         code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
     code  += "\n".join(functions)
 
-    eci = build_eci(False, code, use_micronumpy)
+    eci = build_eci(code, use_micronumpy, translating=True)
     space.fromcache(State).install_dll(eci)
 
     run_bootstrap_functions(space)
@@ -1459,7 +1458,7 @@
                                         relax=True)
             deco(func.get_wrapper(space))
 
-    setup_init_functions(eci, translating=True)
+    setup_init_functions(eci, prefix)
     trunk_include = pypydir.dirpath() / 'include'
     copy_header_files(trunk_include, use_micronumpy)
 

From pypy.commits at gmail.com  Wed Jan  4 10:53:29 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 04 Jan 2017 07:53:29 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Use unmangled names in generated
 code
Message-ID: <586d1a79.2919c20a.cae20.efb2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89365:4bb393bf3f37
Date: 2017-01-04 15:52 +0000
http://bitbucket.org/pypy/pypy/changeset/4bb393bf3f37/

Log:	Use unmangled names in generated code

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1258,14 +1258,14 @@
             assert _name is not None, 'error converting %s' % name
             header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, _name, args))
+            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
             if not translating:
                 callargs = ', '.join('arg%d' % (i,)
                                     for i in range(len(func.argtypes)))
                 if func.restype is lltype.Void:
-                    body = "{ _pypyAPI.%s(%s); }" % (_name, callargs)
+                    body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
                 else:
-                    body = "{ return _pypyAPI.%s(%s); }" % (_name, callargs)
+                    body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
                 functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
 
     for name in VA_TP_LIST:

From pypy.commits at gmail.com  Wed Jan  4 12:44:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 04 Jan 2017 09:44:35 -0800 (PST)
Subject: [pypy-commit] cffi default: Tweaks the docs
Message-ID: <586d3483.ce841c0a.66e63.5214@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2856:3234afed406d
Date: 2017-01-04 18:44 +0100
http://bitbucket.org/cffi/cffi/changeset/3234afed406d/

Log:	Tweaks the docs

diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -175,15 +175,16 @@
 **ffi.from_buffer(python_buffer)**: return a ``<cdata 'char[]'>`` that
 points to the data of the given Python object, which must support the
 buffer interface.  This is the opposite of ``ffi.buffer()``.  It gives
-a reference to the existing data, not a copy; It does not work with the built-in
-type unicode.
+a reference to the existing data, not a copy.
 It is meant to be used on objects
 containing large quantities of raw data, like bytearrays
 or ``array.array`` or numpy
-arrays.  It supports both the old buffer API (in Python 2.x) and the
-new memoryview API.  Note that if you pass a read-only buffer object,
+arrays.  It supports both the old *buffer* API (in Python 2.x) and the
+new *memoryview* API.  Note that if you pass a read-only buffer object,
 you still get a regular ``<cdata 'char[]'>``; it is your responsibility
 not to write there if the original buffer doesn't expect you to.
+*In particular, never modify byte strings!*
+
 The original object is kept alive (and, in case
 of memoryview, locked) as long as the cdata object returned by
 ``ffi.from_buffer()`` is alive.
@@ -193,13 +194,11 @@
 can directly pass ``ffi.from_buffer(python_buffer)`` as argument to
 the call.
 
-*New in version 1.7:* the python_buffer can be a bytearray object.
-Be careful: if the bytearray gets resized (e.g. its ``.append()``
-method is called), then the ``<cdata>`` object will point to freed
-memory and must not be used any more.
-
-*New in version 1.8:* the python_buffer can be a byte string (but still
-not a buffer/memoryview on a string).  Never modify a byte string!
+*New in version 1.10:* the ``python_buffer`` can be anything supporting
+the buffer/memoryview interface (except unicode strings).  Previously,
+bytearray objects were supported in version 1.7 onwards (careful, if you
+resize the bytearray, the ``<cdata>`` object will point to freed
+memory); and byte strings were supported in version 1.8 onwards.
 
 
 ffi.memmove()
@@ -211,8 +210,7 @@
 areas can overlap.  Each of ``dest`` and ``src`` can be either a cdata
 pointer or a Python object supporting the buffer/memoryview interface.
 In the case of ``dest``, the buffer/memoryview must be writable.
-Unlike ``ffi.from_buffer()``, there are no restrictions on the type of
-buffer.  *New in version 1.3.*  Examples:
+*New in version 1.3.*  Examples:
 
 * ``ffi.memmove(myptr, b"hello", 5)`` copies the 5 bytes of
   ``b"hello"`` to the area that ``myptr`` points to.
@@ -223,6 +221,8 @@
 * ``ffi.memmove(myptr + 1, myptr, 100)`` shifts 100 bytes from
   the memory at ``myptr`` to the memory at ``myptr + 1``.
 
+In versions before 1.10, ``ffi.from_buffer()`` had restrictions on the
+type of buffer, which made ``ffi.memmove()`` more general.
 
 .. _ffi-typeof:
 .. _ffi-sizeof:
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,8 +3,8 @@
 ======================
 
 
-v1.9.2
-======
+v1.10
+=====
 
 * Issue #295: use calloc() directly instead of
   PyObject_Malloc()+memset() to handle ffi.new() with a default
@@ -23,8 +23,13 @@
 * Add support for some obscure compilers (non-msvc, non-gcc, non-icc,
   non-clang)
 
-* ``ffi.from_buffer`` allows Python 2 strings and Python 3 bytes to be
-  passed. Unicode is the only type disallowed.
+* Implemented the remaining cases for ``ffi.from_buffer``.  Now all
+  buffer/memoryview objects can be passed.  The one remaining check is
+  against passing unicode strings in Python 2.  (They support the buffer
+  interface, but that gives the raw bytes behind the UTF16/UCS4 storage,
+  which is most of the times not what you expect.  In Python 3 this has
+  been fixed and the unicode strings don't support the memoryview
+  interface any more.)
 
 v1.9
 ====

From pypy.commits at gmail.com  Wed Jan  4 12:52:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 04 Jan 2017 09:52:26 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <586d365a.6737c20a.6877a.c39c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r843:fbfa3842fe51
Date: 2017-01-04 18:52 +0100
http://bitbucket.org/pypy/pypy.org/changeset/fbfa3842fe51/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 63.3
+      value: 63.4
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66502 of $105000 (63.3%)
+   $66521 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2339 left
+    we have $2357 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Wed Jan  4 13:58:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 04 Jan 2017 10:58:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: _thread.Lock.__repr__;
 _thread.RLock.__repr__
Message-ID: <586d45c9.4c9d1c0a.74f50.7ee8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89366:5a4234a11c85
Date: 2017-01-04 19:57 +0100
http://bitbucket.org/pypy/pypy/changeset/5a4234a11c85/

Log:	_thread.Lock.__repr__; _thread.RLock.__repr__

diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -94,13 +94,16 @@
             raise oefmt(space.w_RuntimeError,
                         "cannot release un-acquired lock")
 
+    def _is_locked(self):
+        if self.lock.acquire(False):
+            self.lock.release()
+            return False
+        else:
+            return True
+
     def descr_lock_locked(self, space):
         """Return whether the lock is in the locked state."""
-        if self.lock.acquire(False):
-            self.lock.release()
-            return space.w_False
-        else:
-            return space.w_True
+        return space.newbool(self._is_locked())
 
     def descr__enter__(self, space):
         self.descr_lock_acquire(space)
@@ -116,6 +119,14 @@
     def __exit__(self, *args):
         self.descr_lock_release(self.space)
 
+    def descr__repr__(self, space):
+        classname = space.getfulltypename(self)
+        if self._is_locked():
+            locked = u"locked"
+        else:
+            locked = u"unlocked"
+        return self.getrepr(space, u'%s %s object' % (locked, classname))
+
 Lock.typedef = TypeDef(
     "_thread.lock",
     __doc__="""\
@@ -134,6 +145,7 @@
     locked=interp2app(Lock.descr_lock_locked),
     __enter__=interp2app(Lock.descr__enter__),
     __exit__=interp2app(Lock.descr__exit__),
+    __repr__ = interp2app(Lock.descr__repr__),
     # Obsolete synonyms
     acquire_lock=interp2app(Lock.descr_lock_acquire),
     release_lock=interp2app(Lock.descr_lock_release),
@@ -174,10 +186,14 @@
         W_RLock.__init__(self, space)
         return space.wrap(self)
 
-    def descr__repr__(self):
-        typename = space.type(self).getname(space)
-        return space.wrap(u"<%s owner=%d count=%d>" % (
-                typename, self.rlock_owner, self.rlock_count))
+    def descr__repr__(self, space):
+        classname = space.getfulltypename(self)
+        if self.rlock_count == 0:
+            locked = u"unlocked"
+        else:
+            locked = u"locked"
+        return self.getrepr(space, u'%s %s object owner=%d count=%d' % (
+            locked, classname, self.rlock_owner, self.rlock_count))
 
     @unwrap_spec(blocking=int, timeout=float)
     def acquire_w(self, space, blocking=True, timeout=-1.0):
@@ -285,4 +301,5 @@
     __enter__ = interp2app(W_RLock.descr__enter__),
     __exit__ = interp2app(W_RLock.descr__exit__),
     __weakref__ = make_weakref_descr(W_RLock),
+    __repr__ = interp2app(W_RLock.descr__repr__),
     )
diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py
--- a/pypy/module/thread/test/test_lock.py
+++ b/pypy/module/thread/test/test_lock.py
@@ -302,3 +302,20 @@
         finally:
             signal.signal(signal.SIGALRM, oldalrm)
 
+    def test_lock_repr(self):
+        import _thread
+        lock = _thread.allocate_lock()
+        assert repr(lock).startswith("<unlocked _thread.lock object at ")
+        lock.acquire()
+        assert repr(lock).startswith("<locked _thread.lock object at ")
+
+    def test_rlock_repr(self):
+        import _thread
+        rlock = _thread.RLock()
+        assert repr(rlock).startswith(
+            "<unlocked _thread.RLock object owner=0 count=0 at ")
+        rlock.acquire()
+        rlock.acquire()
+        assert repr(rlock).startswith("<locked _thread.RLock object owner=")
+        assert 'owner=0' not in repr(rlock)
+        assert " count=2 at " in repr(rlock)
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -170,18 +170,7 @@
 
 
 def descr__repr__(space, w_obj):
-    w_type = space.type(w_obj)
-    classname = w_type.name.decode('utf-8')
-    if w_type.is_heaptype():
-        w_module = w_type.lookup("__module__")
-        if w_module is not None:
-            try:
-                modulename = space.unicode_w(w_module)
-            except OperationError as e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-            else:
-                classname = u'%s.%s' % (modulename, classname)
+    classname = space.getfulltypename(w_obj)
     return w_obj.getrepr(space, u'%s object' % (classname,))
 
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -709,3 +709,18 @@
     def is_overloaded(self, w_obj, tp, method):
         return (self.lookup(w_obj, method) is not
                 self.lookup_in_type(tp, method))
+
+    def getfulltypename(self, w_obj):
+        w_type = self.type(w_obj)
+        classname = w_type.name.decode('utf-8')
+        if w_type.is_heaptype():
+            w_module = w_type.lookup("__module__")
+            if w_module is not None:
+                try:
+                    modulename = self.unicode_w(w_module)
+                except OperationError as e:
+                    if not e.match(self, self.w_TypeError):
+                        raise
+                else:
+                    classname = u'%s.%s' % (modulename, classname)
+        return classname

From pypy.commits at gmail.com  Wed Jan  4 19:08:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 04 Jan 2017 16:08:51 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Move generation of ctypes-based
 function implementation shims to build_bridge()
Message-ID: <586d8e93.c64bc20a.84589.5d07@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89367:b24ed9dfc12b
Date: 2017-01-05 00:00 +0000
http://bitbucket.org/pypy/pypy/changeset/b24ed9dfc12b/

Log:	Move generation of ctypes-based function implementation shims to
	build_bridge()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1030,8 +1030,7 @@
     db = LowLevelDatabase()
     prefix = 'cpyexttest'
 
-    functions = generate_decls_and_callbacks(
-            db, prefix=prefix, translating=False)
+    functions = generate_decls_and_callbacks(db, prefix=prefix)
 
     # Structure declaration code
     members = []
@@ -1042,6 +1041,13 @@
                 # added only for the macro, not the decl
                 continue
             restype, args = c_function_signature(db, func)
+            callargs = ', '.join('arg%d' % (i,)
+                                for i in range(len(func.argtypes)))
+            if func.restype is lltype.Void:
+                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+            else:
+                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
             members.append('%s (*%s)(%s);' % (restype, name, args))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
@@ -1207,7 +1213,7 @@
     else:
         return None
 
-def generate_decls_and_callbacks(db, prefix='', translating=False):
+def generate_decls_and_callbacks(db, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
     for name in SYMBOLS_C:
@@ -1259,14 +1265,6 @@
             header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
             header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
-            if not translating:
-                callargs = ', '.join('arg%d' % (i,)
-                                    for i in range(len(func.argtypes)))
-                if func.restype is lltype.Void:
-                    body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
-                else:
-                    body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
-                functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
 
     for name in VA_TP_LIST:
         name_no_star = process_va_name(name)
@@ -1400,8 +1398,7 @@
     db = LowLevelDatabase()
     prefix = 'PyPy'
 
-    functions = generate_decls_and_callbacks(
-            db, prefix=prefix, translating=True)
+    functions = generate_decls_and_callbacks(db, prefix=prefix)
 
     code = "#include <Python.h>\n"
     if use_micronumpy:

From pypy.commits at gmail.com  Wed Jan  4 19:08:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 04 Jan 2017 16:08:53 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Kill VA_TP_LIST and its support
 code, since it's always empty
Message-ID: <586d8e95.0e0a1c0a.5228b.c8a6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89368:feebb9e99ec7
Date: 2017-01-05 00:06 +0000
http://bitbucket.org/pypy/pypy/changeset/feebb9e99ec7/

Log:	Kill VA_TP_LIST and its support code, since it's always empty

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -72,7 +72,6 @@
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
-VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list')
 CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
                                        hints={'nolength': True}),
                           use_cache=False)
@@ -674,12 +673,6 @@
 # a pointer to PyObject
 PyObjectP = rffi.CArrayPtr(PyObject)
 
-VA_TP_LIST = {}
-#{'int': lltype.Signed,
-#              'PyObject*': PyObject,
-#              'PyObject**': PyObjectP,
-#              'int*': rffi.INTP}
-
 def configure_types():
     for config in (CConfig, CConfig2):
         for name, TYPE in rffi_platform.configure(config).iteritems():
@@ -953,15 +946,6 @@
     wrapper_second_level._dont_inline_ = True
     return wrapper_second_level
 
-def process_va_name(name):
-    return name.replace('*', '_star')
-
-def setup_va_functions(eci):
-    for name, TP in VA_TP_LIST.iteritems():
-        name_no_star = process_va_name(name)
-        func = rffi.llexternal('pypy_va_get_%s' % name_no_star, [VA_LIST_P],
-                               TP, compilation_info=eci)
-        globals()['va_get_%s' % name_no_star] = func
 
 def setup_init_functions(eci, prefix):
     # jump through hoops to avoid releasing the GIL during initialization
@@ -1149,7 +1133,6 @@
             pypyAPI[structindex[name]] = ctypes.cast(
                 ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
                 ctypes.c_void_p)
-    setup_va_functions(eci)
 
     setup_init_functions(eci, prefix)
     return modulename.new(ext='')
@@ -1266,13 +1249,6 @@
             restype, args = c_function_signature(db, func)
             header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
 
-    for name in VA_TP_LIST:
-        name_no_star = process_va_name(name)
-        header = ('%s pypy_va_get_%s(va_list* vp)' %
-                  (name, name_no_star))
-        pypy_decls.append('RPY_EXTERN ' + header + ';')
-        functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
-
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
             name, header = name.split("#")
@@ -1409,7 +1385,6 @@
     space.fromcache(State).install_dll(eci)
 
     run_bootstrap_functions(space)
-    setup_va_functions(eci)
 
     # emit uninitialized static data
     builder = space.fromcache(State).builder = TranslationObjBuilder()

From pypy.commits at gmail.com  Wed Jan  4 19:17:42 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 04 Jan 2017 16:17:42 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Remove always-empty 'functions'
 from generate_decls_and_callback()
Message-ID: <586d90a6.8675c20a.dfef3.3616@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89369:083af902cb24
Date: 2017-01-05 00:16 +0000
http://bitbucket.org/pypy/pypy/changeset/083af902cb24/

Log:	Remove always-empty 'functions' from generate_decls_and_callback()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1014,9 +1014,10 @@
     db = LowLevelDatabase()
     prefix = 'cpyexttest'
 
-    functions = generate_decls_and_callbacks(db, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
 
     # Structure declaration code
+    functions = []
     members = []
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
@@ -1222,8 +1223,7 @@
     pypy_macros_h = udir.join('pypy_macros.h')
     pypy_macros_h.write('\n'.join(pypy_macros))
 
-    # implement function callbacks and generate function decls
-    functions = []
+    # generate function decls
     decls = {}
     pypy_decls = decls[pypy_decl] = []
     pypy_decls.append('#define Signed   long           /* xxx temporary fix */\n')
@@ -1267,7 +1267,6 @@
     for header_name, header_decls in decls.iteritems():
         decl_h = udir.join(header_name)
         decl_h.write('\n'.join(header_decls))
-    return functions
 
 separate_module_files = [source_dir / "varargwrapper.c",
                          source_dir / "pyerrors.c",
@@ -1374,12 +1373,11 @@
     db = LowLevelDatabase()
     prefix = 'PyPy'
 
-    functions = generate_decls_and_callbacks(db, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
 
     code = "#include <Python.h>\n"
     if use_micronumpy:
         code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
-    code  += "\n".join(functions)
 
     eci = build_eci(code, use_micronumpy, translating=True)
     space.fromcache(State).install_dll(eci)

From pypy.commits at gmail.com  Wed Jan  4 19:43:21 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 04 Jan 2017 16:43:21 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Raise an exception inside
 mangle_name() when the name is invalid
Message-ID: <586d96a9.46831c0a.675a2.c95d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89370:75557bb61dfd
Date: 2017-01-05 00:42 +0000
http://bitbucket.org/pypy/pypy/changeset/75557bb61dfd/

Log:	Raise an exception inside mangle_name() when the name is invalid

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1195,14 +1195,13 @@
     elif name.startswith('_Py'):
         return '_' + prefix + name[3:]
     else:
-        return None
+        raise ValueError("Error converting '%s'" % name)
 
 def generate_decls_and_callbacks(db, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
     for name in SYMBOLS_C:
         newname = mangle_name(prefix, name)
-        assert newname, name
         pypy_macros.append('#define %s %s' % (name, newname))
 
     # Generate defines
@@ -1244,7 +1243,6 @@
             if not func:
                 continue
             _name = mangle_name(prefix, name)
-            assert _name is not None, 'error converting %s' % name
             header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
             header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
@@ -1423,7 +1421,7 @@
         for name, func in header_functions.iteritems():
             if not func:
                 continue
-            newname = mangle_name(prefix, name) or name
+            newname = mangle_name(prefix, name)
             deco = entrypoint_lowlevel("cpyext", func.argtypes, newname,
                                         relax=True)
             deco(func.get_wrapper(space))
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl, Py_buffer, Py_bufferP)
+    pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
@@ -351,7 +351,7 @@
         else:
             #do not call twice
             return
-        if self.releasebufferproc:        
+        if self.releasebufferproc:
             func_target = rffi.cast(releasebufferproc, self.releasebufferproc)
             with lltype.scoped_alloc(Py_buffer) as pybuf:
                 pybuf.c_buf = self.ptr
@@ -416,7 +416,7 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self, 
+        buf = CPyBuffer(space, ptr[0], size, w_self,
                                releasebuffer=releasebuffer)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
@@ -522,7 +522,7 @@
     w_type = space.gettypeobject(typedef)
 
     header = pypy_decl
-    if mangle_name('', typedef.name) is None:
+    if not (name.startswith('Py') or name.startswith('_Py')):
         header = None
     handled = False
     # unary functions

From pypy.commits at gmail.com  Thu Jan  5 00:18:56 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 04 Jan 2017 21:18:56 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: add passing test, refactor len(s)
Message-ID: <586dd740.8c1f1c0a.f8a7f.049c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89371:ac830f4f76a0
Date: 2017-01-03 22:41 +0200
http://bitbucket.org/pypy/pypy/changeset/ac830f4f76a0/

Log:	add passing test, refactor len(s)

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -80,13 +80,14 @@
     """
     py_str = rffi.cast(PyBytesObject, py_obj)
     s = space.str_w(w_obj)
-    if py_str.c_ob_size  < len(s):
+    len_s = len(s)
+    if py_str.c_ob_size  < len_s:
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
-            py_str.c_ob_size, len(s))
+            py_str.c_ob_size, len_s)
     with rffi.scoped_nonmovingbuffer(s) as s_ptr:
-        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
-    py_str.c_ob_sval[len(s)] = '\0'
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len_s)
+    py_str.c_ob_sval[len_s] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -353,11 +353,15 @@
 
                 data = PyString_AS_STRING(args);
                 len = PyString_GET_SIZE(args);
-                if (data == NULL || len < 1)
+                if (data == NULL)
                     Py_RETURN_NONE;
                 obj = PyArray_Scalar(data, len);
                 return obj;
              """),
+            ("get_len", "METH_O",
+             """
+                return PyLong_FromLong(PyObject_Size(args));
+             """),
             ], prologue="""
                 #include <Python.h>
                 PyTypeObject PyStringArrType_Type = {
@@ -439,6 +443,9 @@
         a = module.newsubstr('abc')
         assert type(a).__name__ == 'string_'
         assert a == 'abc'
+        assert 3 == module.get_len(a)
+        b = module.newsubstr('')
+        assert 0 == module.get_len(b)
 
 class TestBytes(BaseApiTest):
     def test_bytes_resize(self, space, api):

From pypy.commits at gmail.com  Thu Jan  5 00:18:58 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 04 Jan 2017 21:18:58 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: avoid calling tp_hash via
 hash_w(w_obj) during object attach or realize
Message-ID: <586dd742.61c9c20a.d7016.dd47@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89372:7bce3ce5fbce
Date: 2017-01-05 07:17 +0200
http://bitbucket.org/pypy/pypy/changeset/7bce3ce5fbce/

Log:	avoid calling tp_hash via hash_w(w_obj) during object attach or
	realize

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -88,7 +88,10 @@
     with rffi.scoped_nonmovingbuffer(s) as s_ptr:
         rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len_s)
     py_str.c_ob_sval[len_s] = '\0'
-    py_str.c_ob_shash = space.hash_w(w_obj)
+    # if py_obj has a tp_hash, this will try to call it, but the objects are
+    # not fully linked yet
+    #py_str.c_ob_shash = space.hash_w(w_obj)
+    py_str.c_ob_shash = space.hash_w(space.newbytes(s))
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
 def bytes_realize(space, py_obj):
@@ -101,7 +104,9 @@
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(W_BytesObject, w_type)
     w_obj.__init__(s)
-    py_str.c_ob_shash = space.hash_w(w_obj)
+    # if py_obj has a tp_hash, this will try to call it but the object is
+    # not realized yet
+    py_str.c_ob_shash = space.hash_w(space.newbytes(s))
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
     return w_obj
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -437,6 +437,7 @@
                 PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
                 PyStringArrType_Type.tp_itemsize = sizeof(char);
                 PyStringArrType_Type.tp_base = &PyString_Type;
+                PyStringArrType_Type.tp_hash = PyString_Type.tp_hash;
                 if (PyType_Ready(&PyStringArrType_Type) < 0) INITERROR;
             ''')
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -65,9 +65,10 @@
 def unicode_attach(space, py_obj, w_obj, w_userdata=None):
     "Fills a newly allocated PyUnicodeObject with a unicode string"
     py_unicode = rffi.cast(PyUnicodeObject, py_obj)
-    py_unicode.c_length = len(space.unicode_w(w_obj))
+    s = space.unicode_w(w_obj)
+    py_unicode.c_length = len(s)
     py_unicode.c_str = lltype.nullptr(rffi.CWCHARP.TO)
-    py_unicode.c_hash = space.hash_w(w_obj)
+    py_unicode.c_hash = space.hash_w(space.newunicode(s))
     py_unicode.c_defenc = lltype.nullptr(PyObject.TO)
 
 def unicode_realize(space, py_obj):
@@ -80,7 +81,7 @@
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
     w_obj.__init__(s)
-    py_uni.c_hash = space.hash_w(w_obj)
+    py_uni.c_hash = space.hash_w(space.newunicode(s))
     track_reference(space, py_obj, w_obj)
     return w_obj
 

From pypy.commits at gmail.com  Thu Jan  5 03:39:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 05 Jan 2017 00:39:51 -0800 (PST)
Subject: [pypy-commit] pypy default: Test,
 fix (filename too long in the jitcodes directory!)
Message-ID: <586e0657.c5311c0a.d6d87.37f2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89373:606e869ec3f2
Date: 2017-01-05 08:58 +0100
http://bitbucket.org/pypy/pypy/changeset/606e869ec3f2/

Log:	Test, fix (filename too long in the jitcodes directory!)

diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -106,8 +106,9 @@
         else:
             name = 'unnamed' % id(ssarepr)
         i = 1
-        # escape <lambda> names for windows
-        name = name.replace('<lambda>', '_(lambda)_')
+        # escape names like <lambda> for windows by removing any strange
+        # character; then make sure the names are not too long
+        name = ''.join(c for c in name if c.isalnum() or c == '_')[:60]
         extra = ''
         while dir.join(name+extra).check():
             i += 1
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4613,3 +4613,10 @@
         self.check_operations_history(guard_nonnull=0, guard_nonnull_class=0,
                                       guard_class=2,
                                       assert_not_none=2) # before optimization
+
+    def test_call_time_clock(self):
+        import time
+        def g():
+            time.clock()
+            return 0
+        self.interp_operations(g, [])

From pypy.commits at gmail.com  Thu Jan  5 07:29:55 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 05 Jan 2017 04:29:55 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Remove unnecessary
 special-casing of cpyext in space.setup_builtin_modules()
Message-ID: <586e3c43.e576c20a.37a92.09d2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89374:68058ce4557d
Date: 2017-01-05 12:29 +0000
http://bitbucket.org/pypy/pypy/changeset/68058ce4557d/

Log:	Remove unnecessary special-casing of cpyext in
	space.setup_builtin_modules()

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -219,7 +219,7 @@
             # cpyext types that may have only old buffer interface
             w_impl = space.lookup(self, '__wbuffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self, 
+            w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
@@ -665,9 +665,6 @@
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
-        if self.config.objspace.usemodules.cpyext:
-            from pypy.module.cpyext.state import State
-            self.fromcache(State).build_api(self)
         self.getbuiltinmodule('sys')
         self.getbuiltinmodule('imp')
         self.getbuiltinmodule('__builtin__')
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -13,12 +13,15 @@
 
     atexit_funcs = []
 
+    def setup_after_space_initialization(self):
+        self.space.fromcache(State).build_api(self.space)
+
     def startup(self, space):
         space.fromcache(State).startup(space)
         method = pypy.module.cpyext.typeobject.get_new_method_def(space)
         w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap(''))
         space.appexec([space.type(w_obj)], """(methodtype):
-            from pickle import Pickler 
+            from pickle import Pickler
             Pickler.dispatch[methodtype] = Pickler.save_global
         """)
 

From pypy.commits at gmail.com  Thu Jan  5 08:08:45 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 05 Jan 2017 05:08:45 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Put deallocation trigger
 initialisation all in one place
Message-ID: <586e455d.4c9d1c0a.74f50.d448@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89375:64f7a07e347f
Date: 2017-01-05 13:08 +0000
http://bitbucket.org/pypy/pypy/changeset/64f7a07e347f/

Log:	Put deallocation trigger initialisation all in one place

diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -1,5 +1,4 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.interpreter import gateway
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
 
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -2,7 +2,6 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import executioncontext
-from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
@@ -15,9 +14,6 @@
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
         self.builder = None
-        if space.config.translation.gc != "boehm":
-            pyobj_dealloc_action = PyObjDeallocAction(space)
-            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -74,6 +70,12 @@
                 action = BoehmPyObjDeallocAction(self.space)
                 self.space.actionflag.register_periodic_action(action,
                     use_bytecode_counter=True)
+            else:
+                pyobj_dealloc_action = PyObjDeallocAction(space)
+                self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+                rawrefcount.init(
+                    llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
+                    self.dealloc_trigger))
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -89,10 +91,6 @@
         from pypy.module.cpyext.api import INIT_FUNCTIONS
 
         if we_are_translated():
-            if space.config.translation.gc != "boehm":
-                rawrefcount.init(
-                    llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
-                    self.dealloc_trigger))
             self.builder.attach_all(space)
 
         setup_new_method_def(space)

From pypy.commits at gmail.com  Thu Jan  5 08:49:14 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 05 Jan 2017 05:49:14 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Move rawrefcount init code out
 of build_bridge()
Message-ID: <586e4eda.c4811c0a.16ab2.c499@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89376:95ae01dcc691
Date: 2017-01-05 13:48 +0000
http://bitbucket.org/pypy/pypy/changeset/95ae01dcc691/

Log:	Move rawrefcount init code out of build_bridge()

diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -13,7 +13,9 @@
     atexit_funcs = []
 
     def setup_after_space_initialization(self):
-        self.space.fromcache(State).build_api(self.space)
+        state = self.space.fromcache(State)
+        state.setup_rawrefcount()
+        state.build_api()
 
     def startup(self, space):
         space.fromcache(State).startup(space)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1068,25 +1068,11 @@
     space.fromcache(State).install_dll(eci)
     modulename = py.path.local(eci.libraries[-1])
 
-    def dealloc_trigger():
-        from pypy.module.cpyext.pyobject import decref
-        print 'dealloc_trigger...'
-        while True:
-            ob = rawrefcount.next_dead(PyObject)
-            if not ob:
-                break
-            print 'deallocating PyObject', ob
-            decref(space, ob)
-        print 'dealloc_trigger DONE'
-        return "RETRY"
-    rawrefcount.init(dealloc_trigger)
-
     run_bootstrap_functions(space)
 
     # load the bridge, and init structure
     bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
 
-
     # populate static data
     builder = space.fromcache(State).builder = TestingObjBuilder()
     for name, (typ, expr) in GLOBALS.iteritems():
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -54,21 +54,25 @@
                         "Function returned an error result without setting an "
                         "exception")
 
-    def build_api(self, space):
-        """NOT_RPYTHON
-        This function is called when at object space creation,
-        and drives the compilation of the cpyext library
-        """
-        from pypy.module.cpyext import api
-        state = self.space.fromcache(State)
+    def setup_rawrefcount(self):
+        space = self.space
         if not self.space.config.translating:
-            state.api_lib = str(api.build_bridge(self.space))
+            def dealloc_trigger():
+                from pypy.module.cpyext.pyobject import PyObject, decref
+                print 'dealloc_trigger...'
+                while True:
+                    ob = rawrefcount.next_dead(PyObject)
+                    if not ob:
+                        break
+                    print 'deallocating PyObject', ob
+                    decref(space, ob)
+                print 'dealloc_trigger DONE'
+                return "RETRY"
+            rawrefcount.init(dealloc_trigger)
         else:
-            api.setup_library(self.space)
-            #
-            if self.space.config.translation.gc == "boehm":
-                action = BoehmPyObjDeallocAction(self.space)
-                self.space.actionflag.register_periodic_action(action,
+            if space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(space)
+                space.actionflag.register_periodic_action(action,
                     use_bytecode_counter=True)
             else:
                 pyobj_dealloc_action = PyObjDeallocAction(space)
@@ -77,6 +81,17 @@
                     llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
                     self.dealloc_trigger))
 
+    def build_api(self):
+        """NOT_RPYTHON
+        This function is called when at object space creation,
+        and drives the compilation of the cpyext library
+        """
+        from pypy.module.cpyext import api
+        if not self.space.config.translating:
+            self.api_lib = str(api.build_bridge(self.space))
+        else:
+            api.setup_library(self.space)
+
     def install_dll(self, eci):
         """NOT_RPYTHON
         Called when the dll has been compiled"""

From pypy.commits at gmail.com  Thu Jan  5 09:05:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 05 Jan 2017 06:05:59 -0800 (PST)
Subject: [pypy-commit] pypy default: Update to latest cffi's test_c.py
Message-ID: <586e52c7.2105c30a.98f04.d9c6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89377:4927aa24cb71
Date: 2017-01-05 14:53 +0100
http://bitbucket.org/pypy/pypy/changeset/4927aa24cb71/

Log:	Update to latest cffi's test_c.py

diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3419,17 +3419,22 @@
     BCharA = new_array_type(BCharP, None)
     p1 = from_buffer(BCharA, b"foo")
     assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # Python 2 only
         contents = from_buffer(BCharA, buffer(b"foo"))
+        assert len(contents) == len(p1)
         for i in range(len(contents)):
             assert contents[i] == p1[i]
-        p4 = from_buffer(BCharA, b"f\x00\x00\x00o\x00\x00\x00o\x00\x00\x00")
+        p4 = buffer(u+"foo")
         contents = from_buffer(BCharA, buffer(u+"foo"))
+        assert len(contents) == len(p4)
         for i in range(len(contents)):
             assert contents[i] == p4[i]
     try:
@@ -3438,6 +3443,7 @@
         pass
     else:
         contents = from_buffer(BCharA, memoryview(b"foo"))
+        assert len(contents) == len(p1)
         for i in range(len(contents)):
             assert contents[i] == p1[i]
 

From pypy.commits at gmail.com  Thu Jan  5 09:09:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 05 Jan 2017 06:09:23 -0800 (PST)
Subject: [pypy-commit] pypy default: CPython fix
Message-ID: <586e5393.27edc20a.c09ac.5b32@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89378:c71c2d504102
Date: 2017-01-05 15:08 +0100
http://bitbucket.org/pypy/pypy/changeset/c71c2d504102/

Log:	CPython fix

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1320,7 +1320,8 @@
         def __init__(self, string):
             self.ptr = str2charp(string, track_allocation=False)
         def __del__(self):
-            free_charp(self.ptr, track_allocation=False)
+            if free_charp is not None:    # CPython shutdown
+                free_charp(self.ptr, track_allocation=False)
 
     TEST_RAW_ADDR_KEEP_ALIVE = {}
 

From pypy.commits at gmail.com  Thu Jan  5 09:31:47 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 05 Jan 2017 06:31:47 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Sticking Nones into
 FUNCTIONS_BY_HEADER does nothing whatsoever
Message-ID: <586e58d3.054fc20a.50812.8694@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89379:a6c74862ebdf
Date: 2017-01-05 14:30 +0000
http://bitbucket.org/pypy/pypy/changeset/a6c74862ebdf/

Log:	Sticking Nones into FUNCTIONS_BY_HEADER does nothing whatsoever

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1022,9 +1022,6 @@
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                # added only for the macro, not the decl
-                continue
             restype, args = c_function_signature(db, func)
             callargs = ', '.join('arg%d' % (i,)
                                 for i in range(len(func.argtypes)))
@@ -1115,8 +1112,6 @@
     # implement structure initialization code
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if name.startswith('cpyext_') or func is None: # XXX hack
-                continue
             pypyAPI[structindex[name]] = ctypes.cast(
                 ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
                 ctypes.c_void_p)
@@ -1226,8 +1221,6 @@
             header = decls[header_name]
 
         for name, func in sorted(header_functions.iteritems()):
-            if not func:
-                continue
             _name = mangle_name(prefix, name)
             header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
@@ -1341,9 +1334,7 @@
         return use_micronumpy
     # import registers api functions by side-effect, we also need HEADER
     from pypy.module.cpyext.ndarrayobject import HEADER
-    global FUNCTIONS_BY_HEADER, separate_module_files
-    for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
-        FUNCTIONS_BY_HEADER[HEADER][func_name] = None
+    global separate_module_files
     register_global("PyArray_Type",
         'PyTypeObject*',  "space.gettypeobject(W_NDimArray.typedef)",
         header=HEADER)
@@ -1405,8 +1396,6 @@
 
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                continue
             newname = mangle_name(prefix, name)
             deco = entrypoint_lowlevel("cpyext", func.argtypes, newname,
                                         relax=True)

From pypy.commits at gmail.com  Thu Jan  5 11:21:55 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 05 Jan 2017 08:21:55 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Create write_header() function
Message-ID: <586e72a3.8f95c20a.1cc9c.765f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89380:4f9fc8a84053
Date: 2017-01-05 16:18 +0000
http://bitbucket.org/pypy/pypy/changeset/4f9fc8a84053/

Log:	Create write_header() function

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1178,6 +1178,18 @@
     else:
         raise ValueError("Error converting '%s'" % name)
 
+def write_header(header_name, decls):
+    lines = [
+        '#define Signed   long           /* xxx temporary fix */',
+        '#define Unsigned unsigned long  /* xxx temporary fix */',
+        '',] + decls + [
+        '',
+        '#undef Signed    /* xxx temporary fix */',
+        '#undef Unsigned  /* xxx temporary fix */',
+        '']
+    decl_h = udir.join(header_name)
+    decl_h.write('\n'.join(lines))
+
 def generate_decls_and_callbacks(db, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
@@ -1204,22 +1216,12 @@
     pypy_macros_h.write('\n'.join(pypy_macros))
 
     # generate function decls
-    decls = {}
-    pypy_decls = decls[pypy_decl] = []
-    pypy_decls.append('#define Signed   long           /* xxx temporary fix */\n')
-    pypy_decls.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-
+    decls = defaultdict(list)
     for decl in FORWARD_DECLS:
-        pypy_decls.append("%s;" % (decl,))
+        decls[pypy_decl].append("%s;" % (decl,))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
-        if header_name not in decls:
-            header = decls[header_name] = []
-            header.append('#define Signed   long           /* xxx temporary fix */\n')
-            header.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-        else:
-            header = decls[header_name]
-
+        header = decls[header_name]
         for name, func in sorted(header_functions.iteritems()):
             _name = mangle_name(prefix, name)
             header.append("#define %s %s" % (name, _name))
@@ -1236,14 +1238,8 @@
         decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
         decls[header].append('PyAPI_DATA(%s) %s;' % (typ, name))
 
-    for header_name in FUNCTIONS_BY_HEADER.keys():
-        header = decls[header_name]
-        header.append('#undef Signed    /* xxx temporary fix */\n')
-        header.append('#undef Unsigned  /* xxx temporary fix */\n')
-
     for header_name, header_decls in decls.iteritems():
-        decl_h = udir.join(header_name)
-        decl_h.write('\n'.join(header_decls))
+        write_header(header_name, header_decls)
 
 separate_module_files = [source_dir / "varargwrapper.c",
                          source_dir / "pyerrors.c",

From pypy.commits at gmail.com  Thu Jan  5 11:21:57 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 05 Jan 2017 08:21:57 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Call the right version of
 rawrefcount.init() when translating
Message-ID: <586e72a5.45f6c20a.23a87.5dc6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89381:5fa79126d59f
Date: 2017-01-05 16:21 +0000
http://bitbucket.org/pypy/pypy/changeset/5fa79126d59f/

Log:	Call the right version of rawrefcount.init() when translating

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -77,9 +77,6 @@
             else:
                 pyobj_dealloc_action = PyObjDeallocAction(space)
                 self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
-                rawrefcount.init(
-                    llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
-                    self.dealloc_trigger))
 
     def build_api(self):
         """NOT_RPYTHON
@@ -106,6 +103,12 @@
         from pypy.module.cpyext.api import INIT_FUNCTIONS
 
         if we_are_translated():
+            if space.config.translation.gc != "boehm":
+                # This must be called in RPython, the untranslated version
+                # does something different. Sigh.
+                rawrefcount.init(
+                    llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
+                    self.dealloc_trigger))
             self.builder.attach_all(space)
 
         setup_new_method_def(space)

From pypy.commits at gmail.com  Thu Jan  5 11:48:40 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 05 Jan 2017 08:48:40 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-from: change frombuffer/fromobject
Message-ID: <586e78e8.2105c30a.98f04.1e2a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: cpyext-from
Changeset: r89382:34330a47b61f
Date: 2017-01-05 17:43 +0100
http://bitbucket.org/pypy/pypy/changeset/34330a47b61f/

Log:	change frombuffer/fromobject

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -6,9 +6,11 @@
              decref, from_ref, make_typedescr)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.objspace.std.memoryobject import W_MemoryView
 from pypy.module.cpyext.object import _dealloc
 from pypy.module.cpyext.import_ import PyImport_Import
+from rpython.rlib.buffer import Buffer
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView")
 
@@ -95,22 +97,7 @@
     if not isinstance(w_obj, W_MemoryView):
         return lltype.nullptr(Py_buffer)
     py_memobj = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_obj)) # no inc_ref
-    view = py_memobj.c_view
-    ndim = w_obj.buf.getndim()
-    if ndim >= Py_MAX_NDIMS:
-        # XXX warn?
-        return view
-    fill_Py_buffer(space, w_obj.buf, view)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-        #view.c_obj = make_ref(space, w_obj) # NO - this creates a ref cycle!
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
-    except ValueError:
-        w_s = w_obj.descr_tobytes(space)
-        view.c_obj = make_ref(space, w_s)
-        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
-        rffi.setintfield(view, 'c_readonly', 1)
-    return view
+    return py_memobj.c_view
 
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
@@ -147,6 +134,7 @@
         view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    rffi.setintfield(view, 'c_readonly', buf.readonly)
     return 0
 
 def _IsFortranContiguous(view):
@@ -202,9 +190,46 @@
         return (_IsCContiguous(view) or _IsFortranContiguous(view))
     return 0
 
+def _py_memoryview_frombuffer(space, buf):
+    mview = W_MemoryView(buf)
+    return mview
+
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_FromObject(space, w_obj):
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    try:
+        buf = space.buffer_w(w_obj, space.BUF_FULL_RO)
+    except BufferInterfaceNotFound:
+        # review, buffer interface is not found
+        raise oefmt(space.w_TypeError, "cannot make memory view because object "
+                                       "does not have the buffer interface")
+
+    # allocate a new pypy memory view and expose it to cpython using as_pyobj
+    w_mview = W_MemoryView(buf)
+    py_mview = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_mview))
+
+    view = py_mview.c_view
+    fill_Py_buffer(space, buf, view)
+    view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    # the docs say that in PyMemoryView_FromBuffer (and thus FromObject)
+    # this object must be NULL
+    view.c_obj = make_ref(space, w_obj)
+
+    # XXX what about w_mview.base = w_obj (see cpython 2.7 implementation)
+    return py_mview
+
+class Py_bufferBuffer(Buffer):
+    _attrs_ = ['readonly', 'view']
+    _immutable_ = True
+
+    def __init__(self, view):
+        self.view = view
+        self.readonly = view.c_readonly
+
+    def getlength(self):
+        return self.view.c_len
+
+    def get_raw_address(self):
+        return self.view.c_buf
 
 @cpython_api([Py_bufferP], PyObject)
 def PyMemoryView_FromBuffer(space, view):
@@ -212,10 +237,9 @@
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
-    w_obj = from_ref(space, view.c_obj)
-    if isinstance(w_obj, W_MemoryView):
-        return w_obj
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    w_mview = W_MemoryView(Py_bufferBuffer(view))
+    py_mview = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_mview))
+    return py_mview
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -4,6 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
+from pypy.module.cpyext.pyobject import from_ref
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
@@ -19,8 +20,9 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        w_memoryview = api.PyMemoryView_FromObject(w_buf)
-        view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        py_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_memoryview = from_ref(space, py_memoryview)
+        view = api.PyMemoryView_GET_BUFFER(py_memoryview)
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)
         assert f == 'B'
@@ -29,7 +31,7 @@
         assert view.c_len == 5
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
-        w_mv = api.PyMemoryView_FromBuffer(view)
+        w_mv = from_ref(space, api.PyMemoryView_FromBuffer(view))
         for f in ('format', 'itemsize', 'ndim', 'readonly', 
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)

From pypy.commits at gmail.com  Thu Jan  5 13:13:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 05 Jan 2017 10:13:33 -0800 (PST)
Subject: [pypy-commit] pypy default: Rename this variable---it's not the
 size of a pinned object, it's the
Message-ID: <586e8ccd.46831c0a.675a2.3669@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89383:a40f33958400
Date: 2017-01-05 18:16 +0100
http://bitbucket.org/pypy/pypy/changeset/a40f33958400/

Log:	Rename this variable---it's not the size of a pinned object, it's
	the size of the interval between two pinned objects

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1770,11 +1770,11 @@
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
-            pinned_obj_size = llarena.getfakearenaaddress(cur) - prev
+            free_range_size = llarena.getfakearenaaddress(cur) - prev
             if self.gc_nursery_debug:
-                llarena.arena_reset(prev, pinned_obj_size, 3)
+                llarena.arena_reset(prev, free_range_size, 3)
             else:
-                llarena.arena_reset(prev, pinned_obj_size, 0)
+                llarena.arena_reset(prev, free_range_size, 0)
             #
             # clean up object's flags
             obj = cur + size_gc_header
@@ -1784,7 +1784,7 @@
             nursery_barriers.append(cur)
             #
             # update 'prev' to the end of the 'cur' object
-            prev = prev + pinned_obj_size + \
+            prev = prev + free_range_size + \
                 (size_gc_header + self.get_size(obj))
         #
         # reset everything after the last pinned object till the end of the arena

From pypy.commits at gmail.com  Thu Jan  5 17:46:55 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 05 Jan 2017 14:46:55 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-cleanup: Close branch cpyext-cleanup
Message-ID: <586eccdf.44a5c20a.50b3c.d3fc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-cleanup
Changeset: r89384:6eebe65fe506
Date: 2017-01-05 22:46 +0000
http://bitbucket.org/pypy/pypy/changeset/6eebe65fe506/

Log:	Close branch cpyext-cleanup


From pypy.commits at gmail.com  Thu Jan  5 17:47:11 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 05 Jan 2017 14:47:11 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in cpyext-cleanup (pull request
 #504)
Message-ID: <586eccef.45f6c20a.23a87.f49e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89385:88be41751667
Date: 2017-01-05 22:46 +0000
http://bitbucket.org/pypy/pypy/changeset/88be41751667/

Log:	Merged in cpyext-cleanup (pull request #504)

	Refactor cpyext initialisation.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -219,7 +219,7 @@
             # cpyext types that may have only old buffer interface
             w_impl = space.lookup(self, '__wbuffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self, 
+            w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
@@ -665,9 +665,6 @@
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
-        if self.config.objspace.usemodules.cpyext:
-            from pypy.module.cpyext.state import State
-            self.fromcache(State).build_api(self)
         self.getbuiltinmodule('sys')
         self.getbuiltinmodule('imp')
         self.getbuiltinmodule('__builtin__')
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -1,5 +1,4 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.interpreter import gateway
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
 
@@ -13,12 +12,17 @@
 
     atexit_funcs = []
 
+    def setup_after_space_initialization(self):
+        state = self.space.fromcache(State)
+        state.setup_rawrefcount()
+        state.build_api()
+
     def startup(self, space):
         space.fromcache(State).startup(space)
         method = pypy.module.cpyext.typeobject.get_new_method_def(space)
         w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap(''))
         space.appexec([space.type(w_obj)], """(methodtype):
-            from pickle import Pickler 
+            from pickle import Pickler
             Pickler.dispatch[methodtype] = Pickler.save_global
         """)
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1,5 +1,6 @@
 import ctypes
 import sys, os
+from collections import defaultdict
 
 import py
 
@@ -71,7 +72,6 @@
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
-VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list')
 CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
                                        hints={'nolength': True}),
                           use_cache=False)
@@ -364,8 +364,8 @@
         func_name = func.func_name
         if header is not None:
             c_name = None
-            assert func_name not in FUNCTIONS, (
-                "%s already registered" % func_name)
+            if func_name in FUNCTIONS_BY_HEADER[header]:
+                raise ValueError("%s already registered" % func_name)
         else:
             c_name = func_name
         api_function = ApiFunction(argtypes, restype, func, error,
@@ -463,9 +463,7 @@
             return res
 
         if header is not None:
-            if header == DEFAULT_HEADER:
-                FUNCTIONS[func_name] = api_function
-            FUNCTIONS_BY_HEADER.setdefault(header, {})[func_name] = api_function
+            FUNCTIONS_BY_HEADER[header][func_name] = api_function
         INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
         return unwrapper  # used in 'normal' RPython code.
     return decorate
@@ -489,8 +487,7 @@
     GLOBALS[name] = (typ, expr)
 
 INTERPLEVEL_API = {}
-FUNCTIONS = {}
-FUNCTIONS_BY_HEADER = {}
+FUNCTIONS_BY_HEADER = defaultdict(dict)
 
 # These are C symbols which cpyext will export, but which are defined in .c
 # files somewhere in the implementation of cpyext (rather than being defined in
@@ -526,6 +523,8 @@
     'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor',
     'PyCapsule_SetContext', 'PyCapsule_Import', 'PyCapsule_Type', '_Py_get_capsule_type',
 
+    'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
+
     'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
 
     'PyOS_getsig', 'PyOS_setsig',
@@ -572,7 +571,9 @@
     # PyExc_AttributeError, PyExc_OverflowError, PyExc_ImportError,
     # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError,
     # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ...
-    for exc_name in exceptions.Module.interpleveldefs.keys():
+    global all_exceptions
+    all_exceptions = list(exceptions.Module.interpleveldefs)
+    for exc_name in all_exceptions:
         register_global('PyExc_' + exc_name,
             'PyTypeObject*',
             'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, ))
@@ -672,12 +673,6 @@
 # a pointer to PyObject
 PyObjectP = rffi.CArrayPtr(PyObject)
 
-VA_TP_LIST = {}
-#{'int': lltype.Signed,
-#              'PyObject*': PyObject,
-#              'PyObject**': PyObjectP,
-#              'int*': rffi.INTP}
-
 def configure_types():
     for config in (CConfig, CConfig2):
         for name, TYPE in rffi_platform.configure(config).iteritems():
@@ -951,21 +946,8 @@
     wrapper_second_level._dont_inline_ = True
     return wrapper_second_level
 
-def process_va_name(name):
-    return name.replace('*', '_star')
 
-def setup_va_functions(eci):
-    for name, TP in VA_TP_LIST.iteritems():
-        name_no_star = process_va_name(name)
-        func = rffi.llexternal('pypy_va_get_%s' % name_no_star, [VA_LIST_P],
-                               TP, compilation_info=eci)
-        globals()['va_get_%s' % name_no_star] = func
-
-def setup_init_functions(eci, translating):
-    if translating:
-        prefix = 'PyPy'
-    else:
-        prefix = 'cpyexttest'
+def setup_init_functions(eci, prefix):
     # jump through hoops to avoid releasing the GIL during initialization
     # of the cpyext module.  The C functions are called with no wrapper,
     # but must not do anything like calling back PyType_Ready().  We
@@ -1027,23 +1009,27 @@
 # Do not call this more than once per process
 def build_bridge(space):
     "NOT_RPYTHON"
-    from pypy.module.cpyext.pyobject import make_ref
     from rpython.translator.c.database import LowLevelDatabase
     use_micronumpy = setup_micronumpy(space)
     db = LowLevelDatabase()
-    prefix ='cpyexttest'
+    prefix = 'cpyexttest'
 
-    functions = generate_decls_and_callbacks(db, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
 
     # Structure declaration code
+    functions = []
     members = []
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                # added only for the macro, not the decl
-                continue
             restype, args = c_function_signature(db, func)
+            callargs = ', '.join('arg%d' % (i,)
+                                for i in range(len(func.argtypes)))
+            if func.restype is lltype.Void:
+                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+            else:
+                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
             members.append('%s (*%s)(%s);' % (restype, name, args))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
@@ -1055,15 +1041,8 @@
     """ % dict(members=structmembers)
 
     global_objects = []
-    for name, (typ, expr) in GLOBALS.iteritems():
-        if '#' in name:
-            continue
-        if typ == 'PyDateTime_CAPI*':
-            continue
-        elif name.startswith('PyExc_'):
-            global_objects.append('%s _%s;' % (typ[:-1], name))
-        else:
-            global_objects.append('%s %s = NULL;' % (typ, name))
+    for name in all_exceptions:
+        global_objects.append('PyTypeObject _PyExc_%s;' % name)
     global_code = '\n'.join(global_objects)
 
     prologue = ("#include <Python.h>\n"
@@ -1080,90 +1059,69 @@
             '\n' +
             '\n'.join(functions))
 
-    eci = build_eci(True, code, use_micronumpy)
+    eci = build_eci(code, use_micronumpy, translating=False)
     eci = eci.compile_shared_lib(
         outputfilename=str(udir / "module_cache" / "pypyapi"))
+    space.fromcache(State).install_dll(eci)
     modulename = py.path.local(eci.libraries[-1])
 
-    def dealloc_trigger():
-        from pypy.module.cpyext.pyobject import decref
-        print 'dealloc_trigger...'
-        while True:
-            ob = rawrefcount.next_dead(PyObject)
-            if not ob:
-                break
-            print 'deallocating PyObject', ob
-            decref(space, ob)
-        print 'dealloc_trigger DONE'
-        return "RETRY"
-    rawrefcount.init(dealloc_trigger)
-
     run_bootstrap_functions(space)
 
     # load the bridge, and init structure
     bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
 
-    space.fromcache(State).install_dll(eci)
+    # populate static data
+    builder = space.fromcache(State).builder = TestingObjBuilder()
+    for name, (typ, expr) in GLOBALS.iteritems():
+        if '#' in name:
+            name, header = name.split('#')
+            assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
+            isptr = False
+        elif name.startswith('PyExc_'):
+            isptr = False
+        elif typ == 'PyDateTime_CAPI*':
+            isptr = True
+        else:
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
 
-    # populate static data
-    builder = space.fromcache(StaticObjectBuilder)
-    for name, (typ, expr) in GLOBALS.iteritems():
         from pypy.module import cpyext    # for the eval() below
         w_obj = eval(expr)
-        if '#' in name:
-            name = name.split('#')[0]
-            isptr = False
-        else:
-            isptr = True
-        if name.startswith('PyExc_'):
-            isptr = False
-
         INTERPLEVEL_API[name] = w_obj
 
-        name = name.replace('Py', prefix)
+        mname = mangle_name(prefix, name)
         if isptr:
-            ptr = ctypes.c_void_p.in_dll(bridge, name)
-            if typ == 'PyObject*':
-                value = make_ref(space, w_obj)
-            elif typ == 'PyDateTime_CAPI*':
-                value = w_obj
-            else:
-                assert False, "Unknown static pointer: %s %s" % (typ, name)
+            assert typ == 'PyDateTime_CAPI*'
+            value = w_obj
+            ptr = ctypes.c_void_p.in_dll(bridge, mname)
             ptr.value = ctypes.cast(ll2ctypes.lltype2ctypes(value),
                                     ctypes.c_void_p).value
-        elif typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*'):
-            if name.startswith('PyPyExc_') or name.startswith('cpyexttestExc_'):
+        else:
+            if name.startswith('PyExc_'):
                 # we already have the pointer
-                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, in_dll)
             else:
                 # we have a structure, get its address
-                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, ctypes.pointer(in_dll))
             builder.prepare(py_obj, w_obj)
-        else:
-            assert False, "Unknown static object: %s %s" % (typ, name)
-    builder.attach_all()
+    builder.attach_all(space)
 
     pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
 
     # implement structure initialization code
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if name.startswith('cpyext_') or func is None: # XXX hack
-                continue
             pypyAPI[structindex[name]] = ctypes.cast(
                 ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
                 ctypes.c_void_p)
-    setup_va_functions(eci)
 
-    setup_init_functions(eci, translating=False)
+    setup_init_functions(eci, prefix)
     return modulename.new(ext='')
 
 
-class StaticObjectBuilder:
-    def __init__(self, space):
-        self.space = space
+class StaticObjectBuilder(object):
+    def __init__(self):
         self.static_pyobjs = []
         self.static_objs_w = []
         self.cpyext_type_init = None
@@ -1179,13 +1137,12 @@
         self.static_pyobjs.append(py_obj)
         self.static_objs_w.append(w_obj)
 
-    def attach_all(self):
+    def attach_all(self, space):
         # this is RPython, called once in pypy-c when it imports cpyext
         from pypy.module.cpyext.pyobject import get_typedescr, make_ref
         from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
         from pypy.module.cpyext.pyobject import track_reference
         #
-        space = self.space
         static_pyobjs = self.get_static_pyobjs()
         static_objs_w = self.static_objs_w
         for i in range(len(static_objs_w)):
@@ -1206,6 +1163,12 @@
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
+class TestingObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used in tests."""
+
+class TranslationObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used during translation."""
+
 
 def mangle_name(prefix, name):
     if name.startswith('Py'):
@@ -1213,23 +1176,26 @@
     elif name.startswith('_Py'):
         return '_' + prefix + name[3:]
     else:
-        return None
+        raise ValueError("Error converting '%s'" % name)
 
-def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
+def write_header(header_name, decls):
+    lines = [
+        '#define Signed   long           /* xxx temporary fix */',
+        '#define Unsigned unsigned long  /* xxx temporary fix */',
+        '',] + decls + [
+        '',
+        '#undef Signed    /* xxx temporary fix */',
+        '#undef Unsigned  /* xxx temporary fix */',
+        '']
+    decl_h = udir.join(header_name)
+    decl_h.write('\n'.join(lines))
+
+def generate_decls_and_callbacks(db, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
-    export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
-    for name in export_symbols:
-        if '#' in name:
-            name, header = name.split('#')
-        else:
-            header = pypy_decl
+    for name in SYMBOLS_C:
         newname = mangle_name(prefix, name)
-        assert newname, name
-        if header == pypy_decl:
-            pypy_macros.append('#define %s %s' % (name, newname))
-        if name.startswith("PyExc_"):
-            pypy_macros.append('#define _%s _%s' % (name, newname))
+        pypy_macros.append('#define %s %s' % (name, newname))
 
     # Generate defines
     for macro_name, size in [
@@ -1249,50 +1215,18 @@
     pypy_macros_h = udir.join('pypy_macros.h')
     pypy_macros_h.write('\n'.join(pypy_macros))
 
-    # implement function callbacks and generate function decls
-    functions = []
-    decls = {}
-    pypy_decls = decls[pypy_decl] = []
-    pypy_decls.append('#define Signed   long           /* xxx temporary fix */\n')
-    pypy_decls.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-
+    # generate function decls
+    decls = defaultdict(list)
     for decl in FORWARD_DECLS:
-        pypy_decls.append("%s;" % (decl,))
+        decls[pypy_decl].append("%s;" % (decl,))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
-        if header_name not in decls:
-            header = decls[header_name] = []
-            header.append('#define Signed   long           /* xxx temporary fix */\n')
-            header.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-        else:
-            header = decls[header_name]
-
+        header = decls[header_name]
         for name, func in sorted(header_functions.iteritems()):
-            if not func:
-                continue
-            if header == DEFAULT_HEADER:
-                _name = name
-            else:
-                # this name is not included in pypy_macros.h
-                _name = mangle_name(prefix, name)
-                assert _name is not None, 'error converting %s' % name
-                header.append("#define %s %s" % (name, _name))
+            _name = mangle_name(prefix, name)
+            header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, _name, args))
-            if api_struct:
-                callargs = ', '.join('arg%d' % (i,)
-                                    for i in range(len(func.argtypes)))
-                if func.restype is lltype.Void:
-                    body = "{ _pypyAPI.%s(%s); }" % (_name, callargs)
-                else:
-                    body = "{ return _pypyAPI.%s(%s); }" % (_name, callargs)
-                functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
-    for name in VA_TP_LIST:
-        name_no_star = process_va_name(name)
-        header = ('%s pypy_va_get_%s(va_list* vp)' %
-                  (name, name_no_star))
-        pypy_decls.append('RPY_EXTERN ' + header + ';')
-        functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
+            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
 
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
@@ -1301,19 +1235,11 @@
         elif name.startswith('PyExc_'):
             typ = 'PyObject*'
             header = pypy_decl
-        if header != pypy_decl:
-            decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
+        decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
         decls[header].append('PyAPI_DATA(%s) %s;' % (typ, name))
 
-    for header_name in FUNCTIONS_BY_HEADER.keys():
-        header = decls[header_name]
-        header.append('#undef Signed    /* xxx temporary fix */\n')
-        header.append('#undef Unsigned  /* xxx temporary fix */\n')
-
     for header_name, header_decls in decls.iteritems():
-        decl_h = udir.join(header_name)
-        decl_h.write('\n'.join(header_decls))
-    return functions
+        write_header(header_name, header_decls)
 
 separate_module_files = [source_dir / "varargwrapper.c",
                          source_dir / "pyerrors.c",
@@ -1325,6 +1251,7 @@
                          source_dir / "pythonrun.c",
                          source_dir / "sysmodule.c",
                          source_dir / "bufferobject.c",
+                         source_dir / "complexobject.c",
                          source_dir / "cobject.c",
                          source_dir / "structseq.c",
                          source_dir / "capsule.c",
@@ -1334,14 +1261,16 @@
                          source_dir / "pymem.c",
                          ]
 
-def build_eci(building_bridge, code, use_micronumpy=False):
+def build_eci(code, use_micronumpy=False, translating=False):
     "NOT_RPYTHON"
     # Build code and get pointer to the structure
     kwds = {}
 
     compile_extra=['-DPy_BUILD_CORE']
 
-    if building_bridge:
+    if translating:
+        kwds["includes"] = ['Python.h'] # this is our Python.h
+    else:
         if sys.platform == "win32":
             # '%s' undefined; assuming extern returning int
             compile_extra.append("/we4013")
@@ -1351,8 +1280,6 @@
         elif sys.platform.startswith('linux'):
             compile_extra.append("-Werror=implicit-function-declaration")
             compile_extra.append('-g')
-    else:
-        kwds["includes"] = ['Python.h'] # this is our Python.h
 
     # Generate definitions for global structures
     structs = ["#include <Python.h>"]
@@ -1403,9 +1330,7 @@
         return use_micronumpy
     # import registers api functions by side-effect, we also need HEADER
     from pypy.module.cpyext.ndarrayobject import HEADER
-    global FUNCTIONS_BY_HEADER, separate_module_files
-    for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
-        FUNCTIONS_BY_HEADER.setdefault(HEADER, {})[func_name] = None
+    global separate_module_files
     register_global("PyArray_Type",
         'PyTypeObject*',  "space.gettypeobject(W_NDimArray.typedef)",
         header=HEADER)
@@ -1419,21 +1344,19 @@
     db = LowLevelDatabase()
     prefix = 'PyPy'
 
-    functions = generate_decls_and_callbacks(db, api_struct=False, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
+
     code = "#include <Python.h>\n"
     if use_micronumpy:
         code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
-    code  += "\n".join(functions)
 
-    eci = build_eci(False, code, use_micronumpy)
-
+    eci = build_eci(code, use_micronumpy, translating=True)
     space.fromcache(State).install_dll(eci)
 
     run_bootstrap_functions(space)
-    setup_va_functions(eci)
 
     # emit uninitialized static data
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder = TranslationObjBuilder()
     lines = ['PyObject *pypy_static_pyobjs[] = {\n']
     include_lines = ['RPY_EXTERN PyObject *pypy_static_pyobjs[];\n']
     for name, (typ, expr) in sorted(GLOBALS.items()):
@@ -1441,17 +1364,15 @@
             name, header = name.split('#')
             assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
             typ = typ[:-1]
-            if header != pypy_decl:
-                # since the #define is not in pypy_macros, do it here
-                mname = mangle_name(prefix, name)
-                include_lines.append('#define %s %s\n' % (name, mname))
+            mname = mangle_name(prefix, name)
+            include_lines.append('#define %s %s\n' % (name, mname))
         elif name.startswith('PyExc_'):
             typ = 'PyTypeObject'
             name = '_' + name
         elif typ == 'PyDateTime_CAPI*':
             continue
         else:
-            assert False, "Unknown static data: %s %s" % (typ, name)
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
 
         from pypy.module import cpyext     # for the eval() below
         w_obj = eval(expr)
@@ -1471,20 +1392,15 @@
 
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                continue
-            newname = mangle_name('PyPy', name) or name
+            newname = mangle_name(prefix, name)
             deco = entrypoint_lowlevel("cpyext", func.argtypes, newname,
                                         relax=True)
             deco(func.get_wrapper(space))
 
-    setup_init_functions(eci, translating=True)
+    setup_init_functions(eci, prefix)
     trunk_include = pypydir.dirpath() / 'include'
     copy_header_files(trunk_include, use_micronumpy)
 
-def init_static_data_translated(space):
-    builder = space.fromcache(StaticObjectBuilder)
-    builder.attach_all()
 
 def _load_from_cffi(space, name, path, initptr):
     from pypy.module._cffi_backend import cffi1_module
diff --git a/pypy/module/cpyext/include/complexobject.h b/pypy/module/cpyext/include/complexobject.h
--- a/pypy/module/cpyext/include/complexobject.h
+++ b/pypy/module/cpyext/include/complexobject.h
@@ -16,23 +16,8 @@
     Py_complex cval;
 } PyComplexObject;
 
-/* generated function */
-PyAPI_FUNC(int) _PyComplex_AsCComplex(PyObject *, Py_complex *);
-PyAPI_FUNC(PyObject *) _PyComplex_FromCComplex(Py_complex *);
-
-Py_LOCAL_INLINE(Py_complex) PyComplex_AsCComplex(PyObject *obj)
-{
-    Py_complex result;
-    _PyComplex_AsCComplex(obj, &result);
-    return result;
-}
-
-// shmuller 2013/07/30: Make a function, since macro will fail in C++ due to 
-//                      const correctness if called with "const Py_complex"
-//#define PyComplex_FromCComplex(c) _PyComplex_FromCComplex(&c)
-Py_LOCAL_INLINE(PyObject *) PyComplex_FromCComplex(Py_complex c) {
-    return _PyComplex_FromCComplex(&c);
-}
+PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *obj);
+PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex c);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl, Py_buffer, Py_bufferP)
+    pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
@@ -351,7 +351,7 @@
         else:
             #do not call twice
             return
-        if self.releasebufferproc:        
+        if self.releasebufferproc:
             func_target = rffi.cast(releasebufferproc, self.releasebufferproc)
             with lltype.scoped_alloc(Py_buffer) as pybuf:
                 pybuf.c_buf = self.ptr
@@ -416,7 +416,7 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self, 
+        buf = CPyBuffer(space, ptr[0], size, w_self,
                                releasebuffer=releasebuffer)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
@@ -522,7 +522,7 @@
     w_type = space.gettypeobject(typedef)
 
     header = pypy_decl
-    if mangle_name('', typedef.name) is None:
+    if not (name.startswith('Py') or name.startswith('_Py')):
         header = None
     handled = False
     # unary functions
diff --git a/pypy/module/cpyext/src/complexobject.c b/pypy/module/cpyext/src/complexobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/complexobject.c
@@ -0,0 +1,16 @@
+
+#include "Python.h"
+
+Py_complex
+PyComplex_AsCComplex(PyObject *obj)
+{
+    Py_complex result;
+    _PyComplex_AsCComplex(obj, &result);
+    return result;
+}
+
+PyObject *
+PyComplex_FromCComplex(Py_complex c)
+{
+    return _PyComplex_FromCComplex(&c);
+}
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -2,7 +2,6 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import executioncontext
-from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
@@ -14,9 +13,7 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
-        if space.config.translation.gc != "boehm":
-            pyobj_dealloc_action = PyObjDeallocAction(space)
-            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+        self.builder = None
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -57,22 +54,40 @@
                         "Function returned an error result without setting an "
                         "exception")
 
-    def build_api(self, space):
+    def setup_rawrefcount(self):
+        space = self.space
+        if not self.space.config.translating:
+            def dealloc_trigger():
+                from pypy.module.cpyext.pyobject import PyObject, decref
+                print 'dealloc_trigger...'
+                while True:
+                    ob = rawrefcount.next_dead(PyObject)
+                    if not ob:
+                        break
+                    print 'deallocating PyObject', ob
+                    decref(space, ob)
+                print 'dealloc_trigger DONE'
+                return "RETRY"
+            rawrefcount.init(dealloc_trigger)
+        else:
+            if space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(space)
+                space.actionflag.register_periodic_action(action,
+                    use_bytecode_counter=True)
+            else:
+                pyobj_dealloc_action = PyObjDeallocAction(space)
+                self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+
+    def build_api(self):
         """NOT_RPYTHON
         This function is called when at object space creation,
         and drives the compilation of the cpyext library
         """
         from pypy.module.cpyext import api
-        state = self.space.fromcache(State)
         if not self.space.config.translating:
-            state.api_lib = str(api.build_bridge(self.space))
+            self.api_lib = str(api.build_bridge(self.space))
         else:
             api.setup_library(self.space)
-            #
-            if self.space.config.translation.gc == "boehm":
-                action = BoehmPyObjDeallocAction(self.space)
-                self.space.actionflag.register_periodic_action(action,
-                    use_bytecode_counter=True)
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -84,17 +99,17 @@
 
     def startup(self, space):
         "This function is called when the program really starts"
-
         from pypy.module.cpyext.typeobject import setup_new_method_def
         from pypy.module.cpyext.api import INIT_FUNCTIONS
-        from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
             if space.config.translation.gc != "boehm":
+                # This must be called in RPython, the untranslated version
+                # does something different. Sigh.
                 rawrefcount.init(
                     llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
                     self.dealloc_trigger))
-            init_static_data_translated(space)
+            self.builder.attach_all(space)
 
         setup_new_method_def(space)
 
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,9 +89,9 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest1'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
                 == ('Py_ssize_t', 'Py_ssize_t arg0'))
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest2'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
                 == ('Py_ssize_t *', 'Py_ssize_t *arg0'))
 
         PyPy_TypedefTest1(space, 0)
@@ -100,7 +100,7 @@
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
 
- at pytest.mark.skipif(os.environ.get('USER')=='root', 
+ at pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
     api.copy_header_files(tmpdir, True)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -27,8 +27,9 @@
 
 class TestApi:
     def test_signature(self):
-        assert 'PyModule_Check' in api.FUNCTIONS
-        assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
+        common_functions = api.FUNCTIONS_BY_HEADER[api.pypy_decl]
+        assert 'PyModule_Check' in common_functions
+        assert common_functions['PyModule_Check'].argtypes == [api.PyObject]
 
 
 class SpaceCompiler(SystemCompilationInfo):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -87,7 +87,7 @@
     py_getsetdef.c_set = rffi.cast(setter, 0)
     py_getsetdef.c_closure = rffi.cast(rffi.VOIDP, 0)
     return py_getsetdef
-    
+
 
 class W_MemberDescr(GetSetProperty):
     name = 'member_descriptor'
@@ -711,7 +711,7 @@
 
     pto.c_tp_free = llslot(space, PyObject_Free)
     pto.c_tp_alloc = llslot(space, PyType_GenericAlloc)
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder
     if ((pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE) != 0
             and builder.cpyext_type_init is None):
             # this ^^^ is not None only during startup of cpyext.  At that

From pypy.commits at gmail.com  Thu Jan  5 17:50:54 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 05 Jan 2017 14:50:54 -0800 (PST)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <586ecdce.6a7ac20a.609ac.f521@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89386:eb44ca04031a
Date: 2017-01-05 22:50 +0000
http://bitbucket.org/pypy/pypy/changeset/eb44ca04031a/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -87,3 +87,7 @@
 Implement StringBuffer.get_raw_address (missing feature for the buffer protocol).
 More generally it is now possible to obtain the address of any object (if it
 is readonly) without pinning it.
+
+.. branch: cpyext-cleanup
+
+Refactor cpyext initialisation.

From pypy.commits at gmail.com  Fri Jan  6 06:01:13 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 06 Jan 2017 03:01:13 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-from: extend the new Py_bufferBuffer
 object to expose every field from Py_buffer
Message-ID: <586f78f9.e6b0c20a.5560f.dac9@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: cpyext-from
Changeset: r89387:09772a8274ba
Date: 2017-01-06 12:00 +0100
http://bitbucket.org/pypy/pypy/changeset/09772a8274ba/

Log:	extend the new Py_bufferBuffer object to expose every field from
	Py_buffer

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -208,25 +208,53 @@
     py_mview = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_mview))
 
     view = py_mview.c_view
-    fill_Py_buffer(space, buf, view)
+    #_dup_Py_buffer(space, view, buf)
     view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
     # the docs say that in PyMemoryView_FromBuffer (and thus FromObject)
     # this object must be NULL
     view.c_obj = make_ref(space, w_obj)
 
     # XXX what about w_mview.base = w_obj (see cpython 2.7 implementation)
-    return py_mview
+    return w_mview
+
+def _dup_Py_buffer(space, dest, src):
+    if src.c_ndim == 1 and src.c_shape != 0:
+        dest.c_shape = dest.smalltable[0]
+        dest.c_shape[0] = src.smalltable[0]
+    if src.c_ndim == 1 and src.c_strides != 0:
+        dest.strides = dest.smalltable[1]
+        dest.strides[0] = src.strides[0]
 
 class Py_bufferBuffer(Buffer):
-    _attrs_ = ['readonly', 'view']
+    _attrs_ = ['view']
     _immutable_ = True
 
     def __init__(self, view):
         self.view = view
-        self.readonly = view.c_readonly
+
+    @property
+    def readonly(self):
+        return rffi.cast(lltype.Signed, self.view.c_readonly)
+
+    def getformat(self):
+        return self.view.c_format[0]
 
     def getlength(self):
-        return self.view.c_len
+        return rffi.cast(lltype.Signed, self.view.c_len)
+
+    def getndim(self):
+        return rffi.cast(lltype.Signed, self.view.c_ndim)
+
+    def getshape(self):
+        shape = self.view.c_shape
+        return [shape[i] for i in range(self.getndim())]
+
+    def getstrides(self):
+        strides = self.view.c_strides
+        return [strides[i] for i in range(self.getndim())]
+
+    def getitemsize(self):
+        return rffi.cast(lltype.Signed, self.view.c_itemsize)
 
     def get_raw_address(self):
         return self.view.c_buf
@@ -237,9 +265,7 @@
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
-    w_mview = W_MemoryView(Py_bufferBuffer(view))
-    py_mview = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_mview))
-    return py_mview
+    return W_MemoryView(Py_bufferBuffer(view))
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -20,9 +20,8 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        py_memoryview = api.PyMemoryView_FromObject(w_buf)
-        w_memoryview = from_ref(space, py_memoryview)
-        view = api.PyMemoryView_GET_BUFFER(py_memoryview)
+        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        view = api.PyMemoryView_GET_BUFFER(w_memoryview)
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)
         assert f == 'B'
@@ -31,7 +30,7 @@
         assert view.c_len == 5
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
-        w_mv = from_ref(space, api.PyMemoryView_FromBuffer(view))
+        w_mv = api.PyMemoryView_FromBuffer(view)
         for f in ('format', 'itemsize', 'ndim', 'readonly', 
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)

From pypy.commits at gmail.com  Fri Jan  6 07:07:22 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 06 Jan 2017 04:07:22 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <586f887a.12921c0a.5fd96.150a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89388:ed458f15fed3
Date: 2017-01-05 07:19 +0200
http://bitbucket.org/pypy/pypy/changeset/ed458f15fed3/

Log:	merge default into branch

diff too long, truncating to 2000 out of 2328 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -32,7 +32,7 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises((TypeError, ValueError),
+        self.assertRaises(TypeError,
                           (c_char * 16).from_buffer, "a" * 16)
 
     def test_fom_buffer_with_offset(self):
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -85,7 +85,17 @@
 
     def from_buffer(self, obj, offset=0):
         size = self._sizeofinstances()
+        if isinstance(obj, (str, unicode)):
+            # hack, buffer(str) will always return a readonly buffer.
+            # CPython calls PyObject_AsWriteBuffer(...) here!
+            # str cannot be modified, thus raise a type error in this case
+            raise TypeError("Cannot use %s as modifiable buffer" % str(type(obj)))
+
+        # why not just call memoryview(obj)[offset:]?
+        # array in Python 2.7 does not support the buffer protocol and will
+        # fail, even though buffer is supported
         buf = buffer(obj, offset, size)
+
         if len(buf) < size:
             raise ValueError(
                 "Buffer size too small (%d instead of at least %d bytes)"
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -428,11 +428,6 @@
   ``datetime.date`` is the superclass of ``datetime.datetime``).
   Anyway, the proper fix is arguably to use a regular method call in
   the first place: ``datetime.date.today().strftime(...)``
-
-* the ``__dict__`` attribute of new-style classes returns a normal dict, as
-  opposed to a dict proxy like in CPython. Mutating the dict will change the
-  type and vice versa. For builtin types, a dictionary will be returned that
-  cannot be changed (but still looks and behaves like a normal dictionary).
   
 * some functions and attributes of the ``gc`` module behave in a
   slightly different way: for example, ``gc.enable`` and
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -76,3 +76,14 @@
 PyMemoryViewObject with a PyBuffer attached so that the call to 
 ``PyMemoryView_GET_BUFFER`` does not leak a PyBuffer-sized piece of memory.
 Properly call ``bf_releasebuffer`` when not ``NULL``.
+
+.. branch: boehm-rawrefcount
+
+Support translations of cpyext with the Boehm GC (for special cases like
+revdb).
+
+.. branch: strbuf-as-buffer
+
+Implement StringBuffer.get_raw_address (missing feature for the buffer protocol).
+More generally it is now possible to obtain the address of any object (if it
+is readonly) without pinning it.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -305,9 +305,9 @@
             config.objspace.lonepycfiles = False
 
         if config.objspace.usemodules.cpyext:
-            if config.translation.gc != 'incminimark':
+            if config.translation.gc not in ('incminimark', 'boehm'):
                 raise Exception("The 'cpyext' module requires the 'incminimark'"
-                                " GC.  You need either 'targetpypystandalone.py"
+                                " 'boehm' GC.  You need either 'targetpypystandalone.py"
                                 " --withoutmod-cpyext' or '--gc=incminimark'")
 
         config.translating = True
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -4,6 +4,7 @@
 
 from rpython.rlib.buffer import Buffer
 from pypy.interpreter.gateway import unwrap_spec
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
 
 
 class ByteBuffer(Buffer):
@@ -22,6 +23,8 @@
     def setitem(self, index, char):
         self.data[index] = char
 
+    def get_raw_address(self):
+        return nonmoving_raw_ptr_for_resizable_list(self.data)
 
 @unwrap_spec(length=int)
 def bytebuffer(space, length):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -136,6 +136,9 @@
     return _from_buffer(space, w_ctype, w_x)
 
 def _from_buffer(space, w_ctype, w_x):
+    if space.isinstance_w(w_x, space.w_unicode):
+        raise oefmt(space.w_TypeError,
+                        "from_buffer() cannot return the address a unicode")
     buf = _fetch_as_read_buffer(space, w_x)
     if space.isinstance_w(w_x, space.w_str):
         _cdata = get_raw_address_of_string(space, w_x)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3419,24 +3419,28 @@
     BCharA = new_array_type(BCharP, None)
     p1 = from_buffer(BCharA, b"foo")
     assert p1 == from_buffer(BCharA, b"foo")
-    import gc; gc.collect()
-    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
-        # from_buffer(buffer(b"foo")) does not work, because it's not
-        # implemented on pypy; only from_buffer(b"foo") works.
-        py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
-        py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
+        contents = from_buffer(BCharA, buffer(b"foo"))
+        for i in range(len(contents)):
+            assert contents[i] == p1[i]
+        p4 = from_buffer(BCharA, b"f\x00\x00\x00o\x00\x00\x00o\x00\x00\x00")
+        contents = from_buffer(BCharA, buffer(u+"foo"))
+        for i in range(len(contents)):
+            assert contents[i] == p4[i]
     try:
         from __builtin__ import memoryview
     except ImportError:
         pass
     else:
-        py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo"))
+        contents = from_buffer(BCharA, memoryview(b"foo"))
+        for i in range(len(contents)):
+            assert contents[i] == p1[i]
+
 
 def test_from_buffer_bytearray():
     a = bytearray(b"xyz")
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import (
     TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rarithmetic import r_longlong, intmask
@@ -120,6 +121,9 @@
     def setitem(self, index, char):
         self.buf[self.start + index] = char
 
+    def get_raw_address(self):
+        return nonmoving_raw_ptr_for_resizable_list(self.buf)
+
 class BufferedMixin:
     _mixin_ = True
 
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -1,6 +1,6 @@
 from rpython.rlib import rsocket
 from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, r_longlong, r_uint32
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -98,7 +98,8 @@
         proto = space.str_w(w_proto)
 
     if port < 0 or port > 0xffff:
-        raise oefmt(space.w_ValueError, "getservbyport: port must be 0-65535.")
+        raise oefmt(space.w_OverflowError,
+                    "getservbyport: port must be 0-65535.")
 
     try:
         service = rsocket.getservbyport(port, proto)
@@ -163,40 +164,58 @@
         space.wrap(W_Socket(space, sock2))
     ])
 
-# The following 4 functions refuse all negative numbers, like CPython 2.6.
-# They could also check that the argument is not too large, but CPython 2.6
-# is not doing that consistently.
- at unwrap_spec(x="c_uint")
+# The following 4 functions refuse all negative numbers.
+# They also check that the argument is not too large, but note that
+# CPython 2.7 is not doing that consistently (CPython 3.x does).
+LONGLONG_UINT32_MAX = r_longlong(2**32-1)
+
+ at unwrap_spec(x="c_int")
 def ntohs(space, x):
     """ntohs(integer) -> integer
 
     Convert a 16-bit integer from network to host byte order.
     """
+    if x < 0:
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
     return space.wrap(rsocket.ntohs(intmask(x)))
 
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x=r_longlong)
 def ntohl(space, x):
     """ntohl(integer) -> integer
 
     Convert a 32-bit integer from network to host byte order.
     """
-    return space.wrap(rsocket.ntohl(x))
+    if x < r_longlong(0):
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
+    if x > LONGLONG_UINT32_MAX:
+        raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
+    return space.wrap(rsocket.ntohl(r_uint32(x)))
 
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x="c_int")
 def htons(space, x):
     """htons(integer) -> integer
 
     Convert a 16-bit integer from host to network byte order.
     """
-    return space.wrap(rsocket.htons(intmask(x)))
+    if x < 0:
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
+    return space.wrap(rsocket.htons(x))
 
- at unwrap_spec(x="c_uint")
+ at unwrap_spec(x=r_longlong)
 def htonl(space, x):
     """htonl(integer) -> integer
 
     Convert a 32-bit integer from host to network byte order.
     """
-    return space.wrap(rsocket.htonl(x))
+    if x < r_longlong(0):
+        raise oefmt(space.w_OverflowError,
+                    "can't convert negative number to unsigned long")
+    if x > LONGLONG_UINT32_MAX:
+        raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
+    return space.wrap(rsocket.htonl(r_uint32(x)))
 
 @unwrap_spec(ip=str)
 def inet_aton(space, ip):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -424,7 +424,7 @@
             w_addr = w_param3
         try:
             addr = self.addr_from_object(space, w_addr)
-            count = self.sock.sendto(data, flags, addr)
+            count = self.sock.sendto(data, len(data), flags, addr)
         except SocketError as e:
             raise converted_error(space, e)
         return space.wrap(count)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -83,11 +83,6 @@
                          "(_socket, port): return _socket.getservbyport(port)")
     assert space.unwrap(name) == "smtp"
 
-    from pypy.interpreter.error import OperationError
-    exc = raises(OperationError, space.appexec,
-           [w_socket], "(_socket): return _socket.getservbyport(-1)")
-    assert exc.value.match(space, space.w_ValueError)
-
 def test_getprotobyname():
     name = "tcp"
     w_n = space.appexec([w_socket, space.wrap(name)],
@@ -325,6 +320,11 @@
         assert _socket.socket.__name__ == 'socket'
         assert _socket.socket.__module__ == '_socket'
 
+    def test_overflow_errors(self):
+        import _socket
+        raises(OverflowError, _socket.getservbyport, -1)
+        raises(OverflowError, _socket.getservbyport, 65536)
+
     def test_ntoa_exception(self):
         import _socket
         raises(_socket.error, _socket.inet_ntoa, b"ab")
@@ -495,7 +495,8 @@
     def test_socket_connect_typeerrors(self):
         tests = [
             "",
-            ("80"),
+            "80",
+            ("80",),
             ("80", "80"),
             (80, 80),
         ]
@@ -519,44 +520,25 @@
     def test_NtoH(self):
         import sys
         import _socket as socket
-        # This just checks that htons etc. are their own inverse,
-        # when looking at the lower 16 or 32 bits.
+        # This checks that htons etc. are their own inverse,
+        # when looking at the lower 16 or 32 bits.  It also
+        # checks that we get OverflowErrors when calling with -1,
+        # or (for XtoXl()) with too large values.  For XtoXs()
+        # large values are silently truncated instead, like CPython.
         sizes = {socket.htonl: 32, socket.ntohl: 32,
                  socket.htons: 16, socket.ntohs: 16}
         for func, size in sizes.items():
             mask = (1 << size) - 1
-            for i in (0, 1, 0xffff, ~0xffff, 2, 0x01234567, 0x76543210):
+            for i in (0, 1, 0xffff, 0xffff0000, 2, 0x01234567, 0x76543210):
                 assert i & mask == func(func(i&mask)) & mask
 
             swapped = func(mask)
             assert swapped & mask == mask
-            try:
-                func(-1)
-            except (OverflowError, ValueError):
-                pass
-            else:
-                assert False
-            try:
-                func(sys.maxint*2+2)
-            except OverflowError:
-                pass
-            else:
-                assert False
-
-    def test_NtoH_overflow(self):
-        skip("we are not checking for overflowing values yet")
-        import _socket as socket
-        # Checks that we cannot give too large values to htons etc.
-        # Skipped for now; CPython 2.6 is also not consistent.
-        sizes = {socket.htonl: 32, socket.ntohl: 32,
-                 socket.htons: 16, socket.ntohs: 16}
-        for func, size in sizes.items():
-            try:
-                func(1 << size)
-            except OverflowError:
-                pass
-            else:
-                assert False
+            raises(OverflowError, func, -1)
+            raises(OverflowError, func, -1L)
+            if size > 16:    # else, values too large are ignored
+                raises(OverflowError, func, 2 ** size)
+                raises(OverflowError, func, 2L ** size)
 
     def test_newsocket(self):
         import socket
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -2,12 +2,6 @@
 
 @py.test.mark.tryfirst
 def pytest_runtest_setup(item):
-    if 'linux' in sys.platform:
-        # tests require minimally std=c++11
-        cc_info = py.process.cmdexec('gcc -v --help')
-        if not '-std=c++11' in cc_info:
-            py.test.skip('skipping tests because gcc does not support C++11')
-
     if py.path.local.sysfind('genreflex') is None:
         import pypy.module.cppyy.capi.loadable_capi as lcapi
         if 'dummy' in lcapi.reflection_library:
diff --git a/pypy/module/cppyy/test/support.py b/pypy/module/cppyy/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cppyy/test/support.py
@@ -0,0 +1,16 @@
+import py, sys, subprocess
+
+currpath = py.path.local(__file__).dirpath()
+
+
+def setup_make(targetname):
+    if sys.platform == 'win32':
+        py.test.skip("win32 not supported so far")
+    import pypy.module.cppyy.capi.loadable_capi as lcapi
+    popen = subprocess.Popen(["make", targetname], cwd=str(currpath),
+                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    stdout, _ = popen.communicate()
+    if popen.returncode:
+        if '-std=c++11' in stdout:
+            py.test.skip("gcc does not seem to support -std=c++11")
+        raise OSError("'make' failed:\n%s" % (stdout,))
diff --git a/pypy/module/cppyy/test/test_cppyy.py b/pypy/module/cppyy/test/test_cppyy.py
--- a/pypy/module/cppyy/test/test_cppyy.py
+++ b/pypy/module/cppyy/test/test_cppyy.py
@@ -1,18 +1,15 @@
 import py, os, sys
+import subprocess
 
 from pypy.module.cppyy import interp_cppyy, executor
+from .support import setup_make
 
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict.so"))
 
 def setup_module(mod):
-    if sys.platform == 'win32':
-        py.test.skip("win32 not supported so far")
-    import pypy.module.cppyy.capi.loadable_capi as lcapi
-    err = os.system("cd '%s' && make example01Dict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    setup_make("example01Dict.so")
 
 class TestCPPYYImplementation:
     def test01_class_query(self, space):
diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py
--- a/pypy/module/cppyy/test/test_datatypes.py
+++ b/pypy/module/cppyy/test/test_datatypes.py
@@ -1,15 +1,12 @@
 import py, os, sys
+from .support import setup_make
 
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("datatypesDict.so"))
 
 def setup_module(mod):
-    if sys.platform == 'win32':
-        py.test.skip("win32 not supported so far")
-    err = os.system("cd '%s' && make datatypesDict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    setup_make("datatypesDict.so")
 
 class AppTestDATATYPES:
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py
--- a/pypy/module/cppyy/test/test_pythonify.py
+++ b/pypy/module/cppyy/test/test_pythonify.py
@@ -1,17 +1,14 @@
 import py, os, sys
 
 from pypy.module.cppyy import interp_cppyy, executor
+from .support import setup_make
 
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict.so"))
 
 def setup_module(mod):
-    if sys.platform == 'win32':
-        py.test.skip("win32 not supported so far")
-    err = os.system("cd '%s' && make example01Dict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    setup_make("example01Dict.so")
 
 class AppTestPYTHONIFY:
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1,6 +1,5 @@
 import ctypes
 import sys, os
-import atexit
 
 import py
 
@@ -18,7 +17,6 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import NameManager
 from rpython.tool.udir import udir
-from rpython.translator import platform
 from pypy.module.cpyext.state import State
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.baseobjspace import W_Root
@@ -485,6 +483,12 @@
     TYPES[configname] = forward
     return forward
 
+GLOBALS = {}
+def register_global(name, typ, expr, header=None):
+    if header is not None:
+        name = '%s#%s' % (name, header)
+    GLOBALS[name] = (typ, expr)
+
 INTERPLEVEL_API = {}
 FUNCTIONS = {}
 FUNCTIONS_BY_HEADER = {}
@@ -545,18 +549,23 @@
     '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext',
 ]
 TYPES = {}
-GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur
-    '_Py_NoneStruct#%s' % pypy_decl: ('PyObject*', 'space.w_None'),
-    '_Py_TrueStruct#%s' % pypy_decl: ('PyIntObject*', 'space.w_True'),
-    '_Py_ZeroStruct#%s' % pypy_decl: ('PyIntObject*', 'space.w_False'),
-    '_Py_NotImplementedStruct#%s' % pypy_decl: ('PyObject*', 'space.w_NotImplemented'),
-    '_Py_EllipsisObject#%s' % pypy_decl: ('PyObject*', 'space.w_Ellipsis'),
-    'PyDateTimeAPI': ('PyDateTime_CAPI*', 'None'),
-    }
 FORWARD_DECLS = []
 INIT_FUNCTIONS = []
 BOOTSTRAP_FUNCTIONS = []
 
+# this needs to include all prebuilt pto, otherwise segfaults occur
+register_global('_Py_NoneStruct',
+    'PyObject*', 'space.w_None', header=pypy_decl)
+register_global('_Py_TrueStruct',
+    'PyIntObject*', 'space.w_True', header=pypy_decl)
+register_global('_Py_ZeroStruct',
+    'PyIntObject*', 'space.w_False', header=pypy_decl)
+register_global('_Py_NotImplementedStruct',
+    'PyObject*', 'space.w_NotImplemented', header=pypy_decl)
+register_global('_Py_EllipsisObject',
+    'PyObject*', 'space.w_Ellipsis', header=pypy_decl)
+register_global('PyDateTimeAPI', 'PyDateTime_CAPI*', 'None')
+
 def build_exported_objects():
     # Standard exceptions
     # PyExc_BaseException, PyExc_Exception, PyExc_ValueError, PyExc_KeyError,
@@ -565,7 +574,7 @@
     # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError,
     # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ...
     for exc_name in exceptions.Module.interpleveldefs.keys():
-        GLOBALS['PyExc_' + exc_name] = (
+        register_global('PyExc_' + exc_name,
             'PyTypeObject*',
             'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, ))
 
@@ -600,7 +609,7 @@
         'PyCFunction_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)',
         'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)'
         }.items():
-        GLOBALS['%s#%s' % (cpyname, pypy_decl)] = ('PyTypeObject*', pypyexpr)
+        register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
 
     for cpyname in '''PyMethodObject PyListObject PyLongObject
                       PyClassObject'''.split():
@@ -1021,14 +1030,12 @@
 def build_bridge(space):
     "NOT_RPYTHON"
     from pypy.module.cpyext.pyobject import make_ref
+    from rpython.translator.c.database import LowLevelDatabase
+    use_micronumpy = setup_micronumpy(space)
+    db = LowLevelDatabase()
+    prefix ='cpyexttest'
 
-    use_micronumpy = setup_micronumpy(space)
-
-    export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS)
-    from rpython.translator.c.database import LowLevelDatabase
-    db = LowLevelDatabase()
-
-    generate_macros(export_symbols, prefix='cpyexttest')
+    functions = generate_decls_and_callbacks(db, prefix=prefix)
 
     # Structure declaration code
     members = []
@@ -1049,9 +1056,6 @@
     RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
-    functions = generate_decls_and_callbacks(db, export_symbols,
-                                            prefix='cpyexttest')
-
     global_objects = []
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
@@ -1078,7 +1082,7 @@
             '\n' +
             '\n'.join(functions))
 
-    eci = build_eci(True, export_symbols, code, use_micronumpy)
+    eci = build_eci(True, code, use_micronumpy)
     eci = eci.compile_shared_lib(
         outputfilename=str(udir / "module_cache" / "pypyapi"))
     modulename = py.path.local(eci.libraries[-1])
@@ -1099,7 +1103,6 @@
     run_bootstrap_functions(space)
 
     # load the bridge, and init structure
-    import ctypes
     bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
 
     space.fromcache(State).install_dll(eci)
@@ -1119,7 +1122,7 @@
 
         INTERPLEVEL_API[name] = w_obj
 
-        name = name.replace('Py', 'cpyexttest')
+        name = name.replace('Py', prefix)
         if isptr:
             ptr = ctypes.c_void_p.in_dll(bridge, name)
             if typ == 'PyObject*':
@@ -1147,12 +1150,6 @@
     pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
 
     # implement structure initialization code
-    #for name, func in FUNCTIONS.iteritems():
-    #    if name.startswith('cpyext_'): # XXX hack
-    #        continue
-    #    pypyAPI[structindex[name]] = ctypes.cast(
-    #        ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
-    #        ctypes.c_void_p)
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
             if name.startswith('cpyext_') or func is None: # XXX hack
@@ -1242,13 +1239,13 @@
     else:
         return None
 
-def generate_macros(export_symbols, prefix):
+def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
-    renamed_symbols = []
+    export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
     for name in export_symbols:
         if '#' in name:
-            name,header = name.split('#')
+            name, header = name.split('#')
         else:
             header = pypy_decl
         newname = mangle_name(prefix, name)
@@ -1257,8 +1254,6 @@
             pypy_macros.append('#define %s %s' % (name, newname))
         if name.startswith("PyExc_"):
             pypy_macros.append('#define _%s _%s' % (name, newname))
-        renamed_symbols.append(newname)
-    export_symbols[:] = renamed_symbols
 
     # Generate defines
     for macro_name, size in [
@@ -1278,8 +1273,6 @@
     pypy_macros_h = udir.join('pypy_macros.h')
     pypy_macros_h.write('\n'.join(pypy_macros))
 
-def generate_decls_and_callbacks(db, export_symbols, api_struct=True, prefix=''):
-    "NOT_RPYTHON"
     # implement function callbacks and generate function decls
     functions = []
     decls = {}
@@ -1365,7 +1358,7 @@
                          source_dir / "pymem.c",
                          ]
 
-def build_eci(building_bridge, export_symbols, code, use_micronumpy=False):
+def build_eci(building_bridge, code, use_micronumpy=False):
     "NOT_RPYTHON"
     # Build code and get pointer to the structure
     kwds = {}
@@ -1434,31 +1427,29 @@
         return use_micronumpy
     # import registers api functions by side-effect, we also need HEADER
     from pypy.module.cpyext.ndarrayobject import HEADER
-    global GLOBALS, FUNCTIONS_BY_HEADER, separate_module_files
+    global FUNCTIONS_BY_HEADER, separate_module_files
     for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
         FUNCTIONS_BY_HEADER.setdefault(HEADER, {})[func_name] = None
-    GLOBALS["PyArray_Type#%s" % HEADER] = ('PyTypeObject*', "space.gettypeobject(W_NDimArray.typedef)")
+    register_global("PyArray_Type",
+        'PyTypeObject*',  "space.gettypeobject(W_NDimArray.typedef)",
+        header=HEADER)
     separate_module_files.append(source_dir / "ndarrayobject.c")
     return use_micronumpy
 
 def setup_library(space):
     "NOT_RPYTHON"
+    from rpython.translator.c.database import LowLevelDatabase
     use_micronumpy = setup_micronumpy(space)
-    export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
-    from rpython.translator.c.database import LowLevelDatabase
     db = LowLevelDatabase()
     prefix = 'PyPy'
 
-    generate_macros(export_symbols, prefix=prefix)
-
-    functions = generate_decls_and_callbacks(db, [], api_struct=False,
-                                            prefix=prefix)
+    functions = generate_decls_and_callbacks(db, api_struct=False, prefix=prefix)
     code = "#include <Python.h>\n"
     if use_micronumpy:
         code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
     code  += "\n".join(functions)
 
-    eci = build_eci(False, export_symbols, code, use_micronumpy)
+    eci = build_eci(False, code, use_micronumpy)
 
     space.fromcache(State).install_dll(eci)
 
@@ -1610,7 +1601,7 @@
 
 @specialize.memo()
 def make_generic_cpy_call(FT, expect_null):
-    from pypy.module.cpyext.pyobject import make_ref, from_ref, Py_DecRef
+    from pypy.module.cpyext.pyobject import make_ref, from_ref
     from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
     from pypy.module.cpyext.pyobject import get_w_obj_and_decref
     from pypy.module.cpyext.pyerrors import PyErr_Occurred
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -1,7 +1,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.executioncontext import AsyncAction
+from pypy.interpreter import executioncontext
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
@@ -14,8 +14,9 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
-        pyobj_dealloc_action = PyObjDeallocAction(space)
-        self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+        if space.config.translation.gc != "boehm":
+            pyobj_dealloc_action = PyObjDeallocAction(space)
+            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -67,6 +68,11 @@
             state.api_lib = str(api.build_bridge(self.space))
         else:
             api.setup_library(self.space)
+            #
+            if self.space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(self.space)
+                self.space.actionflag.register_periodic_action(action,
+                    use_bytecode_counter=True)
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -84,8 +90,10 @@
         from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
-            rawrefcount.init(llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
-                                      self.dealloc_trigger))
+            if space.config.translation.gc != "boehm":
+                rawrefcount.init(
+                    llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
+                    self.dealloc_trigger))
             init_static_data_translated(space)
 
         setup_new_method_def(space)
@@ -143,15 +151,23 @@
         self.extensions[path] = w_copy
 
 
-class PyObjDeallocAction(AsyncAction):
+def _rawrefcount_perform(space):
+    from pypy.module.cpyext.pyobject import PyObject, decref
+    while True:
+        py_obj = rawrefcount.next_dead(PyObject)
+        if not py_obj:
+            break
+        decref(space, py_obj)
+
+class PyObjDeallocAction(executioncontext.AsyncAction):
     """An action that invokes _Py_Dealloc() on the dying PyObjects.
     """
+    def perform(self, executioncontext, frame):
+        _rawrefcount_perform(self.space)
 
+class BoehmPyObjDeallocAction(executioncontext.PeriodicAsyncAction):
+    # This variant is used with Boehm, which doesn't have the explicit
+    # callback.  Instead we must periodically check ourselves.
     def perform(self, executioncontext, frame):
-        from pypy.module.cpyext.pyobject import PyObject, decref
-
-        while True:
-            py_obj = rawrefcount.next_dead(PyObject)
-            if not py_obj:
-                break
-            decref(self.space, py_obj)
+        if we_are_translated():
+            _rawrefcount_perform(self.space)
diff --git a/pypy/module/cpyext/stubgen.py b/pypy/module/cpyext/stubgen.py
deleted file mode 100644
--- a/pypy/module/cpyext/stubgen.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# -*- coding: utf-8 -*-
-from os import path
-
-from pypy.module.cpyext import api
-
-from sphinx import addnodes
-
-
-TEMPLATE = """
- at cpython_api([%(paramtypes)s], %(rettype)s)
-def %(functionname)s(%(params)s):
-%(docstring)s    raise NotImplementedError
-    %(borrows)s
-"""
-
-C_TYPE_TO_PYPY_TYPE = {
-        "void": "lltype.Void",
-        "int": "rffi.INT_real",
-        "PyTypeObject*": "PyTypeObjectPtr",
-        "PyVarObject*": "PyObject",
-        "const char*": "rffi.CCHARP",
-        "double": "rffi.DOUBLE",
-        "PyObject*": "PyObject",
-        "PyObject**": "PyObjectP",
-        "char*": "rffi.CCHARP",
-        "PyMethodDef*": "PyMethodDef",
-        "Py_ssize_t": "Py_ssize_t",
-        "Py_ssize_t*": "Py_ssize_t",
-        "size_t": "rffi.SIZE_T",
-        "...": "...",
-        "char": "lltype.Char",
-        "long": "lltype.Signed",
-        "Py_buffer*": "Py_buffer",
-        "": "",
-        }
-
-C_TYPE_TO_PYPY_TYPE_ARGS = C_TYPE_TO_PYPY_TYPE.copy()
-C_TYPE_TO_PYPY_TYPE_ARGS.update({
-    "void": "rffi.VOIDP",
-    })
-
-
-def c_param_to_type_and_name(string, is_arg=True):
-    string = string.replace(" **", "** ").replace(" *", "* ")
-    try:
-        typ, name = string.rsplit(" ", 1)
-    except ValueError:
-        typ = string
-        name = ""
-    return [C_TYPE_TO_PYPY_TYPE, C_TYPE_TO_PYPY_TYPE_ARGS][is_arg]\
-            .get(typ, "{" + typ + "}"), name
-
-
-def process_doctree(app, doctree):
-    for node in doctree.traverse(addnodes.desc_content):
-        par = node.parent
-        if par['desctype'] != 'cfunction':
-            continue
-        if not par[0].has_key('names') or not par[0]['names']:
-            continue
-        functionname = par[0]['names'][0]
-        if (functionname in api.FUNCTIONS or
-            functionname in api.SYMBOLS_C):
-            print "Wow, you implemented already", functionname
-            continue
-        borrows = docstring = ""
-        crettype, _, cparameters = par[0]
-        crettype = crettype.astext()
-        cparameters = cparameters.astext()
-        rettype, _ = c_param_to_type_and_name(crettype, False)
-        params = ["space"]
-        paramtypes = []
-        for param in cparameters.split(","):
-            typ, name = c_param_to_type_and_name(param.strip())
-            params.append(name)
-            paramtypes.append(typ)
-        params = ", ".join(params)
-        paramtypes = ", ".join(paramtypes)
-        docstring = node.astext()
-        entry = app._refcounts.get(functionname)
-        if entry and entry.result_type in ("PyObject*", "PyVarObject*"):
-            if entry.result_refs is None:
-                docstring += "\nReturn value: always NULL."
-            else:
-                borrows = ("borrow_from()", "")[entry.result_refs]
-        docstring = "\n    ".join(docstring.splitlines())
-        if docstring:
-            docstring = '    """%s"""\n' % (docstring,)
-        code = TEMPLATE % locals()
-        app._stubgen_f.write(code)
-
-
-def init_apidump(app):
-    fname = path.join(path.dirname(api.__file__), "stubs.py")
-    app._stubgen_f = file(fname, "w")
-    app.connect('doctree-read', process_doctree)
-
-
-def setup(app):
-    app.connect('builder-inited', init_apidump)
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -36,6 +36,30 @@
             assert space.eq_w(space.getattr(w_mv, w_f), 
                               space.getattr(w_memoryview, w_f))
 
+
+class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
+    def test_fillWithObject(self):
+        module = self.import_extension('foo', [
+                ("fillinfo", "METH_VARARGS",
+                 """
+                 Py_buffer buf;
+                 PyObject *str = PyBytes_FromString("hello, world.");
+                 if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
+                                       0, 0)) {
+                     return NULL;
+                 }
+
+                 /* Get rid of our own reference to the object, but
+                  * the Py_buffer should still have a reference.
+                  */
+                 Py_DECREF(str);
+
+                 return PyMemoryView_FromBuffer(&buf);
+                 """)])
+        result = module.fillinfo()
+        assert b"hello, world." == result
+        del result
+
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol_app(self):
         import struct
@@ -62,7 +86,7 @@
                     return NULL;
                 vlen = view.len / view.itemsize;
                 PyBuffer_Release(&view);
-                return PyInt_FromLong(vlen);
+                return PyLong_FromLong(vlen);
              """),
             ("test_buffer", "METH_VARARGS",
              """
@@ -70,10 +94,10 @@
                 PyObject* obj = PyTuple_GetItem(args, 0);
                 PyObject* memoryview = PyMemoryView_FromObject(obj);
                 if (memoryview == NULL)
-                    return PyInt_FromLong(-1);
+                    return PyLong_FromLong(-1);
                 view = PyMemoryView_GET_BUFFER(memoryview);
                 Py_DECREF(memoryview);
-                return PyInt_FromLong(view->len / view->itemsize);
+                return PyLong_FromLong(view->len / view->itemsize);
             """)])
         module = self.import_module(name='buffer_test')
         arr = module.PyMyArray(10)
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -17,9 +17,6 @@
         assert isinstance(buf, Buffer)
         self.buf = buf
 
-    def _finalize_(self):
-        return self.buf.releasebuffer()
-
     def buffer_w(self, space, flags):
         space.check_buf_flags(flags, self.buf.readonly)
         return self.buf
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -199,7 +199,9 @@
         raises(TypeError, "buf[MyInt(0):MyInt(5)]")
 
     def test_pypy_raw_address_base(self):
-        raises(ValueError, buffer("foobar")._pypy_raw_address)
-        raises(ValueError, buffer(u"foobar")._pypy_raw_address)
-        a = buffer(bytearray("foobar"))._pypy_raw_address()
+        a = buffer("foobar")._pypy_raw_address()
         assert a != 0
+        b = buffer(u"foobar")._pypy_raw_address()
+        assert b != 0
+        c = buffer(bytearray("foobar"))._pypy_raw_address()
+        assert c != 0
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -56,6 +56,7 @@
         assert u"abc" != memoryview("abc")
 
     def test_pypy_raw_address_base(self):
-        raises(ValueError, memoryview("foobar")._pypy_raw_address)
-        a = memoryview(bytearray("foobar"))._pypy_raw_address()
+        a = memoryview("foobar")._pypy_raw_address()
         assert a != 0
+        b = memoryview(bytearray("foobar"))._pypy_raw_address()
+        assert b != 0
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -398,6 +398,8 @@
         s_result = self.specialize(inputcells, op)
         if isinstance(s_result, FunctionGraph):
             s_result = s_result.getreturnvar().annotation
+            if s_result is None:
+                s_result = s_ImpossibleValue
         s_result = unionof(s_result, s_previous_result)
         return s_result
 
diff --git a/rpython/jit/codewriter/assembler.py b/rpython/jit/codewriter/assembler.py
--- a/rpython/jit/codewriter/assembler.py
+++ b/rpython/jit/codewriter/assembler.py
@@ -5,6 +5,7 @@
 from rpython.jit.codewriter.jitcode import SwitchDictDescr, JitCode
 from rpython.jit.codewriter import heaptracker, longlong
 from rpython.rlib.objectmodel import ComputedIntSymbolic
+from rpython.rlib.rarithmetic import r_int
 from rpython.flowspace.model import Constant
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper import rclass
@@ -82,6 +83,8 @@
             if not isinstance(value, (llmemory.AddressAsInt,
                                       ComputedIntSymbolic)):
                 value = lltype.cast_primitive(lltype.Signed, value)
+                if type(value) is r_int:
+                    value = int(value)
                 if allow_short:
                     try:
                         short_num = -128 <= value <= 127
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -1,6 +1,7 @@
 from rpython.jit.metainterp.history import AbstractDescr, ConstInt
 from rpython.jit.codewriter import heaptracker
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import base_int
 
 
 class JitCode(AbstractDescr):
@@ -21,6 +22,10 @@
               liveness=None, startpoints=None, alllabels=None,
               resulttypes=None):
         self.code = code
+        for x in constants_i:
+            assert not isinstance(x, base_int), (
+                "found constant %r of type %r, must not appear in "
+                "JitCode.constants_i" % (x, type(x)))
         # if the following lists are empty, use a single shared empty list
         self.constants_i = constants_i or self._empty_i
         self.constants_r = constants_r or self._empty_r
diff --git a/rpython/jit/codewriter/test/test_assembler.py b/rpython/jit/codewriter/test/test_assembler.py
--- a/rpython/jit/codewriter/test/test_assembler.py
+++ b/rpython/jit/codewriter/test/test_assembler.py
@@ -7,6 +7,7 @@
 from rpython.jit.metainterp.history import AbstractDescr
 from rpython.flowspace.model import Constant
 from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rlib.rarithmetic import r_int, r_uint
 
 
 def test_assemble_simple():
@@ -239,3 +240,17 @@
         ]
     assembler = Assembler()
     py.test.raises(AssemblerError, assembler.assemble, ssarepr)
+
+def test_assemble_r_int():
+    # r_int is a strange type, which the jit should replace with int.
+    # r_uint is also replaced with int.
+    ssarepr = SSARepr("test")
+    i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2)
+    ssarepr.insns = [
+        ('uint_add', i0, Constant(r_uint(42424242), lltype.Unsigned), '->', i1),
+        ('int_add', i0, Constant(r_int(42424243), lltype.Signed), '->', i2),
+        ]
+    assembler = Assembler()
+    jitcode = assembler.assemble(ssarepr)
+    assert jitcode.constants_i == [42424242, 42424243]
+    assert map(type, jitcode.constants_i) == [int, int]
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -6,6 +6,7 @@
 from rpython.jit.metainterp.history import MissingValue
 from rpython.rlib import longlong2float
 from rpython.rlib.debug import ll_assert, make_sure_not_resized
+from rpython.rlib.debug import check_annotation
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from rpython.rlib.unroll import unrolling_iterable
@@ -183,7 +184,7 @@
                 if lltype.typeOf(result) is lltype.Bool:
                     result = int(result)
                 assert lltype.typeOf(result) is lltype.Signed
-                self.registers_i[ord(code[position])] = result
+                self.registers_i[ord(code[position])] = plain_int(result)
                 position += 1
             elif resulttype == 'r':
                 # argcode should be 'r' too
@@ -213,7 +214,7 @@
                     if lltype.typeOf(result) is lltype.Bool:
                         result = int(result)
                     assert lltype.typeOf(result) is lltype.Signed
-                    self.registers_i[ord(code[position])] = result
+                    self.registers_i[ord(code[position])] = plain_int(result)
                     position += 1
             elif resulttype == 'L':
                 assert result >= 0
@@ -251,6 +252,23 @@
         if b < 0 or b >= LONG_BIT:
             raise ValueError("Shift count, %d,  not in valid range, 0 .. %d." % (b, LONG_BIT-1))
 
+def check_list_of_plain_integers(s_arg, bookkeeper):
+    """Check that 'BlackhopeInterpreter.registers_i' is annotated as a
+    non-resizable list of plain integers (and not r_int's for example)."""
+    from rpython.annotator import model as annmodel
+    assert isinstance(s_arg, annmodel.SomeList)
+    s_arg.listdef.never_resize()
+    assert s_arg.listdef.listitem.s_value.knowntype is int
+
+def _check_int(s_arg, bookkeeper):
+    assert s_arg.knowntype is int
+
+def plain_int(x):
+    """Check that 'x' is annotated as a plain integer (and not r_int)"""
+    check_annotation(x, _check_int)
+    return x
+
+
 class BlackholeInterpreter(object):
 
     def __init__(self, builder, count_interpreter):
@@ -277,6 +295,7 @@
         self.tmpreg_r = default_r
         self.tmpreg_f = default_f
         self.jitcode = None
+        check_annotation(self.registers_i, check_list_of_plain_integers)
 
     def __repr__(self):
         return '<BHInterp #%d>' % self.count_interpreter
@@ -295,7 +314,7 @@
 
     def setarg_i(self, index, value):
         assert lltype.typeOf(value) is lltype.Signed
-        self.registers_i[index] = value
+        self.registers_i[index] = plain_int(value)
 
     def setarg_r(self, index, value):
         assert lltype.typeOf(value) == llmemory.GCREF
@@ -1573,7 +1592,8 @@
     # 'xxx_call_yyy' instructions from the caller frame
     def _setup_return_value_i(self, result):
         assert lltype.typeOf(result) is lltype.Signed
-        self.registers_i[ord(self.jitcode.code[self.position-1])] = result
+        self.registers_i[ord(self.jitcode.code[self.position-1])] = plain_int(
+                                                                        result)
     def _setup_return_value_r(self, result):
         assert lltype.typeOf(result) == llmemory.GCREF
         self.registers_r[ord(self.jitcode.code[self.position-1])] = result
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -158,7 +158,11 @@
 # record that ignore_finalizer() has been called
 GCFLAG_IGNORE_FINALIZER = first_gcflag << 10
 
-_GCFLAG_FIRST_UNUSED = first_gcflag << 11    # the first unused bit
+# shadow objects can have its memory initialized when it is created.
+# It does not need an additional copy in trace out
+GCFLAG_SHADOW_INITIALIZED   = first_gcflag << 11
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 12    # the first unused bit
 
 
 # States for the incremental GC
@@ -729,6 +733,16 @@
         obj = self.external_malloc(typeid, length, alloc_young=True)
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
+    def move_out_of_nursery(self, obj):
+        # called twice, it should return the same shadow object,
+        # and not creating another shadow object
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW:
+            shadow = self.nursery_objects_shadows.get(obj)
+            ll_assert(shadow != llmemory.NULL,
+                      "GCFLAG_HAS_SHADOW but no shadow found")
+            return shadow
+
+        return self._allocate_shadow(obj, copy=True)
 
     def collect(self, gen=2):
         """Do a minor (gen=0), start a major (gen=1), or do a full
@@ -1982,6 +1996,9 @@
                 and self.young_rawmalloced_objects.contains(obj)):
                 self._visit_young_rawmalloced_object(obj)
             return
+        # copy the contents of the object? usually yes, but not for some
+        # shadow objects
+        copy = True
         #
         size_gc_header = self.gcheaderbuilder.size_gc_header
         if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
@@ -2037,13 +2054,18 @@
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
             # copied to the shadow itself.
             self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+            tid = self.header(obj).tid
+            if (tid & GCFLAG_SHADOW_INITIALIZED) != 0:
+                copy = False
+                self.header(obj).tid &= ~GCFLAG_SHADOW_INITIALIZED
             #
             totalsize = size_gc_header + self.get_size(obj)
             self.nursery_surviving_size += raw_malloc_usage(totalsize)
         #
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.
-        llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
+        if copy:
+            llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
         #
         # Set the old object's tid to -42 (containing all flags) and
         # replace the old object's content with the target address.
@@ -2570,7 +2592,8 @@
     # ----------
     # id() and identityhash() support
 
-    def _allocate_shadow(self, obj):
+    @specialize.arg(2)
+    def _allocate_shadow(self, obj, copy=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         size = self.get_size(obj)
         shadowhdr = self._malloc_out_of_nursery(size_gc_header +
@@ -2592,6 +2615,12 @@
         #
         self.header(obj).tid |= GCFLAG_HAS_SHADOW
         self.nursery_objects_shadows.setitem(obj, shadow)
+
+        if copy:
+            self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED
+            totalsize = size_gc_header + self.get_size(obj)
+            llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize)
+
         return shadow
 
     def _find_shadow(self, obj):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -551,6 +551,13 @@
                                               [s_gc, SomeAddress()],
                                               annmodel.s_None)
 
+        self.move_out_of_nursery_ptr = None
+        if hasattr(GCClass, 'move_out_of_nursery'):
+            self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
+                                              [s_gc, SomeAddress()],
+                                              SomeAddress())
+
+
     def create_custom_trace_funcs(self, gc, rtyper):
         custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
         rtyper.custom_trace_funcs = custom_trace_funcs
@@ -1585,6 +1592,17 @@
             hop.genop("direct_call", [self.ignore_finalizer_ptr,
                                       self.c_const_gc, v_adr])
 
+    def gct_gc_move_out_of_nursery(self, hop):
+        if self.move_out_of_nursery_ptr is not None:
+            v_adr = hop.genop("cast_ptr_to_adr", [hop.spaceop.args[0]],
+                              resulttype=llmemory.Address)
+            v_ret = hop.genop("direct_call", [self.move_out_of_nursery_ptr,
+                                      self.c_const_gc, v_adr],
+                                      resulttype=llmemory.Address)
+            hop.genop("cast_adr_to_ptr", [v_ret],
+                      resultvar = hop.spaceop.result)
+
+
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -2,6 +2,8 @@
 Buffer protocol support.
 """
 from rpython.rlib import jit
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+        nonmoving_raw_ptr_for_resizable_list)
 
 
 class Buffer(object):
@@ -84,7 +86,7 @@
 
     def __init__(self, value):
         self.value = value
-        self.readonly = True
+        self.readonly = 1
 
     def getlength(self):
         return len(self.value)
@@ -108,6 +110,9 @@
             return self.value[start:stop]
         return Buffer.getslice(self, start, stop, step, size)
 
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        return rffi.get_raw_address_of_string(self.value)
 
 class SubBuffer(Buffer):
     _attrs_ = ['buffer', 'offset', 'size', 'readonly']
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -4,10 +4,11 @@
 #  This is meant for pypy's cpyext module, but is a generally
 #  useful interface over our GC.  XXX "pypy" should be removed here
 #
-import sys, weakref
-from rpython.rtyper.lltypesystem import lltype, llmemory
+import sys, weakref, py
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.objectmodel import we_are_translated, specialize, not_rpython
 from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib import rgc
 
 
@@ -245,6 +246,11 @@
         v_p, v_ob = hop.inputargs(*hop.args_r)
         hop.exception_cannot_occur()
         hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)])
+        #
+        if hop.rtyper.annotator.translator.config.translation.gc == "boehm":
+            c_func = hop.inputconst(lltype.typeOf(func_boehm_eci),
+                                    func_boehm_eci)
+            hop.genop('direct_call', [c_func])
 
 
 class Entry(ExtRegistryEntry):
@@ -297,3 +303,10 @@
         v_ob = hop.genop('gc_rawrefcount_next_dead', [],
                          resulttype = llmemory.Address)
         return _spec_ob(hop, v_ob)
+
+src_dir = py.path.local(__file__).dirpath() / 'src'
+boehm_eci = ExternalCompilationInfo(
+    post_include_bits     = [(src_dir / 'boehm-rawrefcount.h').read()],
+    separate_module_files = [(src_dir / 'boehm-rawrefcount.c')],
+)
+func_boehm_eci = rffi.llexternal_use_eci(boehm_eci)
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -535,6 +535,25 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     llop.gc_ignore_finalizer(lltype.Void, obj)
 
+ at jit.dont_look_inside
+def move_out_of_nursery(obj):
+    """ Returns another object which is a copy of obj; but at any point
+        (either now or in the future) the returned object might suddenly
+        become identical to the one returned.
+
+        NOTE: Only use for immutable objects!
+    """
+    pass
+
+class MoveOutOfNurseryEntry(ExtRegistryEntry):
+    _about_ = move_out_of_nursery
+
+    def compute_result_annotation(self, s_obj):
+        return s_obj
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -997,12 +997,12 @@
                 if signal_checker is not None:
                     signal_checker()
 
-    def sendto(self, data, flags, address):
+    def sendto(self, data, length, flags, address):
         """Like send(data, flags) but allows specifying the destination
         address.  (Note that 'flags' is mandatory here.)"""
         self.wait_for_data(True)
         addr = address.lock()
-        res = _c.sendto(self.fd, data, len(data), flags,
+        res = _c.sendto(self.fd, data, length, flags,
                         addr, address.addrlen)
         address.unlock()
         if res < 0:
diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -0,0 +1,282 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <assert.h>
+#include <limits.h>
+#include <gc/gc.h>
+#include <gc/gc_mark.h>
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+#  define RPY_EXTERN  /* nothing */
+#else
+#  include "common_header.h"
+#endif
+
+
+#define REFCNT_FROM_PYPY  (LONG_MAX / 4 + 1)
+
+typedef struct pypy_header0 gcobj_t;    /* opaque here */
+
+#ifndef _WIN32
+typedef intptr_t Py_ssize_t;
+#else
+typedef long Py_ssize_t;
+#endif
+
+/* this is the first two words of the PyObject structure used in
+   pypy/module/cpyext */
+typedef struct {
+    Py_ssize_t ob_refcnt;
+    Py_ssize_t ob_pypy_link;
+} pyobj_t;
+
+struct link_s {
+    pyobj_t *pyobj;    /* NULL if entry unused */
+    uintptr_t gcenc;
+    struct link_s *next_in_bucket;
+};
+
+#define MARKER_LIST_START  ((pyobj_t *)-1)
+
+static struct link_s **hash_buckets, *hash_list, *hash_free_list;
+static uintptr_t hash_mask_bucket;
+static intptr_t hash_list_walk_next = -1;
+
+static uintptr_t hash_get_hash(gcobj_t *gcobj)
+{
+    assert(gcobj != NULL);
+    uintptr_t h = (uintptr_t)gcobj;
+    assert((h & 1) == 0);
+    h -= (h >> 6);
+    return h & hash_mask_bucket;
+}
+
+static gcobj_t *decode_gcenc(uintptr_t gcenc)
+{
+    if (gcenc & 1)
+        gcenc = ~gcenc;
+    return (gcobj_t *)gcenc;
+}
+
+static void hash_link(struct link_s *lnk)
+{
+    uintptr_t h = hash_get_hash(decode_gcenc(lnk->gcenc));
+    lnk->next_in_bucket = hash_buckets[h];
+    hash_buckets[h] = lnk;
+}
+
+static void boehm_is_about_to_collect(void);
+
+static void hash_grow_table(void)
+{
+    static int rec = 0;
+    assert(!rec);   /* recursive hash_grow_table() */
+    rec = 1;
+
+    if (hash_buckets == NULL)
+        GC_set_start_callback(boehm_is_about_to_collect);
+
+    uintptr_t i, num_buckets = (hash_mask_bucket + 1) * 2;
+    if (num_buckets < 16) num_buckets = 16;
+    assert((num_buckets & (num_buckets - 1)) == 0);  /* power of two */
+
+    /* The new hash_buckets: an array of pointers to struct link_s, of
+       length a power of two, used as a dictionary hash table.  It is
+       not allocated with Boehm because there is no point in Boehm looking
+       in it.
+     */
+    struct link_s **new_buckets = calloc(num_buckets, sizeof(struct link_s *));
+    assert(new_buckets);
+
+    /* The new hash_list: the array of all struct link_s.  Their order
+       is irrelevant.  There is a GC_register_finalizer() on the 'gcenc'
+       field, so we don't move the array; instead we allocate a new array
+       to use in addition to the old one.  There are a total of 2 to 4
+       times as many 'struct link_s' as the length of 'buckets'.
+     */
+    uintptr_t num_list = num_buckets * 2;
+    struct link_s *new_list = GC_MALLOC(num_list * sizeof(struct link_s));
+    for (i = num_list; i-- > 1; ) {
+        new_list[i].next_in_bucket = hash_free_list;
+        hash_free_list = &new_list[i];
+    }
+    /* list[0] is abused to store a pointer to the previous list and
+       the length of the current list */
+    struct link_s *old_list = hash_list;
+    new_list[0].next_in_bucket = old_list;
+    new_list[0].gcenc = num_list;
+    new_list[0].pyobj = MARKER_LIST_START;
+
+    hash_list = new_list;
+    free(hash_buckets);
+    hash_buckets = new_buckets;
+    hash_mask_bucket = num_buckets - 1;
+    hash_list_walk_next = hash_mask_bucket;
+
+    /* re-add all old 'struct link_s' to the hash_buckets */
+    struct link_s *plist = old_list;
+    while (plist != NULL) {
+        uintptr_t count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc != 0)
+                hash_link(&plist[i]);
+        }
+        plist = plist[0].next_in_bucket;
+    }
+    GC_reachable_here(old_list);
+
+    rec = 0;
+}
+
+static void hash_add_entry(gcobj_t *gcobj, pyobj_t *pyobj)
+{
+    if (hash_free_list == NULL) {
+        hash_grow_table();
+    }
+    assert(pyobj->ob_pypy_link == 0);
+
+    struct link_s *lnk = hash_free_list;
+    hash_free_list = lnk->next_in_bucket;
+    lnk->pyobj = pyobj;
+    lnk->gcenc = (uintptr_t)gcobj;
+    pyobj->ob_pypy_link = (Py_ssize_t)lnk;
+
+    hash_link(lnk);
+
+    if (GC_base(gcobj) == NULL) {
+        /* 'gcobj' is probably a prebuilt object - it makes no */
+        /* sense to register it then, and it crashes Boehm in */
+        /* quite obscure ways */
+    }
+    else {
+        int j = GC_general_register_disappearing_link(
+                                    (void **)&lnk->gcenc, gcobj);
+        assert(j == GC_SUCCESS);
+    }
+}
+
+static pyobj_t *hash_get_entry(gcobj_t *gcobj)
+{
+    if (hash_buckets == NULL)
+        return NULL;
+    uintptr_t h = hash_get_hash(gcobj);
+    struct link_s *lnk = hash_buckets[h];
+    while (lnk != NULL) {
+        assert(lnk->pyobj != NULL);
+        if (decode_gcenc(lnk->gcenc) == gcobj)
+            return lnk->pyobj;
+        lnk = lnk->next_in_bucket;
+    }
+    return NULL;
+}
+
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_next_dead(void)
+{
+    while (hash_list_walk_next >= 0) {
+        struct link_s *p, **pp = &hash_buckets[hash_list_walk_next];
+        while (1) {
+            p = *pp;
+            if (p == NULL)
+                break;
+            assert(p->pyobj != NULL);
+            if (p->gcenc == 0) {
+                /* quadratic time on the number of links from the same
+                   bucket chain, but it should be small with very high
+                   probability */
+                pyobj_t *result = p->pyobj;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+                printf("next_dead: %p\n", result);
+#endif
+                assert(result->ob_refcnt == REFCNT_FROM_PYPY);
+                result->ob_refcnt = 1;
+                p->pyobj = NULL;
+                *pp = p->next_in_bucket;
+                p->next_in_bucket = hash_free_list;
+                hash_free_list = p;
+                return result;
+            }
+            else {
+                assert(p->gcenc != ~(uintptr_t)0);
+                pp = &p->next_in_bucket;
+            }
+        }
+        hash_list_walk_next--;
+    }
+    return NULL;
+}
+
+RPY_EXTERN
+void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                     /*pyobj_t*/void *pyobj)
+{
+    gcobj_t *gcobj1 = (gcobj_t *)gcobj;
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    assert(pyobj1->ob_pypy_link == 0);
+    /*assert(pyobj1->ob_refcnt >= REFCNT_FROM_PYPY);*/
+    /*^^^ could also be fixed just after the call to create_link_pypy()*/
+
+    hash_add_entry(gcobj1, pyobj1);
+}
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj)
+{
+    return hash_get_entry((gcobj_t *)gcobj);
+}
+
+RPY_EXTERN
+/*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj)
+{
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    if (pyobj1->ob_pypy_link == 0)
+        return NULL;
+
+    struct link_s *lnk = (struct link_s *)pyobj1->ob_pypy_link;
+    assert(lnk->pyobj == pyobj1);
+    
+    gcobj_t *g = decode_gcenc(lnk->gcenc);
+    assert(g != NULL);
+    return g;
+}
+
+static void boehm_is_about_to_collect(void)
+{
+    struct link_s *plist = hash_list;
+    uintptr_t gcenc_union = 0;
+    while (plist != NULL) {
+        uintptr_t i, count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc == 0)
+                continue;
+
+            pyobj_t *p = plist[i].pyobj;
+            assert(p != NULL);
+            assert(p->ob_refcnt >= REFCNT_FROM_PYPY);
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+            printf("plist[%d].gcenc: %p ", (int)i, (void *)plist[i].gcenc);
+#endif
+
+            if ((plist[i].gcenc & 1) ^ (p->ob_refcnt == REFCNT_FROM_PYPY)) {
+                /* ob_refcnt > FROM_PYPY: non-zero regular refcnt, 
+                   the gc obj must stay alive.  decode gcenc.
+                   ---OR---
+                   ob_refcnt == FROM_PYPY: no refs from C code, the
+                   gc obj must not (necessarily) stay alive.  encode gcenc.
+                */
+                plist[i].gcenc = ~plist[i].gcenc;
+            }
+            gcenc_union |= plist[i].gcenc;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+            printf("-> %p\n", (void *)plist[i].gcenc);
+#endif
+    }
+        plist = plist[0].next_in_bucket;
+    }
+    if (gcenc_union & 1)   /* if there is at least one item potentially dead */
+        hash_list_walk_next = hash_mask_bucket;
+}
diff --git a/rpython/rlib/src/boehm-rawrefcount.h b/rpython/rlib/src/boehm-rawrefcount.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.h
@@ -0,0 +1,26 @@
+
+/* Missing:
+   OP_GC_RAWREFCOUNT_INIT(callback, r): the callback is not supported here
+   OP_GC_RAWREFCOUNT_CREATE_LINK_PYOBJ(): not implemented, maybe not needed
+*/
+
+#define OP_GC_RAWREFCOUNT_CREATE_LINK_PYPY(gcobj, pyobj, r)   \
+    gc_rawrefcount_create_link_pypy(gcobj, pyobj)
+
+#define OP_GC_RAWREFCOUNT_FROM_OBJ(gcobj, r)   \
+    r = gc_rawrefcount_from_obj(gcobj)
+
+#define OP_GC_RAWREFCOUNT_TO_OBJ(pyobj, r)   \
+    r = gc_rawrefcount_to_obj(pyobj)
+
+#define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
+    r = gc_rawrefcount_next_dead()
+
+#define OP_GC_RAWREFCOUNT_MARK_DEALLOCATING(gcobj, pyobj, r)  /* nothing */
+
+
+RPY_EXTERN void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                                /*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj);
+RPY_EXTERN /*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_next_dead(void);
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -1,4 +1,4 @@
-from rpython.rlib.buffer import *
+from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer
 from rpython.annotator.annrpython import RPythonAnnotator
 from rpython.annotator.model import SomeInteger
 
@@ -64,3 +64,10 @@
     for i in range(9999, 9, -1):
         buf = SubBuffer(buf, 1, i)
     assert buf.getlength() == 10
+
+def test_string_buffer_as_buffer():
+    buf = StringBuffer(b'hello world')
+    addr = buf.get_raw_address()
+    assert addr[0] == b'h'
+    assert addr[4] == b'o'
+    assert addr[6] == b'w'
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -1,7 +1,7 @@
 import weakref
 from rpython.rlib import rawrefcount, objectmodel, rgc
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT
-from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.translator.c.test.test_standalone import StandaloneTests
 from rpython.config.translationoption import get_combined_translation_config
@@ -264,6 +264,9 @@
             if rawrefcount.next_dead(PyObject) != ob:
                 print "NEXT_DEAD != OB"
                 return 1
+            if ob.c_ob_refcnt != 1:
+                print "next_dead().ob_refcnt != 1"
+                return 1
             if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
                 print "NEXT_DEAD second time != NULL"
                 return 1
diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -0,0 +1,308 @@
+import itertools, os, subprocess, py
+from hypothesis import given, strategies
+from rpython.tool.udir import udir
+from rpython.rlib import rawrefcount, rgc
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.test.test_rawrefcount import W_Root, PyObject, PyObjectS
+from rpython.rtyper.lltypesystem import lltype
+from rpython.translator.c.test.test_standalone import StandaloneTests
+from rpython.config.translationoption import get_combined_translation_config
+
+
+def compile_test(basename):
+    srcdir = os.path.dirname(os.path.dirname(
+        os.path.abspath(os.path.join(__file__))))
+    srcdir = os.path.join(srcdir, 'src')
+
+    err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o %s %s.c"
+                    % (udir, srcdir, basename, basename))
+    return err
+
+def setup_module():
+    filename = str(udir.join("test-rawrefcount-boehm-check.c"))
+    with open(filename, "w") as f:
+        print >> f, '#include "gc/gc_mark.h"'
+        print >> f, '#include <stdio.h>'
+        print >> f, 'int main(void) {'
+        print >> f, '    printf("%p", &GC_set_start_callback);'
+        print >> f, '    return 0;'
+        print >> f, '}'
+
+    if compile_test("test-rawrefcount-boehm-check") != 0:
+        py.test.skip("Boehm GC not installed or too old version")
+
+
+
+TEST_CODE = r"""
+#define TEST_BOEHM_RAWREFCOUNT
+#include "boehm-rawrefcount.c"
+
+static gcobj_t *alloc_gcobj(void)   /* for tests */
+{
+    gcobj_t *g = GC_MALLOC(1000);
+    printf("gc obj: %p\n", g);
+    return g;
+}
+
+static pyobj_t *alloc_pyobj(void)   /* for tests */
+{
+    pyobj_t *p = malloc(1000);
+    p->ob_refcnt = 1;
+    p->ob_pypy_link = 0;
+    printf("py obj: %p\n", p);
+    return p;
+}
+
+static void decref(pyobj_t *p)      /* for tests */
+{
+    p->ob_refcnt--;
+    if (p->ob_refcnt == 0) {
+        printf("decref to zero: %p\n", p);
+        free(p);
+    }
+    assert(p->ob_refcnt >= REFCNT_FROM_PYPY ||
+           p->ob_refcnt < REFCNT_FROM_PYPY * 0.99);
+}
+
+void run_test(void);     /* forward declaration, produced by the test */
+
+int main(void)
+{
+    run_test();
+    while (gc_rawrefcount_next_dead() != NULL)
+        ;
+    return 0;
+}
+"""
+
+
+operations = strategies.sampled_from([
+    'new_pyobj',
+    'new_gcobj',
+    'create_link',
+    'from_obj',
+    'to_obj',
+    'forget_pyobj',
+    'forget_gcobj',
+    'collect',
+    'dead',
+    ])
+
+
+ at strategies.composite
+def make_code(draw):
+    code = []
+    pyobjs = []
+    gcobjs = []
+    num_gcobj = itertools.count()
+    num_pyobj = itertools.count()
+    links_g2p = {}
+    links_p2g = {}
+
+    def new_gcobj():
+        varname = 'g%d' % next(num_gcobj)
+        code.append('gcobj_t *volatile %s = alloc_gcobj();' % varname)
+        gcobjs.append(varname)
+        return varname
+
+    def new_pyobj():
+        varname = 'p%d' % next(num_pyobj)
+        code.append('pyobj_t *%s = alloc_pyobj();' % varname)
+        pyobjs.append(varname)
+        return varname
+
+    for op in draw(strategies.lists(operations, average_size=250)):
+        if op == 'new_gcobj':
+            new_gcobj()
+        elif op == 'new_pyobj':
+            new_pyobj()
+        elif op == 'create_link':
+            gvars = [varname for varname in gcobjs if varname not in links_g2p]
+            if gvars == []:
+                gvars.append(new_gcobj())
+            pvars = [varname for varname in pyobjs if varname not in links_p2g]
+            if pvars == []:
+                pvars.append(new_pyobj())
+            gvar = draw(strategies.sampled_from(gvars))
+            pvar = draw(strategies.sampled_from(pvars))
+            code.append(r'printf("create_link %%p-%%p\n", %s, %s); '
+                            % (gvar, pvar) +
+                        "%s->ob_refcnt += REFCNT_FROM_PYPY; " % pvar +
+                        "gc_rawrefcount_create_link_pypy(%s, %s);"
+                            % (gvar, pvar))
+            links_g2p[gvar] = pvar
+            links_p2g[pvar] = gvar
+        elif op == 'from_obj':
+            if gcobjs:
+                prnt = False
+                gvar = draw(strategies.sampled_from(gcobjs))
+                if gvar not in links_g2p:
+                    check = "== NULL"
+                elif links_g2p[gvar] in pyobjs:
+                    check = "== %s" % (links_g2p[gvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_from_obj(%s) %s);"
+                            % (gvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", %s, '
+                        'gc_rawrefcount_from_obj(%s));' % (gvar, gvar))
+        elif op == 'to_obj':
+            if pyobjs:
+                prnt = False
+                pvar = draw(strategies.sampled_from(pyobjs))
+                if pvar not in links_p2g:
+                    check = "== NULL"
+                elif links_p2g[pvar] in gcobjs:
+                    check = "== %s" % (links_p2g[pvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_to_obj(%s) %s);"
+                            % (pvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", '
+                        'gc_rawrefcount_to_obj(%s), %s);' % (pvar, pvar))
+        elif op == 'forget_pyobj':
+            if pyobjs:
+                index = draw(strategies.sampled_from(range(len(pyobjs))))
+                pvar = pyobjs.pop(index)
+                code.append(r'printf("-p%%p\n", %s); ' % pvar +
+                            "decref(%s); %s = NULL;" % (pvar, pvar))
+        elif op == 'forget_gcobj':
+            if gcobjs:
+                index = draw(strategies.sampled_from(range(len(gcobjs))))
+                gvar = gcobjs.pop(index)
+                code.append(r'printf("-g%%p\n", %s); ' % gvar +
+                            "%s = NULL;" % (gvar,))
+        elif op == 'collect':
+            code.append("GC_gcollect();")
+        elif op == 'dead':
+            code.append('gc_rawrefcount_next_dead();')
+        else:
+            assert False, op
+
+    return '\n'.join(code)
+
+
+ at given(make_code())
+def test_random(code):
+    filename = str(udir.join("test-rawrefcount-boehm.c"))
+    with open(filename, "w") as f:
+        print >> f, TEST_CODE
+        print >> f, 'void run_test(void) {'
+        print >> f, code
+        print >> f, '}'
+
+    err = compile_test("test-rawrefcount-boehm")
+    if err != 0:
+        raise OSError("gcc failed")
+    p = subprocess.Popen("./test-rawrefcount-boehm", stdout=subprocess.PIPE,
+                         cwd=str(udir))
+    stdout, _ = p.communicate()
+    assert p.wait() == 0
+
+    gcobjs = {}
+    pyobjs = {}
+    links_p2g = {}
+    links_g2p = {}
+    for line in stdout.splitlines():
+        if line.startswith('py obj: '):
+            p = line[8:]
+            assert not pyobjs.get(p)
+            pyobjs[p] = True
+            assert p not in links_p2g
+        elif line.startswith('gc obj: '):
+            g = line[8:]
+            assert not gcobjs.get(g)
+            gcobjs[g] = True
+            if g in links_g2p: del links_g2p[g]

From pypy.commits at gmail.com  Fri Jan  6 07:07:24 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 06 Jan 2017 04:07:24 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-from2: another approach, leaks memory
Message-ID: <586f887c.2105c30a.98f04.7f2b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-from2
Changeset: r89389:07be3b1bba7a
Date: 2017-01-06 14:06 +0200
http://bitbucket.org/pypy/pypy/changeset/07be3b1bba7a/

Log:	another approach, leaks memory

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -36,7 +36,22 @@
     Fills a newly allocated PyMemoryViewObject with the given W_MemoryView object.
     """
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
-    py_obj.c_view.c_obj = rffi.cast(PyObject, 0)
+    view = py_obj.c_view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return view
+    fill_Py_buffer(space, w_obj.buf, view)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_userdata)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s),
+                                             track_allocation=False))
+        rffi.setintfield(view, 'c_readonly', 1)
 
 def memory_realize(space, py_obj):
     """
@@ -88,29 +103,13 @@
     return ret
 
 @cpython_api([PyObject], Py_bufferP, error=CANNOT_FAIL)
-def PyMemoryView_GET_BUFFER(space, w_obj):
+def PyMemoryView_GET_BUFFER(space, pyobj):
     """Return a pointer to the buffer-info structure wrapped by the given
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
-    if not isinstance(w_obj, W_MemoryView):
-        return lltype.nullptr(Py_buffer)
-    py_memobj = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_obj)) # no inc_ref
-    view = py_memobj.c_view
-    ndim = w_obj.buf.getndim()
-    if ndim >= Py_MAX_NDIMS:
-        # XXX warn?
-        return view
-    fill_Py_buffer(space, w_obj.buf, view)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-        #view.c_obj = make_ref(space, w_obj) # NO - this creates a ref cycle!
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
-    except ValueError:
-        w_s = w_obj.descr_tobytes(space)
-        view.c_obj = make_ref(space, w_s)
-        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
-        rffi.setintfield(view, 'c_readonly', 1)
-    return view
+    # XXX move to a c-macro
+    py_memobj = rffi.cast(PyMemoryViewObject, pyobj) # no inc_ref
+    return py_memobj.c_view
 
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
@@ -204,7 +203,9 @@
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_FromObject(space, w_obj):
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    w_memview = space.call_method(space.builtin, "memoryview", w_obj)
+    py_memview = make_ref(space, w_memview, w_obj)
+    return py_memview
 
 @cpython_api([Py_bufferP], PyObject)
 def PyMemoryView_FromBuffer(space, view):
@@ -212,6 +213,7 @@
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
+    assert view.c_obj
     w_obj = from_ref(space, view.c_obj)
     if isinstance(w_obj, W_MemoryView):
         return w_obj

From pypy.commits at gmail.com  Fri Jan  6 07:10:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jan 2017 04:10:18 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix ``"".replace("", "x",
 num)`` to give the same result as CPython
Message-ID: <586f892a.2972c20a.d3693.014d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89390:bb49f0339d82
Date: 2017-01-06 13:09 +0100
http://bitbucket.org/pypy/pypy/changeset/bb49f0339d82/

Log:	Fix ``"".replace("", "x", num)`` to give the same result as CPython

diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -548,6 +548,10 @@
 
         sub = self._op_val(space, w_old)
         by = self._op_val(space, w_new)
+        # the following two lines are for being bug-to-bug compatible
+        # with CPython: see issue #2448
+        if count >= 0 and len(input) == 0:
+            return self._empty()
         try:
             res = replace(input, sub, by, count)
         except OverflowError:
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -809,6 +809,16 @@
         s = b"a" * (2**16)
         raises(OverflowError, s.replace, b"", s)
 
+    def test_replace_issue2448(self):
+        # CPython's replace() method has a bug that makes
+        #   ''.replace('', 'x')  gives a different answer than
+        #   ''.replace('', 'x', 1000).  This is the case in all
+        # known versions, at least until 2.7.13.  Some people
+        # call that a feature on the CPython issue report and
+        # the discussion dies out, so it might never be fixed.
+        assert ''.replace('', 'x') == 'x'
+        assert ''.replace('', 'x', 1000) == ''
+
     def test_getslice(self):
         assert "foobar".__getslice__(4, 4321) == "ar"
         s = b"abc"

From pypy.commits at gmail.com  Fri Jan  6 07:32:27 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 06 Jan 2017 04:32:27 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-from2: fix tests, add result_is_ll
Message-ID: <586f8e5b.c4811c0a.980b4.16db@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-from2
Changeset: r89391:14fae6e9cb91
Date: 2017-01-06 14:30 +0200
http://bitbucket.org/pypy/pypy/changeset/14fae6e9cb91/

Log:	fix tests, add result_is_ll

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -201,7 +201,7 @@
         return (_IsCContiguous(view) or _IsFortranContiguous(view))
     return 0
 
- at cpython_api([PyObject], PyObject)
+ at cpython_api([PyObject], PyObject, result_is_ll=True)
 def PyMemoryView_FromObject(space, w_obj):
     w_memview = space.call_method(space.builtin, "memoryview", w_obj)
     py_memview = make_ref(space, w_memview, w_obj)
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -4,6 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
+from pypy.module.cpyext.pyobject import from_ref
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
@@ -11,7 +12,7 @@
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
-        w_view = api.PyMemoryView_FromObject(w_hello)
+        w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
         assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
@@ -19,7 +20,7 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_memoryview = from_ref(space, api.PyMemoryView_FromObject(w_buf))
         view = api.PyMemoryView_GET_BUFFER(w_memoryview)
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)

From pypy.commits at gmail.com  Fri Jan  6 07:58:50 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 06 Jan 2017 04:58:50 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: hg merge default
Message-ID: <586f948a.62f2c20a.cb640.9232@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89392:d93975c9dee0
Date: 2017-01-06 12:57 +0000
http://bitbucket.org/pypy/pypy/changeset/d93975c9dee0/

Log:	hg merge default

diff too long, truncating to 2000 out of 31758 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
diff --git a/ctypes_configure/__init__.py b/ctypes_configure/__init__.py
deleted file mode 100644
diff --git a/ctypes_configure/cbuild.py b/ctypes_configure/cbuild.py
deleted file mode 100644
--- a/ctypes_configure/cbuild.py
+++ /dev/null
@@ -1,456 +0,0 @@
-
-import os, sys, inspect, re, imp, py
-from ctypes_configure import stdoutcapture
-import distutils
-
-debug = 0
-
-configdir = py.path.local.make_numbered_dir(prefix='ctypes_configure-')
-
-class ExternalCompilationInfo(object):
-
-    _ATTRIBUTES = ['pre_include_lines', 'includes', 'include_dirs',
-                   'post_include_lines', 'libraries', 'library_dirs',
-                   'separate_module_sources', 'separate_module_files']
-    _AVOID_DUPLICATES = ['separate_module_files', 'libraries', 'includes',
-                         'include_dirs', 'library_dirs', 'separate_module_sources']
-
-    def __init__(self,
-                 pre_include_lines       = [],
-                 includes                = [],
-                 include_dirs            = [],
-                 post_include_lines      = [],
-                 libraries               = [],
-                 library_dirs            = [],
-                 separate_module_sources = [],
-                 separate_module_files   = []):
-        """
-        pre_include_lines: list of lines that should be put at the top
-        of the generated .c files, before any #include.  They shouldn't
-        contain an #include themselves.
-
-        includes: list of .h file names to be #include'd from the
-        generated .c files.
-
-        include_dirs: list of dir names that is passed to the C compiler
-
-        post_include_lines: list of lines that should be put at the top
-        of the generated .c files, after the #includes.
-
-        libraries: list of library names that is passed to the linker
-
-        library_dirs: list of dir names that is passed to the linker
-
-        separate_module_sources: list of multiline strings that are
-        each written to a .c file and compiled separately and linked
-        later on.  (If function prototypes are needed for other .c files
-        to access this, they can be put in post_include_lines.)
-
-        separate_module_files: list of .c file names that are compiled
-        separately and linked later on.  (If an .h file is needed for
-        other .c files to access this, it can be put in includes.)
-        """
-        for name in self._ATTRIBUTES:
-            value = locals()[name]
-            assert isinstance(value, (list, tuple))
-            setattr(self, name, tuple(value))
-
-    def _value(self):
-        return tuple([getattr(self, x) for x in self._ATTRIBUTES])
-
-    def __hash__(self):
-        return hash(self._value())
-
-    def __eq__(self, other):
-        return self.__class__ is other.__class__ and \
-               self._value() == other._value()
-
-    def __ne__(self, other):
-        return not self == other
-
-    def __repr__(self):
-        info = []
-        for attr in self._ATTRIBUTES:
-            val = getattr(self, attr)
-            info.append("%s=%s" % (attr, repr(val)))
-        return "<ExternalCompilationInfo (%s)>" % ", ".join(info)
-
-    def merge(self, *others):
-        others = list(others)
-        attrs = {}
-        for name in self._ATTRIBUTES:
-            if name not in self._AVOID_DUPLICATES:
-                s = []
-                for i in [self] + others:
-                    s += getattr(i, name)
-                attrs[name] = s
-            else:
-                s = set()
-                attr = []
-                for one in [self] + others:
-                    for elem in getattr(one, name):
-                        if elem not in s:
-                            s.add(elem)
-                            attr.append(elem)
-                attrs[name] = attr
-        return ExternalCompilationInfo(**attrs)
-
-    def write_c_header(self, fileobj):
-        for line in self.pre_include_lines:
-            print >> fileobj, line
-        for path in self.includes:
-            print >> fileobj, '#include <%s>' % (path,)
-        for line in self.post_include_lines:
-            print >> fileobj, line
-
-    def _copy_attributes(self):
-        d = {}
-        for attr in self._ATTRIBUTES:
-            d[attr] = getattr(self, attr)
-        return d
-
-    def convert_sources_to_files(self, cache_dir=None, being_main=False):
-        if not self.separate_module_sources:
-            return self
-        if cache_dir is None:
-            cache_dir = configdir.join('module_cache').ensure(dir=1)
-        num = 0
-        files = []
-        for source in self.separate_module_sources:
-            while 1:
-                filename = cache_dir.join('module_%d.c' % num)
-                num += 1
-                if not filename.check():
-                    break
-            f = filename.open("w")
-            if being_main:
-                f.write("#define PYPY_NOT_MAIN_FILE\n")
-            self.write_c_header(f)
-            source = str(source)
-            f.write(source)
-            if not source.endswith('\n'):
-                f.write('\n')
-            f.close()
-            files.append(str(filename))
-        d = self._copy_attributes()
-        d['separate_module_sources'] = ()
-        d['separate_module_files'] += tuple(files)
-        return ExternalCompilationInfo(**d)
-
-    def compile_shared_lib(self):
-        self = self.convert_sources_to_files()
-        if not self.separate_module_files:
-            return self
-        lib = compile_c_module([], 'externmod', self)
-        d = self._copy_attributes()
-        d['libraries'] += (lib,)
-        d['separate_module_files'] = ()
-        d['separate_module_sources'] = ()
-        return ExternalCompilationInfo(**d)
-
-if sys.platform == 'win32':
-    so_ext = '.dll'
-else:
-    so_ext = '.so'
-
-def compiler_command():
-    # e.g. for tcc, you might set this to
-    #    "tcc -shared -o %s.so %s.c"
-    return os.getenv('PYPY_CC')
-
-def enable_fast_compilation():
-    if sys.platform == 'win32':
-        dash = '/'
-    else:
-        dash = '-'
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt:
-        opt = re.sub('%sO\d+' % dash, '%sO0' % dash, opt)
-    else:
-        opt = '%sO0' % dash
-    gcv['OPT'] = opt
-
-def ensure_correct_math():
-    if sys.platform != 'win32':
-        return # so far
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt and '/Op' not in opt:
-        opt += '/Op'
-    gcv['OPT'] = opt
-
-
-def try_compile(c_files, eci):
-    try:
-        build_executable(c_files, eci)
-        result = True
-    except (distutils.errors.CompileError,
-            distutils.errors.LinkError):
-        result = False
-    return result
-
-def compile_c_module(cfiles, modbasename, eci, tmpdir=None):
-    #try:
-    #    from distutils.log import set_threshold
-    #    set_threshold(10000)
-    #except ImportError:
-    #    print "ERROR IMPORTING"
-    #    pass
-    cfiles = [py.path.local(f) for f in cfiles]
-    if tmpdir is None:
-        tmpdir = configdir.join("module_cache").ensure(dir=1)
-    num = 0
-    cfiles += eci.separate_module_files
-    include_dirs = list(eci.include_dirs)
-    library_dirs = list(eci.library_dirs)
-    if (sys.platform == 'darwin' or    # support Fink & Darwinports
-            sys.platform.startswith('freebsd')):
-        for s in ('/sw/', '/opt/local/', '/usr/local/'):
-            if s + 'include' not in include_dirs and \
-               os.path.exists(s + 'include'):
-                include_dirs.append(s + 'include')
-            if s + 'lib' not in library_dirs and \
-               os.path.exists(s + 'lib'):
-                library_dirs.append(s + 'lib')
-
-    num = 0
-    modname = modbasename
-    while 1:
-        if not tmpdir.join(modname + so_ext).check():
-            break
-        num += 1
-        modname = '%s_%d' % (modbasename, num)
-
-    lastdir = tmpdir.chdir()
-    libraries = eci.libraries
-    ensure_correct_math()
-    try:
-        if debug: print "modname", modname
-        c = stdoutcapture.Capture(mixed_out_err = True)
-        try:
-            try:
-                if compiler_command():
-                    # GCC-ish options only
-                    from distutils import sysconfig
-                    gcv = sysconfig.get_config_vars()
-                    cmd = compiler_command().replace('%s',
-                                                     str(tmpdir.join(modname)))
-                    for dir in [gcv['INCLUDEPY']] + list(include_dirs):
-                        cmd += ' -I%s' % dir
-                    for dir in library_dirs:
-                        cmd += ' -L%s' % dir
-                    os.system(cmd)
-                else:
-                    from distutils.dist import Distribution
-                    from distutils.extension import Extension
-                    from distutils.ccompiler import get_default_compiler
-                    saved_environ = os.environ.items()
-                    try:
-                        # distutils.core.setup() is really meant for end-user
-                        # interactive usage, because it eats most exceptions and
-                        # turn them into SystemExits.  Instead, we directly
-                        # instantiate a Distribution, which also allows us to
-                        # ignore unwanted features like config files.
-                        extra_compile_args = []
-                        # ensure correct math on windows
-                        if sys.platform == 'win32':
-                            extra_compile_args.append('/Op') # get extra precision
-                        if get_default_compiler() == 'unix':
-                            old_version = False
-                            try:
-                                g = os.popen('gcc --version', 'r')
-                                verinfo = g.read()
-                                g.close()
-                            except (OSError, IOError):
-                                pass
-                            else:
-                                old_version = verinfo.startswith('2')
-                            if not old_version:
-                                extra_compile_args.extend(["-Wno-unused-label",
-                                                        "-Wno-unused-variable"])
-                        attrs = {
-                            'name': "testmodule",
-                            'ext_modules': [
-                                Extension(modname, [str(cfile) for cfile in cfiles],
-                                    include_dirs=include_dirs,
-                                    library_dirs=library_dirs,
-                                    extra_compile_args=extra_compile_args,
-                                    libraries=list(libraries),)
-                                ],
-                            'script_name': 'setup.py',
-                            'script_args': ['-q', 'build_ext', '--inplace', '--force'],
-                            }
-                        dist = Distribution(attrs)
-                        if not dist.parse_command_line():
-                            raise ValueError, "distutils cmdline parse error"
-                        dist.run_commands()
-                    finally:
-                        for key, value in saved_environ:
-                            if os.environ.get(key) != value:
-                                os.environ[key] = value
-            finally:
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = open("%s.errors" % modname, "w")
-                    fdump.write(data)
-                    fdump.close()
-            # XXX do we need to do some check on fout/ferr?
-            # XXX not a nice way to import a module
-        except:
-            print >>sys.stderr, data
-            raise
-    finally:
-        lastdir.chdir()
-    return str(tmpdir.join(modname) + so_ext)
-
-def make_module_from_c(cfile, eci):
-    cfile = py.path.local(cfile)
-    modname = cfile.purebasename
-    compile_c_module([cfile], modname, eci)
-    return import_module_from_directory(cfile.dirpath(), modname)
-
-def import_module_from_directory(dir, modname):
-    file, pathname, description = imp.find_module(modname, [str(dir)])
-    try:
-        mod = imp.load_module(modname, file, pathname, description)
-    finally:
-        if file:
-            file.close()
-    return mod
-
-
-def log_spawned_cmd(spawn):
-    def spawn_and_log(cmd, *args, **kwds):
-        if debug:
-            print ' '.join(cmd)
-        return spawn(cmd, *args, **kwds)
-    return spawn_and_log
-
-
-class ProfOpt(object):
-    #XXX assuming gcc style flags for now
-    name = "profopt"
-    
-    def __init__(self, compiler):
-        self.compiler = compiler
-
-    def first(self):
-        self.build('-fprofile-generate')
-
-    def probe(self, exe, args):
-        # 'args' is a single string typically containing spaces
-        # and quotes, which represents several arguments.
-        os.system("'%s' %s" % (exe, args))
-
-    def after(self):
-        self.build('-fprofile-use')
-
-    def build(self, option):
-        compiler = self.compiler
-        compiler.compile_extra.append(option)
-        compiler.link_extra.append(option)
-        try:
-            compiler._build()
-        finally:
-            compiler.compile_extra.pop()
-            compiler.link_extra.pop()
-            
-class CCompiler:
-
-    def __init__(self, cfilenames, eci, outputfilename=None,
-                 compiler_exe=None, profbased=None):
-        self.cfilenames = cfilenames
-        ext = ''
-        self.compile_extra = []
-        self.link_extra = []
-        self.libraries = list(eci.libraries)
-        self.include_dirs = list(eci.include_dirs)
-        self.library_dirs = list(eci.library_dirs)
-        self.compiler_exe = compiler_exe
-        self.profbased = profbased
-        if not sys.platform in ('win32', 'darwin', 'cygwin'): # xxx
-            if 'm' not in self.libraries:
-                self.libraries.append('m')
-            if 'pthread' not in self.libraries:
-                self.libraries.append('pthread')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer', '-pthread']
-            self.link_extra += ['-pthread']
-        if sys.platform == 'win32':
-            self.link_extra += ['/DEBUG'] # generate .pdb file
-        if (sys.platform == 'darwin' or    # support Fink & Darwinports
-                sys.platform.startswith('freebsd')):
-            for s in ('/sw/', '/opt/local/', '/usr/local/'):
-                if s + 'include' not in self.include_dirs and \
-                   os.path.exists(s + 'include'):
-                    self.include_dirs.append(s + 'include')
-                if s + 'lib' not in self.library_dirs and \
-                   os.path.exists(s + 'lib'):
-                    self.library_dirs.append(s + 'lib')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer']
-
-        if outputfilename is None:
-            self.outputfilename = py.path.local(cfilenames[0]).new(ext=ext)
-        else: 
-            self.outputfilename = py.path.local(outputfilename)
-
-    def build(self, noerr=False):
-        basename = self.outputfilename.new(ext='')
-        data = ''
-        try:
-            saved_environ = os.environ.copy()
-            c = stdoutcapture.Capture(mixed_out_err = True)
-            try:
-                self._build()
-            finally:
-                # workaround for a distutils bugs where some env vars can
-                # become longer and longer every time it is used
-                for key, value in saved_environ.items():
-                    if os.environ.get(key) != value:
-                        os.environ[key] = value
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = basename.new(ext='errors').open("w")
-                    fdump.write(data)
-                    fdump.close()
-        except:
-            if not noerr:
-                print >>sys.stderr, data
-            raise
- 
-    def _build(self):
-        from distutils.ccompiler import new_compiler 
-        compiler = new_compiler(force=1)
-        if self.compiler_exe is not None:
-            for c in '''compiler compiler_so compiler_cxx
-                        linker_exe linker_so'''.split():
-                compiler.executables[c][0] = self.compiler_exe
-        compiler.spawn = log_spawned_cmd(compiler.spawn)
-        objects = []
-        for cfile in self.cfilenames: 
-            cfile = py.path.local(cfile)
-            old = cfile.dirpath().chdir() 
-            try: 
-                res = compiler.compile([cfile.basename], 
-                                       include_dirs=self.include_dirs,
-                                       extra_preargs=self.compile_extra)
-                assert len(res) == 1
-                cobjfile = py.path.local(res[0]) 
-                assert cobjfile.check()
-                objects.append(str(cobjfile))
-            finally: 
-                old.chdir() 
-        compiler.link_executable(objects, str(self.outputfilename),
-                                 libraries=self.libraries,
-                                 extra_preargs=self.link_extra,
-                                 library_dirs=self.library_dirs)
-
-def build_executable(*args, **kwds):
-    noerr = kwds.pop('noerr', False)
-    compiler = CCompiler(*args, **kwds)
-    compiler.build(noerr=noerr)
-    return str(compiler.outputfilename)
diff --git a/ctypes_configure/configure.py b/ctypes_configure/configure.py
deleted file mode 100755
--- a/ctypes_configure/configure.py
+++ /dev/null
@@ -1,621 +0,0 @@
-#! /usr/bin/env python
-
-import os, py, sys
-import ctypes
-from ctypes_configure.cbuild import build_executable, configdir, try_compile
-from ctypes_configure.cbuild import ExternalCompilationInfo
-import distutils
-
-# ____________________________________________________________
-#
-# Helpers for simple cases
-
-def eci_from_header(c_header_source):
-    return ExternalCompilationInfo(
-        pre_include_lines=c_header_source.split("\n")
-    )
-
-
-def getstruct(name, c_header_source, interesting_fields):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        STRUCT = Struct(name, interesting_fields)
-    return configure(CConfig)['STRUCT']
-
-def getsimpletype(name, c_header_source, ctype_hint=ctypes.c_int):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        TYPE = SimpleType(name, ctype_hint)
-    return configure(CConfig)['TYPE']
-
-def getconstantinteger(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        CONST = ConstantInteger(name)
-    return configure(CConfig)['CONST']
-
-def getdefined(macro, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        DEFINED = Defined(macro)
-    return configure(CConfig)['DEFINED']
-
-def has(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        HAS = Has(name)
-    return configure(CConfig)['HAS']
-
-def check_eci(eci):
-    """Check if a given ExternalCompilationInfo compiles and links."""
-    class CConfig:
-        _compilation_info_ = eci
-        WORKS = Works()
-    return configure(CConfig)['WORKS']
-
-def sizeof(name, eci, **kwds):
-    class CConfig:
-        _compilation_info_ = eci
-        SIZE = SizeOf(name)
-    for k, v in kwds.items():
-        setattr(CConfig, k, v)
-    return configure(CConfig)['SIZE']
-
-def memory_alignment():
-    """Return the alignment (in bytes) of memory allocations.
-    This is enough to make sure a structure with pointers and 'double'
-    fields is properly aligned."""
-    global _memory_alignment
-    if _memory_alignment is None:
-        S = getstruct('struct memory_alignment_test', """
-           struct memory_alignment_test {
-               double d;
-               void* p;
-           };
-        """, [])
-        result = ctypes.alignment(S)
-        assert result & (result-1) == 0, "not a power of two??"
-        _memory_alignment = result
-    return _memory_alignment
-_memory_alignment = None
-
-# ____________________________________________________________
-#
-# General interface
-
-class ConfigResult:
-    def __init__(self, CConfig, info, entries):
-        self.CConfig = CConfig
-        self.result = {}
-        self.info = info
-        self.entries = entries
-        
-    def get_entry_result(self, entry):
-        try:
-            return self.result[entry]
-        except KeyError:
-            pass
-        name = self.entries[entry]
-        info = self.info[name]
-        self.result[entry] = entry.build_result(info, self)
-
-    def get_result(self):
-        return dict([(name, self.result[entry])
-                     for entry, name in self.entries.iteritems()])
-
-
-class _CWriter(object):
-    """ A simple class which aggregates config parts
-    """
-    def __init__(self, CConfig):
-        self.path = uniquefilepath()
-        self.f = self.path.open("w")
-        self.config = CConfig
-
-    def write_header(self):
-        f = self.f
-        CConfig = self.config
-        CConfig._compilation_info_.write_c_header(f)
-        print >> f, C_HEADER
-        print >> f
-
-    def write_entry(self, key, entry):
-        f = self.f
-        print >> f, 'void dump_section_%s(void) {' % (key,)
-        for line in entry.prepare_code():
-            if line and line[0] != '#':
-                line = '\t' + line
-            print >> f, line
-        print >> f, '}'
-        print >> f
-
-    def write_entry_main(self, key):
-        print >> self.f, '\tprintf("-+- %s\\n");' % (key,)
-        print >> self.f, '\tdump_section_%s();' % (key,)
-        print >> self.f, '\tprintf("---\\n");'
-
-    def start_main(self):
-        print >> self.f, 'int main(int argc, char *argv[]) {'
-
-    def close(self):
-        f = self.f
-        print >> f, '\treturn 0;'
-        print >> f, '}'
-        f.close()
-
-    def ask_gcc(self, question):
-        self.start_main()
-        self.f.write(question + "\n")
-        self.close()
-        eci = self.config._compilation_info_
-        return try_compile([self.path], eci)
-
-        
-def configure(CConfig, noerr=False):
-    """Examine the local system by running the C compiler.
-    The CConfig class contains CConfigEntry attribues that describe
-    what should be inspected; configure() returns a dict mapping
-    names to the results.
-    """
-    for attr in ['_includes_', '_libraries_', '_sources_', '_library_dirs_',
-                 '_include_dirs_', '_header_']:
-        assert not hasattr(CConfig, attr), "Found legacy attribut %s on CConfig" % (attr,)
-    entries = []
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigEntry):
-            entries.append((key, value))
-
-    if entries:   # can be empty if there are only CConfigSingleEntries
-        writer = _CWriter(CConfig)
-        writer.write_header()
-        for key, entry in entries:
-            writer.write_entry(key, entry)
-
-        f = writer.f
-        writer.start_main()
-        for key, entry in entries:
-            writer.write_entry_main(key)
-        writer.close()
-
-        eci = CConfig._compilation_info_
-        infolist = list(run_example_code(writer.path, eci, noerr=noerr))
-        assert len(infolist) == len(entries)
-
-        resultinfo = {}
-        resultentries = {}
-        for info, (key, entry) in zip(infolist, entries):
-            resultinfo[key] = info
-            resultentries[entry] = key
-
-        result = ConfigResult(CConfig, resultinfo, resultentries)
-        for name, entry in entries:
-            result.get_entry_result(entry)
-        res = result.get_result()
-    else:
-        res = {}
-
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigSingleEntry):
-            writer = _CWriter(CConfig)
-            writer.write_header()
-            res[key] = value.question(writer.ask_gcc)
-    return res
-
-# ____________________________________________________________
-
-
-class CConfigEntry(object):
-    "Abstract base class."
-
-class Struct(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined structure.
-    """
-    def __init__(self, name, interesting_fields, ifdef=None):
-        self.name = name
-        self.interesting_fields = interesting_fields
-        self.ifdef = ifdef
-
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield 'typedef struct {'
-        yield '    char c;'
-        yield '    ctypesplatcheck_t s;'
-        yield '} ctypesplatcheck2_t;'
-        yield ''
-        yield 'ctypesplatcheck_t s;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("align", offsetof(ctypesplatcheck2_t, s));'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        for fieldname, fieldtype in self.interesting_fields:
-            yield 'dump("fldofs %s", offsetof(ctypesplatcheck_t, %s));'%(
-                fieldname, fieldname)
-            yield 'dump("fldsize %s",   sizeof(s.%s));' % (
-                fieldname, fieldname)
-            if fieldtype in integer_class:
-                yield 's.%s = 0; s.%s = ~s.%s;' % (fieldname,
-                                                   fieldname,
-                                                   fieldname)
-                yield 'dump("fldunsigned %s", s.%s > 0);' % (fieldname,
-                                                             fieldname)
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None:
-            if not info['defined']:
-                return None
-        alignment = 1
-        layout = [None] * info['size']
-        for fieldname, fieldtype in self.interesting_fields:
-            if isinstance(fieldtype, Struct):
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                c_fieldtype = config_result.get_entry_result(fieldtype)
-                layout_addfield(layout, offset, c_fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(c_fieldtype))
-            else:
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                sign   = info.get('fldunsigned ' + fieldname, False)
-                if (size, sign) != size_and_sign(fieldtype):
-                    fieldtype = fixup_ctype(fieldtype, fieldname, (size, sign))
-                layout_addfield(layout, offset, fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(fieldtype))
-
-        # try to enforce the same alignment as the one of the original
-        # structure
-        if alignment < info['align']:
-            choices = [ctype for ctype in alignment_types
-                             if ctype_alignment(ctype) == info['align']]
-            assert choices, "unsupported alignment %d" % (info['align'],)
-            choices = [(ctypes.sizeof(ctype), i, ctype)
-                       for i, ctype in enumerate(choices)]
-            csize, _, ctype = min(choices)
-            for i in range(0, info['size'] - csize + 1, info['align']):
-                if layout[i:i+csize] == [None] * csize:
-                    layout_addfield(layout, i, ctype, '_alignment')
-                    break
-            else:
-                raise AssertionError("unenforceable alignment %d" % (
-                    info['align'],))
-
-        n = 0
-        for i, cell in enumerate(layout):
-            if cell is not None:
-                continue
-            layout_addfield(layout, i, ctypes.c_char, '_pad%d' % (n,))
-            n += 1
-
-        # build the ctypes Structure
-        seen = {}
-        fields = []
-        for cell in layout:
-            if cell in seen:
-                continue
-            fields.append((cell.name, cell.ctype))
-            seen[cell] = True
-
-        class S(ctypes.Structure):
-            _fields_ = fields
-        name = self.name
-        if name.startswith('struct '):
-            name = name[7:]
-        S.__name__ = name
-        return S
-
-class SimpleType(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined simple numeric type.
-    """
-    def __init__(self, name, ctype_hint=ctypes.c_int, ifdef=None):
-        self.name = name
-        self.ctype_hint = ctype_hint
-        self.ifdef = ifdef
-        
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield ''
-        yield 'ctypesplatcheck_t x;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        if self.ctype_hint in integer_class:
-            yield 'x = 0; x = ~x;'
-            yield 'dump("unsigned", x > 0);'
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None and not info['defined']:
-            return None
-        size = info['size']
-        sign = info.get('unsigned', False)
-        ctype = self.ctype_hint
-        if (size, sign) != size_and_sign(ctype):
-            ctype = fixup_ctype(ctype, self.name, (size, sign))
-        return ctype
-
-class ConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant.
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'if ((%s) < 0) {' % (self.name,)
-        yield '    long long x = (long long)(%s);' % (self.name,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.name,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-
-    def build_result(self, info, config_result):
-        return info['value']
-
-class DefinedConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant. If not #defined the value will be None.
-    """
-    def __init__(self, macro):
-        self.name = self.macro = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'if ((%s) < 0) {' % (self.macro,)
-        yield '    long long x = (long long)(%s);' % (self.macro,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.macro,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            return info['value']
-        return None
-
-
-class DefinedConstantString(CConfigEntry):
-    """
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'int i;'
-        yield 'char *p = %s;' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'for (i = 0; p[i] != 0; i++ ) {'
-        yield '  printf("value_%d: %d\\n", i, (int)(unsigned char)p[i]);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            string = ''
-            d = 0
-            while info.has_key('value_%d' % d):
-                string += chr(info['value_%d' % d])
-                d += 1
-            return string
-        return None
-
-
-class Defined(CConfigEntry):
-    """A boolean, corresponding to an #ifdef.
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % (self.macro,)
-        yield 'dump("defined", 1);'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        return bool(info['defined'])
-
-class CConfigSingleEntry(object):
-    """ An abstract class of type which requires
-    gcc succeeding/failing instead of only asking
-    """
-    pass
-
-class Has(CConfigSingleEntry):
-    def __init__(self, name):
-        self.name = name
-    
-    def question(self, ask_gcc):
-        return ask_gcc(self.name + ';')
-
-class Works(CConfigSingleEntry):
-    def question(self, ask_gcc):
-        return ask_gcc("")
-
-class SizeOf(CConfigEntry):
-    """An entry in a CConfig class that stands for
-    some external opaque type
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'dump("size",  sizeof(%s));' % self.name
-
-    def build_result(self, info, config_result):
-        return info['size']
-
-# ____________________________________________________________
-#
-# internal helpers
-
-def ctype_alignment(c_type):
-    if issubclass(c_type, ctypes.Structure):
-        return max([ctype_alignment(fld_type)
-                     for fld_name, fld_type in c_type._fields_])
-    
-    return ctypes.alignment(c_type)
-
-def uniquefilepath(LAST=[0]):
-    i = LAST[0]
-    LAST[0] += 1
-    return configdir.join('ctypesplatcheck_%d.c' % i)
-
-alignment_types = [
-    ctypes.c_short,
-    ctypes.c_int,
-    ctypes.c_long,
-    ctypes.c_float,
-    ctypes.c_double,
-    ctypes.c_char_p,
-    ctypes.c_void_p,
-    ctypes.c_longlong,
-    ctypes.c_wchar,
-    ctypes.c_wchar_p,
-    ]
-
-integer_class = [ctypes.c_byte,     ctypes.c_ubyte,
-                 ctypes.c_short,    ctypes.c_ushort,
-                 ctypes.c_int,      ctypes.c_uint,
-                 ctypes.c_long,     ctypes.c_ulong,
-                 ctypes.c_longlong, ctypes.c_ulonglong,
-                 ]
-float_class = [ctypes.c_float, ctypes.c_double]
-
-class Field(object):
-    def __init__(self, name, ctype):
-        self.name = name
-        self.ctype = ctype
-    def __repr__(self):
-        return '<field %s: %s>' % (self.name, self.ctype)
-
-def layout_addfield(layout, offset, ctype, prefix):
-    size = ctypes.sizeof(ctype)
-    name = prefix
-    i = 0
-    while name in layout:
-        i += 1
-        name = '%s_%d' % (prefix, i)
-    field = Field(name, ctype)
-    for i in range(offset, offset+size):
-        assert layout[i] is None, "%s overlaps %r" % (fieldname, layout[i])
-        layout[i] = field
-    return field
-
-def size_and_sign(ctype):
-    return (ctypes.sizeof(ctype),
-            ctype in integer_class and ctype(-1).value > 0)
-
-def fixup_ctype(fieldtype, fieldname, expected_size_and_sign):
-    for typeclass in [integer_class, float_class]:
-        if fieldtype in typeclass:
-            for ctype in typeclass:
-                if size_and_sign(ctype) == expected_size_and_sign:
-                    return ctype
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_char):
-        # for now, assume it is an array of chars; otherwise we'd also
-        # have to check the exact integer type of the elements of the array
-        size, sign = expected_size_and_sign
-        return ctypes.c_char * size
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_ubyte):
-        # grumble, fields of type 'c_char array' have automatic cast-to-
-        # Python-string behavior in ctypes, which may not be what you
-        # want, so here is the same with c_ubytes instead...
-        size, sign = expected_size_and_sign
-        return ctypes.c_ubyte * size
-    raise TypeError("conflicting field type %r for %r" % (fieldtype,
-                                                          fieldname))
-
-
-C_HEADER = """
-#include <stdio.h>
-#include <stddef.h>   /* for offsetof() */
-#ifndef _WIN32
-#  include <stdint.h>   /* FreeBSD: for uint64_t */
-#endif
-
-void dump(char* key, int value) {
-    printf("%s: %d\\n", key, value);
-}
-"""
-
-def run_example_code(filepath, eci, noerr=False):
-    executable = build_executable([filepath], eci, noerr=noerr)
-    output = py.process.cmdexec(executable)
-    section = None
-    for line in output.splitlines():
-        line = line.strip()
-        if line.startswith('-+- '):      # start of a new section
-            section = {}
-        elif line == '---':              # section end
-            assert section is not None
-            yield section
-            section = None
-        elif line:
-            assert section is not None
-            key, value = line.split(': ')
-            section[key] = int(value)
-
-# ____________________________________________________________
-
-def get_python_include_dir():
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    return gcv['INCLUDEPY']
-
-if __name__ == '__main__':
-    doc = """Example:
-    
-       ctypes_platform.py  -h sys/types.h  -h netinet/in.h
-                           'struct sockaddr_in'
-                           sin_port  c_int
-    """
-    import sys, getopt
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'h:')
-    if not args:
-        print >> sys.stderr, doc
-    else:
-        assert len(args) % 2 == 1
-        headers = []
-        for opt, value in opts:
-            if opt == '-h':
-                headers.append('#include <%s>' % (value,))
-        name = args[0]
-        fields = []
-        for i in range(1, len(args), 2):
-            ctype = getattr(ctypes, args[i+1])
-            fields.append((args[i], ctype))
-
-        S = getstruct(name, '\n'.join(headers), fields)
-
-        for key, value in S._fields_:
-            print key, value
diff --git a/ctypes_configure/doc/configure.html b/ctypes_configure/doc/configure.html
deleted file mode 100644
--- a/ctypes_configure/doc/configure.html
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="latin1" ?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=latin1" />
-<meta name="generator" content="Docutils 0.4.1: http://docutils.sourceforge.net/" />
-<title>ctypes configure</title>
-</head>
-<body>
-<div class="document" id="ctypes-configure">
-<h1 class="title">ctypes configure</h1>
-<div class="section">
-<h1><a id="idea" name="idea">idea</a></h1>
-<p>One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes gcc
-for various platform-dependent details like
-exact sizes of types (for example size_t), #defines, exact outline
-of structures etc. It replaces in this regard code generator (h2py).</p>
-</div>
-<div class="section">
-<h1><a id="installation" name="installation">installation</a></h1>
-<p><tt class="docutils literal"><span class="pre">easy_install</span> <span class="pre">ctypes_configure</span></tt></p>
-</div>
-<div class="section">
-<h1><a id="usage" name="usage">usage</a></h1>
-<p><a class="reference" href="http://codespeak.net/svn/pypy/dist/ctypes_configure/doc/sample.py">sample.py</a> explains in details how to use it.</p>
-</div>
-</div>
-</body>
-</html>
diff --git a/ctypes_configure/doc/configure.txt b/ctypes_configure/doc/configure.txt
deleted file mode 100644
--- a/ctypes_configure/doc/configure.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-=================
-ctypes configure
-=================
-
-idea
-====
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes gcc
-for various platform-dependent details like
-exact sizes of types (for example size\_t), #defines, exact outline
-of structures etc. It replaces in this regard code generator (h2py).
-
-installation
-============
-
-``easy_install ctypes_configure``
-
-usage
-=====
-
-:source:`sample.py <ctypes_configure/doc/sample.py>` explains in details how to use it.
diff --git a/ctypes_configure/doc/sample.py b/ctypes_configure/doc/sample.py
deleted file mode 100644
--- a/ctypes_configure/doc/sample.py
+++ /dev/null
@@ -1,72 +0,0 @@
-
-from ctypes_configure import configure
-import ctypes
-
-class CConfigure:
-    _compilation_info_ = configure.ExternalCompilationInfo(
-        
-        # all lines landing in C header before includes
-        pre_include_lines = [],
-
-        # list of .h files to include
-        includes = ['time.h', 'sys/time.h', 'unistd.h'],
-
-        # list of directories to search for include files
-        include_dirs = [],
-
-        # all lines landing in C header after includes
-        post_include_lines = [],
-
-        # libraries to link with
-        libraries = [],
-
-        # library directories
-        library_dirs = [],
-
-        # additional C sources to compile with (that go to
-        # created .c files)
-        separate_module_sources = [],
-
-        # additional existing C source file names
-        separate_module_files = [],
-        )
-
-    # get real int type out of hint and name
-    size_t = configure.SimpleType('size_t', ctypes.c_int)
-
-    # grab value of numerical #define
-    NULL = configure.ConstantInteger('NULL')
-
-    # grab #define, whether it's defined or not
-    EXISTANT = configure.Defined('NULL')
-    NOT_EXISTANT = configure.Defined('XXXNOTNULL')
-
-    # check for existance of C functions
-    has_write = configure.Has('write')
-    no_xxxwrite = configure.Has('xxxwrite')
-
-    # check for size of type
-    sizeof_size_t = configure.SizeOf('size_t')
-
-    # structure, with given hints for interesting fields,
-    # types does not need to be too specific.
-    # all interesting fields would end up with right offset
-    # size and order
-    struct_timeval = configure.Struct('struct timeval',[
-        ('tv_sec', ctypes.c_int),
-        ('tv_usec', ctypes.c_int)])
-
-info = configure.configure(CConfigure)
-
-assert info['has_write']
-assert not info['no_xxxwrite']
-assert info['NULL'] == 0
-size_t = info['size_t']
-print "size_t in ctypes is ", size_t
-assert ctypes.sizeof(size_t) == info['sizeof_size_t']
-assert info['EXISTANT']
-assert not info['NOT_EXISTANT']
-print
-print "fields of struct timeval are "
-for name, value in info['struct_timeval']._fields_:
-    print "  ", name, " ", value
diff --git a/ctypes_configure/dumpcache.py b/ctypes_configure/dumpcache.py
deleted file mode 100644
--- a/ctypes_configure/dumpcache.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os, sys
-import ctypes
-
-
-def dumpcache(referencefilename, filename, config):
-    dirname = os.path.dirname(referencefilename)
-    filename = os.path.join(dirname, filename)
-    f = open(filename, 'w')
-    print >> f, 'import ctypes'
-    print >> f
-    names = config.keys()
-    names.sort()
-    print >> f, '__all__ = %r' % (tuple(names),)
-    print >> f
-    for key in names:
-        val = config[key]
-        if isinstance(val, (int, long)):
-            f.write("%s = %d\n" % (key, val))
-        elif val is None:
-            f.write("%s = None\n" % key)
-        elif isinstance(val, ctypes.Structure.__class__):
-            f.write("class %s(ctypes.Structure):\n" % key)
-            f.write("    _fields_ = [\n")
-            for k, v in val._fields_:
-                f.write("        ('%s', %s),\n" % (k, ctypes_repr(v)))
-            f.write("    ]\n")
-        elif isinstance(val, (tuple, list)):
-            for x in val:
-                assert isinstance(x, (int, long, str)), \
-                       "lists of integers or strings only"
-            f.write("%s = %r\n" % (key, val))
-        else:
-            # a simple type, hopefully
-            f.write("%s = %s\n" % (key, ctypes_repr(val)))
-    f.close()
-    print 'Wrote %s.' % (filename,)
-    sys.stdout.flush()
-
-def ctypes_repr(cls):
-    # ctypes_configure does not support nested structs so far
-    # so let's ignore it
-    if isinstance(cls, ctypes._SimpleCData.__class__):
-        return "ctypes." + cls.__name__
-    if hasattr(cls, '_length_') and hasattr(cls, '_type_'):  # assume an array
-        return '%s*%d' % (ctypes_repr(cls._type_), cls._length_)
-    raise NotImplementedError("saving of object with type %r" % type(cls))
diff --git a/ctypes_configure/stdoutcapture.py b/ctypes_configure/stdoutcapture.py
deleted file mode 100644
--- a/ctypes_configure/stdoutcapture.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-A quick hack to capture stdout/stderr.
-"""
-
-import os, sys
-
-
-class Capture:
-    
-    def __init__(self, mixed_out_err = False):
-        "Start capture of the Unix-level stdout and stderr."
-        if (not hasattr(os, 'tmpfile') or
-            not hasattr(os, 'dup') or
-            not hasattr(os, 'dup2') or
-            not hasattr(os, 'fdopen')):
-            self.dummy = 1
-        else:
-            try:
-                self.tmpout = os.tmpfile()
-                if mixed_out_err:
-                    self.tmperr = self.tmpout
-                else:
-                    self.tmperr = os.tmpfile()
-            except OSError:     # bah?  on at least one Windows box
-                self.dummy = 1
-                return
-            self.dummy = 0
-            # make new stdout/stderr files if needed
-            self.localoutfd = os.dup(1)
-            self.localerrfd = os.dup(2)
-            if hasattr(sys.stdout, 'fileno') and sys.stdout.fileno() == 1:
-                self.saved_stdout = sys.stdout
-                sys.stdout = os.fdopen(self.localoutfd, 'w', 1)
-            else:
-                self.saved_stdout = None
-            if hasattr(sys.stderr, 'fileno') and sys.stderr.fileno() == 2:
-                self.saved_stderr = sys.stderr
-                sys.stderr = os.fdopen(self.localerrfd, 'w', 0)
-            else:
-                self.saved_stderr = None
-            os.dup2(self.tmpout.fileno(), 1)
-            os.dup2(self.tmperr.fileno(), 2)
-
-    def done(self):
-        "End capture and return the captured text (stdoutfile, stderrfile)."
-        if self.dummy:
-            import cStringIO
-            return cStringIO.StringIO(), cStringIO.StringIO()
-        else:
-            os.dup2(self.localoutfd, 1)
-            os.dup2(self.localerrfd, 2)
-            if self.saved_stdout is not None:
-                f = sys.stdout
-                sys.stdout = self.saved_stdout
-                f.close()
-            else:
-                os.close(self.localoutfd)
-            if self.saved_stderr is not None:
-                f = sys.stderr
-                sys.stderr = self.saved_stderr
-                f.close()
-            else:
-                os.close(self.localerrfd)
-            self.tmpout.seek(0)
-            self.tmperr.seek(0)
-            return self.tmpout, self.tmperr
-
-
-if __name__ == '__main__':
-    # test
-    c = Capture()
-    try:
-        os.system('echo hello')
-    finally:
-        fout, ferr = c.done()
-    print 'Output:', `fout.read()`
-    print 'Error:', `ferr.read()`
diff --git a/ctypes_configure/test/__init__.py b/ctypes_configure/test/__init__.py
deleted file mode 100644
diff --git a/ctypes_configure/test/test_configure.py b/ctypes_configure/test/test_configure.py
deleted file mode 100644
--- a/ctypes_configure/test/test_configure.py
+++ /dev/null
@@ -1,212 +0,0 @@
-import py, sys, struct
-from ctypes_configure import configure
-from ctypes_configure.cbuild import ExternalCompilationInfo
-import ctypes
-
-def test_dirent():
-    dirent = configure.getstruct("struct dirent",
-                                       """
-           struct dirent  /* for this example only, not the exact dirent */
-           {
-               long d_ino;
-               int d_off;
-               unsigned short d_reclen;
-               char d_name[32];
-           };
-                                       """,
-                                       [("d_reclen", ctypes.c_ushort)])
-    assert issubclass(dirent, ctypes.Structure)
-    ssize = (ctypes.sizeof(ctypes.c_long) +
-             ctypes.sizeof(ctypes.c_int) +
-             ctypes.sizeof(ctypes.c_ushort) +
-             32)
-    extra_padding = (-ssize) % ctypes.alignment(ctypes.c_long)
-
-    assert dirent._fields_ == [('_alignment', ctypes.c_long),
-                               ('_pad0', ctypes.c_char),
-                               ('_pad1', ctypes.c_char),
-                               ('_pad2', ctypes.c_char),
-                               ('_pad3', ctypes.c_char),
-                               ('d_reclen', ctypes.c_ushort),
-                               ] + [
-                               ('_pad%d' % n, ctypes.c_char)
-                                    for n in range(4, 4+32+extra_padding)]
-    assert ctypes.sizeof(dirent) == ssize + extra_padding
-    assert ctypes.alignment(dirent) == ctypes.alignment(ctypes.c_long)
-
-def test_fit_type():
-    S = configure.getstruct("struct S",
-                                  """
-           struct S {
-               signed char c;
-               unsigned char uc;
-               short s;
-               unsigned short us;
-               int i;
-               unsigned int ui;
-               long l;
-               unsigned long ul;
-               long long ll;
-               unsigned long long ull;
-               float f;
-               double d;
-           };
-                                  """,
-                                  [("c",   ctypes.c_int),
-                                   ("uc",  ctypes.c_int),
-                                   ("s",   ctypes.c_uint),
-                                   ("us",  ctypes.c_int),
-                                   ("i",   ctypes.c_int),
-                                   ("ui",  ctypes.c_int),
-                                   ("l",   ctypes.c_int),
-                                   ("ul",  ctypes.c_int),
-                                   ("ll",  ctypes.c_int),
-                                   ("ull", ctypes.c_int),
-                                   ("f",   ctypes.c_double),
-                                   ("d",   ctypes.c_float)])
-    assert issubclass(S, ctypes.Structure)
-    fields = dict(S._fields_)
-    assert fields["c"] == ctypes.c_byte
-    assert fields["uc"] == ctypes.c_ubyte
-    assert fields["s"] == ctypes.c_short
-    assert fields["us"] == ctypes.c_ushort
-    assert fields["i"] == ctypes.c_int
-    assert fields["ui"] == ctypes.c_uint
-    assert fields["l"] == ctypes.c_long
-    assert fields["ul"] == ctypes.c_ulong
-    assert fields["ll"] == ctypes.c_longlong
-    assert fields["ull"] == ctypes.c_ulonglong
-    assert fields["f"] == ctypes.c_float
-    assert fields["d"] == ctypes.c_double
-
-def test_simple_type():
-    ctype = configure.getsimpletype('test_t',
-                                          'typedef unsigned short test_t;',
-                                          ctypes.c_int)
-    assert ctype == ctypes.c_ushort
-
-def test_constant_integer():
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (6*7)')
-    assert value == 42
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (-2147483648LL)')
-    assert value == -2147483648
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (3333333333ULL)')
-    assert value == 3333333333
-
-def test_defined():
-    res = configure.getdefined('ALFKJLKJFLKJFKLEJDLKEWMECEE', '')
-    assert not res
-    res = configure.getdefined('ALFKJLKJFLKJFKLEJDLKEWMECEE',
-                                     '#define ALFKJLKJFLKJFKLEJDLKEWMECEE')
-    assert res
-
-def test_configure():
-    configdir = configure.configdir
-    test_h = configdir.join('test_ctypes_platform.h')
-    test_h.write('#define XYZZY 42\n')
-
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            pre_include_lines = ["/* a C comment */",
-                                 "#include <stdio.h>",
-                                 "#include <test_ctypes_platform.h>"],
-            include_dirs = [str(configdir)]
-        )
-
-        FILE = configure.Struct('FILE', [])
-        ushort = configure.SimpleType('unsigned short')
-        XYZZY = configure.ConstantInteger('XYZZY')
-
-    res = configure.configure(CConfig)
-    assert issubclass(res['FILE'], ctypes.Structure)
-    assert res == {'FILE': res['FILE'],
-                   'ushort': ctypes.c_ushort,
-                   'XYZZY': 42}
-
-def test_ifdef():
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            post_include_lines = ['/* a C comment */',
-                                  '#define XYZZY 42',
-                                  'typedef int foo;',
-                                  'struct s {',
-                                  'int i;',
-                                  'double f;'
-                                  '};'])
-
-
-        s = configure.Struct('struct s', [('i', ctypes.c_int)],
-                                   ifdef='XYZZY')
-        z = configure.Struct('struct z', [('i', ctypes.c_int)],
-                                   ifdef='FOOBAR')
-
-        foo = configure.SimpleType('foo', ifdef='XYZZY')
-        bar = configure.SimpleType('bar', ifdef='FOOBAR')
-
-    res = configure.configure(CConfig)
-    assert res['s'] is not None
-    assert res['z'] is None
-    assert res['foo'] is not None
-    assert res['bar'] is None
-
-def test_nested_structs():
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            post_include_lines="""
-            struct x {
-            int foo;
-            unsigned long bar;
-            };
-            struct y {
-            char c;
-            struct x x;
-            };
-            """.split("\n"))
-
-        x = configure.Struct("struct x", [("bar", ctypes.c_short)])
-        y = configure.Struct("struct y", [("x", x)])
-
-    res = configure.configure(CConfig)
-    c_x = res["x"]
-    c_y = res["y"]
-    c_y_fields = dict(c_y._fields_)
-    assert issubclass(c_x , ctypes.Structure)
-    assert issubclass(c_y, ctypes.Structure)
-    assert c_y_fields["x"] is c_x
-
-def test_array():
-    dirent = configure.getstruct("struct dirent",
-                                       """
-           struct dirent  /* for this example only, not the exact dirent */
-           {
-               long d_ino;
-               int d_off;
-               unsigned short d_reclen;
-               char d_name[32];
-           };
-                                       """,
-                                       [("d_name", ctypes.c_char * 0)])
-    assert dirent.d_name.size == 32
-
-def test_has():
-    assert configure.has("x", "int x = 3;")
-    assert not configure.has("x", "")
-    # has() should also not crash if it is given an invalid #include
-    assert not configure.has("x", "#include <some/path/which/cannot/exist>")
-
-def test_check_eci():
-    eci = ExternalCompilationInfo()
-    assert configure.check_eci(eci)
-    eci = ExternalCompilationInfo(libraries=['some_name_that_doesnt_exist_'])
-    assert not configure.check_eci(eci)
-
-def test_sizeof():
-    assert configure.sizeof("char", ExternalCompilationInfo()) == 1
-
-def test_memory_alignment():
-    a = configure.memory_alignment()
-    print a
-    assert a % struct.calcsize("P") == 0
diff --git a/ctypes_configure/test/test_dumpcache.py b/ctypes_configure/test/test_dumpcache.py
deleted file mode 100644
--- a/ctypes_configure/test/test_dumpcache.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import ctypes
-from ctypes_configure import configure, dumpcache
-from ctypes_configure.cbuild import ExternalCompilationInfo
-
-
-def test_cache():
-    configdir = configure.configdir
-    test_h = configdir.join('test_ctypes_platform2.h')
-    test_h.write('#define XYZZY 42\n'
-                 "#define large 2147483648L\n")
-
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            pre_include_lines = ["/* a C comment */",
-                                 "#include <stdio.h>",
-                                 "#include <test_ctypes_platform2.h>"],
-            include_dirs = [str(configdir)]
-        )
-
-        FILE = configure.Struct('FILE', [])
-        ushort = configure.SimpleType('unsigned short')
-        XYZZY = configure.ConstantInteger('XYZZY')
-        XUZ = configure.Has('XUZ')
-        large = configure.DefinedConstantInteger('large')
-        undef = configure.Defined('really_undefined')
-
-    res = configure.configure(CConfig)
-
-    cachefile = configdir.join('cache')
-    dumpcache.dumpcache('', str(cachefile), res)
-
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['XYZZY'] == res['XYZZY']
-    assert d['ushort'] == res['ushort']
-    assert d['FILE']._fields_ == res['FILE']._fields_
-    assert d['FILE'].__mro__[1:] == res['FILE'].__mro__[1:]
-    assert d['undef'] == res['undef']
-    assert d['large'] == res['large']
-    assert d['XUZ'] == res['XUZ']
-
-
-def test_cache_array():
-    configdir = configure.configdir
-    res = {'foo': ctypes.c_short * 27}
-    cachefile = configdir.join('cache_array')
-    dumpcache.dumpcache('', str(cachefile), res)
-    #
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['foo'] == res['foo']
-
-def test_cache_array_array():
-    configdir = configure.configdir
-    res = {'foo': (ctypes.c_int * 2) * 3}
-    cachefile = configdir.join('cache_array_array')
-    dumpcache.dumpcache('', str(cachefile), res)
-    #
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['foo'] == res['foo']
diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py
--- a/lib-python/2.7/SimpleXMLRPCServer.py
+++ b/lib-python/2.7/SimpleXMLRPCServer.py
@@ -188,7 +188,7 @@
         are considered private and will not be called by
         SimpleXMLRPCServer.
 
-        If a registered function matches a XML-RPC request, then it
+        If a registered function matches an XML-RPC request, then it
         will be called instead of the registered instance.
 
         If the optional allow_dotted_names argument is true and the
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -274,7 +274,7 @@
     Even though IOBase does not declare read, readinto, or write because
     their signatures will vary, implementations and clients should
     consider those methods part of the interface. Also, implementations
-    may raise a IOError when operations they do not support are called.
+    may raise an IOError when operations they do not support are called.
 
     The basic type used for binary data read from or written to a file is
     the bytes type. Method arguments may also be bytearray or memoryview of
diff --git a/lib-python/2.7/calendar.py b/lib-python/2.7/calendar.py
--- a/lib-python/2.7/calendar.py
+++ b/lib-python/2.7/calendar.py
@@ -174,22 +174,23 @@
         Like itermonthdates(), but will yield (day number, weekday number)
         tuples. For days outside the specified month the day number is 0.
         """
-        for date in self.itermonthdates(year, month):
-            if date.month != month:
-                yield (0, date.weekday())
-            else:
-                yield (date.day, date.weekday())
+        for i, d in enumerate(self.itermonthdays(year, month), self.firstweekday):
+            yield d, i % 7
 
     def itermonthdays(self, year, month):
         """
         Like itermonthdates(), but will yield day numbers. For days outside
         the specified month the day number is 0.
         """
-        for date in self.itermonthdates(year, month):
-            if date.month != month:
-                yield 0
-            else:
-                yield date.day
+        day1, ndays = monthrange(year, month)
+        days_before = (day1 - self.firstweekday) % 7
+        for _ in range(days_before):
+            yield 0
+        for d in range(1, ndays + 1):
+            yield d
+        days_after = (self.firstweekday - day1 - ndays) % 7
+        for _ in range(days_after):
+            yield 0
 
     def monthdatescalendar(self, year, month):
         """
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -21,7 +21,7 @@
 usage of the Chunk class defined here is to instantiate an instance at
 the start of each chunk and read from the instance until it reaches
 the end, after which a new instance can be instantiated.  At the end
-of the file, creating a new instance will fail with a EOFError
+of the file, creating a new instance will fail with an EOFError
 exception.
 
 Usage:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -252,7 +252,7 @@
     """
     def __init__(self, errors='strict'):
         """
-        Creates a IncrementalDecoder instance.
+        Creates an IncrementalDecoder instance.
 
         The IncrementalDecoder may use different error handling schemes by
         providing the errors keyword argument. See the module docstring
@@ -1012,7 +1012,7 @@
     """
     Encoding iterator.
 
-    Encodes the input strings from the iterator using a IncrementalEncoder.
+    Encodes the input strings from the iterator using an IncrementalEncoder.
 
     errors and kwargs are passed through to the IncrementalEncoder
     constructor.
@@ -1030,7 +1030,7 @@
     """
     Decoding iterator.
 
-    Decodes the input strings from the iterator using a IncrementalDecoder.
+    Decodes the input strings from the iterator using an IncrementalDecoder.
 
     errors and kwargs are passed through to the IncrementalDecoder
     constructor.
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -113,7 +113,7 @@
     """
     if t is None: t = time.time()
     year, mon, mday, hour, min, sec, wday = time.gmtime(t)[:7]
-    return "%s %02d-%s-%04d %02d:%02d:%02d GMT" % (
+    return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % (
         DAYS[wday], mday, MONTHS[mon-1], year, hour, min, sec)
 
 
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -1,3 +1,4 @@
+import functools
 import unittest
 from ctypes import *
 from ctypes.test import need_symbol
@@ -248,6 +249,40 @@
         self.assertEqual(result,
                          callback(1.1*1.1, 2.2*2.2, 3.3*3.3, 4.4*4.4, 5.5*5.5))
 
+    def test_callback_large_struct(self):
+        class Check: pass
+
+        class X(Structure):
+            _fields_ = [
+                ('first', c_ulong),
+                ('second', c_ulong),
+                ('third', c_ulong),
+            ]
+
+        def callback(check, s):
+            check.first = s.first
+            check.second = s.second
+            check.third = s.third
+
+        check = Check()
+        s = X()
+        s.first = 0xdeadbeef
+        s.second = 0xcafebabe
+        s.third = 0x0bad1dea
+
+        CALLBACK = CFUNCTYPE(None, X)
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_cbk_large_struct
+        func.argtypes = (X, CALLBACK)
+        func.restype = None
+        # the function just calls the callback with the passed structure
+        func(s, CALLBACK(functools.partial(callback, check)))
+        self.assertEqual(check.first, s.first)
+        self.assertEqual(check.second, s.second)
+        self.assertEqual(check.third, s.third)
+        self.assertEqual(check.first, 0xdeadbeef)
+        self.assertEqual(check.second, 0xcafebabe)
+        self.assertEqual(check.third, 0x0bad1dea)
 
 ################################################################
 
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -1,6 +1,7 @@
 import unittest
-import os
+import os.path
 import sys
+from test import test_support
 from ctypes import *
 from ctypes.util import find_library
 from ctypes.test import is_resource_enabled
@@ -65,28 +66,10 @@
         if self.gle:
             self.gle.gleGetJoinStyle
 
-# On platforms where the default shared library suffix is '.so',
-# at least some libraries can be loaded as attributes of the cdll
-# object, since ctypes now tries loading the lib again
-# with '.so' appended of the first try fails.
-#
-# Won't work for libc, unfortunately.  OTOH, it isn't
-# needed for libc since this is already mapped into the current
-# process (?)
-#
-# On MAC OSX, it won't work either, because dlopen() needs a full path,
-# and the default suffix is either none or '.dylib'.
- at unittest.skip('test disabled')
- at unittest.skipUnless(os.name=="posix" and sys.platform != "darwin",
-                     'test not suitable for this platform')
-class LoadLibs(unittest.TestCase):
-    def test_libm(self):
-        import math
-        libm = cdll.libm
-        sqrt = libm.sqrt
-        sqrt.argtypes = (c_double,)
-        sqrt.restype = c_double
-        self.assertEqual(sqrt(2), math.sqrt(2))
+    def test_shell_injection(self):
+        result = find_library('; echo Hello shell > ' + test_support.TESTFN)
+        self.assertFalse(os.path.lexists(test_support.TESTFN))
+        self.assertIsNone(result)
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -32,7 +32,7 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises((TypeError, ValueError),
+        self.assertRaises(TypeError,
                           (c_char * 16).from_buffer, "a" * 16)
 
     def test_fom_buffer_with_offset(self):
@@ -77,5 +77,13 @@
         self.assertRaises(ValueError,
                           (c_int * 1).from_buffer_copy, a, 16 * sizeof(c_int))
 
+    def test_abstract(self):
+        self.assertRaises(TypeError, Array.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, Structure.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, Union.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, Array.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, Structure.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, Union.from_buffer_copy, b"123")
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py
--- a/lib-python/2.7/ctypes/test/test_numbers.py
+++ b/lib-python/2.7/ctypes/test/test_numbers.py
@@ -77,7 +77,7 @@
             self.assertEqual(t(v).value, truth(v))
 
     def test_typeerror(self):
-        # Only numbers are allowed in the contructor,
+        # Only numbers are allowed in the constructor,
         # otherwise TypeError is raised
         for t in signed_types + unsigned_types + float_types:
             self.assertRaises(TypeError, t, "")
diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py
--- a/lib-python/2.7/ctypes/test/test_structures.py
+++ b/lib-python/2.7/ctypes/test/test_structures.py
@@ -106,7 +106,7 @@
         self.assertEqual(alignment(XX), alignment(X))
         self.assertEqual(sizeof(XX), calcsize("3s 3s 0s"))
 
-    def test_emtpy(self):
+    def test_empty(self):
         # I had problems with these
         #
         # Although these are pathological cases: Empty Structures!
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -1,4 +1,6 @@
-import sys, os
+import os
+import subprocess
+import sys
 
 # find_library(name) returns the pathname of a library, or None.
 if os.name == "nt":
@@ -87,25 +89,28 @@
 
     def _findLib_gcc(name):
         import tempfile
+        # Run GCC's linker with the -t (aka --trace) option and examine the
+        # library name it prints out. The GCC command will fail because we
+        # haven't supplied a proper program with main(), but that does not
+        # matter.
         expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
-        fdout, ccout = tempfile.mkstemp()
-        os.close(fdout)
-        cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; elif type cc >/dev/null 2>&1; then CC=cc;else exit 10; fi;' \
-              'LANG=C LC_ALL=C $CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name
+        cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; elif type cc >/dev/null 2>&1; then CC=cc;else exit; fi;' \
+              'LANG=C LC_ALL=C $CC -Wl,-t -o "$2" 2>&1 -l"$1"'
+
+        temp = tempfile.NamedTemporaryFile()
         try:
-            f = os.popen(cmd)
-            try:
-                trace = f.read()
-            finally:
-                rv = f.close()
+            proc = subprocess.Popen((cmd, '_findLib_gcc', name, temp.name),
+                                    shell=True,
+                                    stdout=subprocess.PIPE)
+            [trace, _] = proc.communicate()
         finally:
             try:
-                os.unlink(ccout)
+                temp.close()
             except OSError, e:
+                # ENOENT is raised if the file was already removed, which is
+                # the normal behaviour of GCC if linking fails
                 if e.errno != errno.ENOENT:
                     raise
-        if rv == 10:
-            raise OSError, 'gcc or cc command not found'
         res = re.search(expr, trace)
         if not res:
             return None
@@ -117,13 +122,17 @@
         def _get_soname(f):
             if not f:
                 return None
-            cmd = "/usr/ccs/bin/dump -Lpv 2>/dev/null " + f
-            f = os.popen(cmd)
+
+            null = open(os.devnull, "wb")
             try:
-                data = f.read()
-            finally:
-                f.close()
-            res = re.search(r'\[.*\]\sSONAME\s+([^\s]+)', data)
+                with null:
+                    proc = subprocess.Popen(("/usr/ccs/bin/dump", "-Lpv", f),
+                                            stdout=subprocess.PIPE,
+                                            stderr=null)
+            except OSError:  # E.g. command not found
+                return None
+            [data, _] = proc.communicate()
+            res = re.search(br'\[.*\]\sSONAME\s+([^\s]+)', data)
             if not res:
                 return None
             return res.group(1)
@@ -132,16 +141,12 @@
             # assuming GNU binutils / ELF
             if not f:
                 return None
-            cmd = 'if ! type objdump >/dev/null 2>&1; then exit 10; fi;' \
-                  "objdump -p -j .dynamic 2>/dev/null " + f
-            f = os.popen(cmd)

From pypy.commits at gmail.com  Fri Jan  6 08:43:52 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 06 Jan 2017 05:43:52 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove unused function
Message-ID: <586f9f18.c515c20a.a3d17.149d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89393:366bc91ff0ea
Date: 2016-12-16 16:38 +0000
http://bitbucket.org/pypy/pypy/changeset/366bc91ff0ea/

Log:	Remove unused function

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -617,14 +617,6 @@
                              % (cpyname, ))
 build_exported_objects()
 
-def get_structtype_for_ctype(ctype):
-    from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
-    from pypy.module.cpyext.cdatetime import PyDateTime_CAPI
-    from pypy.module.cpyext.intobject import PyIntObject
-    return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr,
-            "PyIntObject*": PyIntObject,
-            "PyDateTime_CAPI*": lltype.Ptr(PyDateTime_CAPI)}[ctype]
-
 # Note: as a special case, "PyObject" is the pointer type in RPython,
 # corresponding to "PyObject *" in C.  We do that only for PyObject.
 # For example, "PyTypeObject" is the struct type even in RPython.

From pypy.commits at gmail.com  Fri Jan  6 08:50:12 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 06 Jan 2017 05:50:12 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-from2: translation fixes
Message-ID: <586fa094.c5371c0a.12a8b.31aa@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-from2
Changeset: r89394:09411e996fff
Date: 2017-01-06 15:45 +0200
http://bitbucket.org/pypy/pypy/changeset/09411e996fff/

Log:	translation fixes

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -35,12 +35,13 @@
     """
     Fills a newly allocated PyMemoryViewObject with the given W_MemoryView object.
     """
+    assert isinstance(w_obj, W_MemoryView)
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
     view = py_obj.c_view
     ndim = w_obj.buf.getndim()
     if ndim >= Py_MAX_NDIMS:
         # XXX warn?
-        return view
+        return
     fill_Py_buffer(space, w_obj.buf, view)
     try:
         view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
@@ -108,7 +109,7 @@
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
     # XXX move to a c-macro
-    py_memobj = rffi.cast(PyMemoryViewObject, pyobj) # no inc_ref
+    py_memobj = rffi.cast(PyMemoryViewObject, pyobj)
     return py_memobj.c_view
 
 def fill_Py_buffer(space, buf, view):

From pypy.commits at gmail.com  Fri Jan  6 08:50:14 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 06 Jan 2017 05:50:14 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-from2: merge default into branch
Message-ID: <586fa096.92ae1c0a.76308.3153@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-from2
Changeset: r89395:7e0eb703936d
Date: 2017-01-06 15:46 +0200
http://bitbucket.org/pypy/pypy/changeset/7e0eb703936d/

Log:	merge default into branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -87,3 +87,7 @@
 Implement StringBuffer.get_raw_address (missing feature for the buffer protocol).
 More generally it is now possible to obtain the address of any object (if it
 is readonly) without pinning it.
+
+.. branch: cpyext-cleanup
+
+Refactor cpyext initialisation.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -219,7 +219,7 @@
             # cpyext types that may have only old buffer interface
             w_impl = space.lookup(self, '__wbuffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self, 
+            w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
@@ -665,9 +665,6 @@
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
-        if self.config.objspace.usemodules.cpyext:
-            from pypy.module.cpyext.state import State
-            self.fromcache(State).build_api(self)
         self.getbuiltinmodule('sys')
         self.getbuiltinmodule('imp')
         self.getbuiltinmodule('__builtin__')
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -1,5 +1,4 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.interpreter import gateway
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
 
@@ -13,12 +12,17 @@
 
     atexit_funcs = []
 
+    def setup_after_space_initialization(self):
+        state = self.space.fromcache(State)
+        state.setup_rawrefcount()
+        state.build_api()
+
     def startup(self, space):
         space.fromcache(State).startup(space)
         method = pypy.module.cpyext.typeobject.get_new_method_def(space)
         w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap(''))
         space.appexec([space.type(w_obj)], """(methodtype):
-            from pickle import Pickler 
+            from pickle import Pickler
             Pickler.dispatch[methodtype] = Pickler.save_global
         """)
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1,5 +1,6 @@
 import ctypes
 import sys, os
+from collections import defaultdict
 
 import py
 
@@ -71,7 +72,6 @@
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
-VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list')
 CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
                                        hints={'nolength': True}),
                           use_cache=False)
@@ -364,8 +364,8 @@
         func_name = func.func_name
         if header is not None:
             c_name = None
-            assert func_name not in FUNCTIONS, (
-                "%s already registered" % func_name)
+            if func_name in FUNCTIONS_BY_HEADER[header]:
+                raise ValueError("%s already registered" % func_name)
         else:
             c_name = func_name
         api_function = ApiFunction(argtypes, restype, func, error,
@@ -463,9 +463,7 @@
             return res
 
         if header is not None:
-            if header == DEFAULT_HEADER:
-                FUNCTIONS[func_name] = api_function
-            FUNCTIONS_BY_HEADER.setdefault(header, {})[func_name] = api_function
+            FUNCTIONS_BY_HEADER[header][func_name] = api_function
         INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
         return unwrapper  # used in 'normal' RPython code.
     return decorate
@@ -489,8 +487,7 @@
     GLOBALS[name] = (typ, expr)
 
 INTERPLEVEL_API = {}
-FUNCTIONS = {}
-FUNCTIONS_BY_HEADER = {}
+FUNCTIONS_BY_HEADER = defaultdict(dict)
 
 # These are C symbols which cpyext will export, but which are defined in .c
 # files somewhere in the implementation of cpyext (rather than being defined in
@@ -526,6 +523,8 @@
     'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor',
     'PyCapsule_SetContext', 'PyCapsule_Import', 'PyCapsule_Type', '_Py_get_capsule_type',
 
+    'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
+
     'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
 
     'PyOS_getsig', 'PyOS_setsig',
@@ -572,7 +571,9 @@
     # PyExc_AttributeError, PyExc_OverflowError, PyExc_ImportError,
     # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError,
     # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ...
-    for exc_name in exceptions.Module.interpleveldefs.keys():
+    global all_exceptions
+    all_exceptions = list(exceptions.Module.interpleveldefs)
+    for exc_name in all_exceptions:
         register_global('PyExc_' + exc_name,
             'PyTypeObject*',
             'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, ))
@@ -616,14 +617,6 @@
                              % (cpyname, ))
 build_exported_objects()
 
-def get_structtype_for_ctype(ctype):
-    from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
-    from pypy.module.cpyext.cdatetime import PyDateTime_CAPI
-    from pypy.module.cpyext.intobject import PyIntObject
-    return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr,
-            "PyIntObject*": PyIntObject,
-            "PyDateTime_CAPI*": lltype.Ptr(PyDateTime_CAPI)}[ctype]
-
 # Note: as a special case, "PyObject" is the pointer type in RPython,
 # corresponding to "PyObject *" in C.  We do that only for PyObject.
 # For example, "PyTypeObject" is the struct type even in RPython.
@@ -672,12 +665,6 @@
 # a pointer to PyObject
 PyObjectP = rffi.CArrayPtr(PyObject)
 
-VA_TP_LIST = {}
-#{'int': lltype.Signed,
-#              'PyObject*': PyObject,
-#              'PyObject**': PyObjectP,
-#              'int*': rffi.INTP}
-
 def configure_types():
     for config in (CConfig, CConfig2):
         for name, TYPE in rffi_platform.configure(config).iteritems():
@@ -951,21 +938,8 @@
     wrapper_second_level._dont_inline_ = True
     return wrapper_second_level
 
-def process_va_name(name):
-    return name.replace('*', '_star')
 
-def setup_va_functions(eci):
-    for name, TP in VA_TP_LIST.iteritems():
-        name_no_star = process_va_name(name)
-        func = rffi.llexternal('pypy_va_get_%s' % name_no_star, [VA_LIST_P],
-                               TP, compilation_info=eci)
-        globals()['va_get_%s' % name_no_star] = func
-
-def setup_init_functions(eci, translating):
-    if translating:
-        prefix = 'PyPy'
-    else:
-        prefix = 'cpyexttest'
+def setup_init_functions(eci, prefix):
     # jump through hoops to avoid releasing the GIL during initialization
     # of the cpyext module.  The C functions are called with no wrapper,
     # but must not do anything like calling back PyType_Ready().  We
@@ -1027,23 +1001,27 @@
 # Do not call this more than once per process
 def build_bridge(space):
     "NOT_RPYTHON"
-    from pypy.module.cpyext.pyobject import make_ref
     from rpython.translator.c.database import LowLevelDatabase
     use_micronumpy = setup_micronumpy(space)
     db = LowLevelDatabase()
-    prefix ='cpyexttest'
+    prefix = 'cpyexttest'
 
-    functions = generate_decls_and_callbacks(db, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
 
     # Structure declaration code
+    functions = []
     members = []
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                # added only for the macro, not the decl
-                continue
             restype, args = c_function_signature(db, func)
+            callargs = ', '.join('arg%d' % (i,)
+                                for i in range(len(func.argtypes)))
+            if func.restype is lltype.Void:
+                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+            else:
+                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
             members.append('%s (*%s)(%s);' % (restype, name, args))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
@@ -1055,15 +1033,8 @@
     """ % dict(members=structmembers)
 
     global_objects = []
-    for name, (typ, expr) in GLOBALS.iteritems():
-        if '#' in name:
-            continue
-        if typ == 'PyDateTime_CAPI*':
-            continue
-        elif name.startswith('PyExc_'):
-            global_objects.append('%s _%s;' % (typ[:-1], name))
-        else:
-            global_objects.append('%s %s = NULL;' % (typ, name))
+    for name in all_exceptions:
+        global_objects.append('PyTypeObject _PyExc_%s;' % name)
     global_code = '\n'.join(global_objects)
 
     prologue = ("#include <Python.h>\n"
@@ -1080,90 +1051,69 @@
             '\n' +
             '\n'.join(functions))
 
-    eci = build_eci(True, code, use_micronumpy)
+    eci = build_eci(code, use_micronumpy, translating=False)
     eci = eci.compile_shared_lib(
         outputfilename=str(udir / "module_cache" / "pypyapi"))
+    space.fromcache(State).install_dll(eci)
     modulename = py.path.local(eci.libraries[-1])
 
-    def dealloc_trigger():
-        from pypy.module.cpyext.pyobject import decref
-        print 'dealloc_trigger...'
-        while True:
-            ob = rawrefcount.next_dead(PyObject)
-            if not ob:
-                break
-            print 'deallocating PyObject', ob
-            decref(space, ob)
-        print 'dealloc_trigger DONE'
-        return "RETRY"
-    rawrefcount.init(dealloc_trigger)
-
     run_bootstrap_functions(space)
 
     # load the bridge, and init structure
     bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
 
-    space.fromcache(State).install_dll(eci)
+    # populate static data
+    builder = space.fromcache(State).builder = TestingObjBuilder()
+    for name, (typ, expr) in GLOBALS.iteritems():
+        if '#' in name:
+            name, header = name.split('#')
+            assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
+            isptr = False
+        elif name.startswith('PyExc_'):
+            isptr = False
+        elif typ == 'PyDateTime_CAPI*':
+            isptr = True
+        else:
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
 
-    # populate static data
-    builder = space.fromcache(StaticObjectBuilder)
-    for name, (typ, expr) in GLOBALS.iteritems():
         from pypy.module import cpyext    # for the eval() below
         w_obj = eval(expr)
-        if '#' in name:
-            name = name.split('#')[0]
-            isptr = False
-        else:
-            isptr = True
-        if name.startswith('PyExc_'):
-            isptr = False
-
         INTERPLEVEL_API[name] = w_obj
 
-        name = name.replace('Py', prefix)
+        mname = mangle_name(prefix, name)
         if isptr:
-            ptr = ctypes.c_void_p.in_dll(bridge, name)
-            if typ == 'PyObject*':
-                value = make_ref(space, w_obj)
-            elif typ == 'PyDateTime_CAPI*':
-                value = w_obj
-            else:
-                assert False, "Unknown static pointer: %s %s" % (typ, name)
+            assert typ == 'PyDateTime_CAPI*'
+            value = w_obj
+            ptr = ctypes.c_void_p.in_dll(bridge, mname)
             ptr.value = ctypes.cast(ll2ctypes.lltype2ctypes(value),
                                     ctypes.c_void_p).value
-        elif typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*'):
-            if name.startswith('PyPyExc_') or name.startswith('cpyexttestExc_'):
+        else:
+            if name.startswith('PyExc_'):
                 # we already have the pointer
-                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, in_dll)
             else:
                 # we have a structure, get its address
-                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, ctypes.pointer(in_dll))
             builder.prepare(py_obj, w_obj)
-        else:
-            assert False, "Unknown static object: %s %s" % (typ, name)
-    builder.attach_all()
+    builder.attach_all(space)
 
     pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
 
     # implement structure initialization code
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if name.startswith('cpyext_') or func is None: # XXX hack
-                continue
             pypyAPI[structindex[name]] = ctypes.cast(
                 ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
                 ctypes.c_void_p)
-    setup_va_functions(eci)
 
-    setup_init_functions(eci, translating=False)
+    setup_init_functions(eci, prefix)
     return modulename.new(ext='')
 
 
-class StaticObjectBuilder:
-    def __init__(self, space):
-        self.space = space
+class StaticObjectBuilder(object):
+    def __init__(self):
         self.static_pyobjs = []
         self.static_objs_w = []
         self.cpyext_type_init = None
@@ -1179,13 +1129,12 @@
         self.static_pyobjs.append(py_obj)
         self.static_objs_w.append(w_obj)
 
-    def attach_all(self):
+    def attach_all(self, space):
         # this is RPython, called once in pypy-c when it imports cpyext
         from pypy.module.cpyext.pyobject import get_typedescr, make_ref
         from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
         from pypy.module.cpyext.pyobject import track_reference
         #
-        space = self.space
         static_pyobjs = self.get_static_pyobjs()
         static_objs_w = self.static_objs_w
         for i in range(len(static_objs_w)):
@@ -1206,6 +1155,12 @@
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
+class TestingObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used in tests."""
+
+class TranslationObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used during translation."""
+
 
 def mangle_name(prefix, name):
     if name.startswith('Py'):
@@ -1213,23 +1168,26 @@
     elif name.startswith('_Py'):
         return '_' + prefix + name[3:]
     else:
-        return None
+        raise ValueError("Error converting '%s'" % name)
 
-def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
+def write_header(header_name, decls):
+    lines = [
+        '#define Signed   long           /* xxx temporary fix */',
+        '#define Unsigned unsigned long  /* xxx temporary fix */',
+        '',] + decls + [
+        '',
+        '#undef Signed    /* xxx temporary fix */',
+        '#undef Unsigned  /* xxx temporary fix */',
+        '']
+    decl_h = udir.join(header_name)
+    decl_h.write('\n'.join(lines))
+
+def generate_decls_and_callbacks(db, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
-    export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
-    for name in export_symbols:
-        if '#' in name:
-            name, header = name.split('#')
-        else:
-            header = pypy_decl
+    for name in SYMBOLS_C:
         newname = mangle_name(prefix, name)
-        assert newname, name
-        if header == pypy_decl:
-            pypy_macros.append('#define %s %s' % (name, newname))
-        if name.startswith("PyExc_"):
-            pypy_macros.append('#define _%s _%s' % (name, newname))
+        pypy_macros.append('#define %s %s' % (name, newname))
 
     # Generate defines
     for macro_name, size in [
@@ -1249,50 +1207,18 @@
     pypy_macros_h = udir.join('pypy_macros.h')
     pypy_macros_h.write('\n'.join(pypy_macros))
 
-    # implement function callbacks and generate function decls
-    functions = []
-    decls = {}
-    pypy_decls = decls[pypy_decl] = []
-    pypy_decls.append('#define Signed   long           /* xxx temporary fix */\n')
-    pypy_decls.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-
+    # generate function decls
+    decls = defaultdict(list)
     for decl in FORWARD_DECLS:
-        pypy_decls.append("%s;" % (decl,))
+        decls[pypy_decl].append("%s;" % (decl,))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
-        if header_name not in decls:
-            header = decls[header_name] = []
-            header.append('#define Signed   long           /* xxx temporary fix */\n')
-            header.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-        else:
-            header = decls[header_name]
-
+        header = decls[header_name]
         for name, func in sorted(header_functions.iteritems()):
-            if not func:
-                continue
-            if header == DEFAULT_HEADER:
-                _name = name
-            else:
-                # this name is not included in pypy_macros.h
-                _name = mangle_name(prefix, name)
-                assert _name is not None, 'error converting %s' % name
-                header.append("#define %s %s" % (name, _name))
+            _name = mangle_name(prefix, name)
+            header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, _name, args))
-            if api_struct:
-                callargs = ', '.join('arg%d' % (i,)
-                                    for i in range(len(func.argtypes)))
-                if func.restype is lltype.Void:
-                    body = "{ _pypyAPI.%s(%s); }" % (_name, callargs)
-                else:
-                    body = "{ return _pypyAPI.%s(%s); }" % (_name, callargs)
-                functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
-    for name in VA_TP_LIST:
-        name_no_star = process_va_name(name)
-        header = ('%s pypy_va_get_%s(va_list* vp)' %
-                  (name, name_no_star))
-        pypy_decls.append('RPY_EXTERN ' + header + ';')
-        functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
+            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
 
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
@@ -1301,19 +1227,11 @@
         elif name.startswith('PyExc_'):
             typ = 'PyObject*'
             header = pypy_decl
-        if header != pypy_decl:
-            decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
+        decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
         decls[header].append('PyAPI_DATA(%s) %s;' % (typ, name))
 
-    for header_name in FUNCTIONS_BY_HEADER.keys():
-        header = decls[header_name]
-        header.append('#undef Signed    /* xxx temporary fix */\n')
-        header.append('#undef Unsigned  /* xxx temporary fix */\n')
-
     for header_name, header_decls in decls.iteritems():
-        decl_h = udir.join(header_name)
-        decl_h.write('\n'.join(header_decls))
-    return functions
+        write_header(header_name, header_decls)
 
 separate_module_files = [source_dir / "varargwrapper.c",
                          source_dir / "pyerrors.c",
@@ -1325,6 +1243,7 @@
                          source_dir / "pythonrun.c",
                          source_dir / "sysmodule.c",
                          source_dir / "bufferobject.c",
+                         source_dir / "complexobject.c",
                          source_dir / "cobject.c",
                          source_dir / "structseq.c",
                          source_dir / "capsule.c",
@@ -1334,14 +1253,16 @@
                          source_dir / "pymem.c",
                          ]
 
-def build_eci(building_bridge, code, use_micronumpy=False):
+def build_eci(code, use_micronumpy=False, translating=False):
     "NOT_RPYTHON"
     # Build code and get pointer to the structure
     kwds = {}
 
     compile_extra=['-DPy_BUILD_CORE']
 
-    if building_bridge:
+    if translating:
+        kwds["includes"] = ['Python.h'] # this is our Python.h
+    else:
         if sys.platform == "win32":
             # '%s' undefined; assuming extern returning int
             compile_extra.append("/we4013")
@@ -1351,8 +1272,6 @@
         elif sys.platform.startswith('linux'):
             compile_extra.append("-Werror=implicit-function-declaration")
             compile_extra.append('-g')
-    else:
-        kwds["includes"] = ['Python.h'] # this is our Python.h
 
     # Generate definitions for global structures
     structs = ["#include <Python.h>"]
@@ -1403,9 +1322,7 @@
         return use_micronumpy
     # import registers api functions by side-effect, we also need HEADER
     from pypy.module.cpyext.ndarrayobject import HEADER
-    global FUNCTIONS_BY_HEADER, separate_module_files
-    for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
-        FUNCTIONS_BY_HEADER.setdefault(HEADER, {})[func_name] = None
+    global separate_module_files
     register_global("PyArray_Type",
         'PyTypeObject*',  "space.gettypeobject(W_NDimArray.typedef)",
         header=HEADER)
@@ -1419,21 +1336,19 @@
     db = LowLevelDatabase()
     prefix = 'PyPy'
 
-    functions = generate_decls_and_callbacks(db, api_struct=False, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
+
     code = "#include <Python.h>\n"
     if use_micronumpy:
         code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
-    code  += "\n".join(functions)
 
-    eci = build_eci(False, code, use_micronumpy)
-
+    eci = build_eci(code, use_micronumpy, translating=True)
     space.fromcache(State).install_dll(eci)
 
     run_bootstrap_functions(space)
-    setup_va_functions(eci)
 
     # emit uninitialized static data
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder = TranslationObjBuilder()
     lines = ['PyObject *pypy_static_pyobjs[] = {\n']
     include_lines = ['RPY_EXTERN PyObject *pypy_static_pyobjs[];\n']
     for name, (typ, expr) in sorted(GLOBALS.items()):
@@ -1441,17 +1356,15 @@
             name, header = name.split('#')
             assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
             typ = typ[:-1]
-            if header != pypy_decl:
-                # since the #define is not in pypy_macros, do it here
-                mname = mangle_name(prefix, name)
-                include_lines.append('#define %s %s\n' % (name, mname))
+            mname = mangle_name(prefix, name)
+            include_lines.append('#define %s %s\n' % (name, mname))
         elif name.startswith('PyExc_'):
             typ = 'PyTypeObject'
             name = '_' + name
         elif typ == 'PyDateTime_CAPI*':
             continue
         else:
-            assert False, "Unknown static data: %s %s" % (typ, name)
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
 
         from pypy.module import cpyext     # for the eval() below
         w_obj = eval(expr)
@@ -1471,20 +1384,15 @@
 
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                continue
-            newname = mangle_name('PyPy', name) or name
+            newname = mangle_name(prefix, name)
             deco = entrypoint_lowlevel("cpyext", func.argtypes, newname,
                                         relax=True)
             deco(func.get_wrapper(space))
 
-    setup_init_functions(eci, translating=True)
+    setup_init_functions(eci, prefix)
     trunk_include = pypydir.dirpath() / 'include'
     copy_header_files(trunk_include, use_micronumpy)
 
-def init_static_data_translated(space):
-    builder = space.fromcache(StaticObjectBuilder)
-    builder.attach_all()
 
 def _load_from_cffi(space, name, path, initptr):
     from pypy.module._cffi_backend import cffi1_module
diff --git a/pypy/module/cpyext/include/complexobject.h b/pypy/module/cpyext/include/complexobject.h
--- a/pypy/module/cpyext/include/complexobject.h
+++ b/pypy/module/cpyext/include/complexobject.h
@@ -16,23 +16,8 @@
     Py_complex cval;
 } PyComplexObject;
 
-/* generated function */
-PyAPI_FUNC(int) _PyComplex_AsCComplex(PyObject *, Py_complex *);
-PyAPI_FUNC(PyObject *) _PyComplex_FromCComplex(Py_complex *);
-
-Py_LOCAL_INLINE(Py_complex) PyComplex_AsCComplex(PyObject *obj)
-{
-    Py_complex result;
-    _PyComplex_AsCComplex(obj, &result);
-    return result;
-}
-
-// shmuller 2013/07/30: Make a function, since macro will fail in C++ due to 
-//                      const correctness if called with "const Py_complex"
-//#define PyComplex_FromCComplex(c) _PyComplex_FromCComplex(&c)
-Py_LOCAL_INLINE(PyObject *) PyComplex_FromCComplex(Py_complex c) {
-    return _PyComplex_FromCComplex(&c);
-}
+PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *obj);
+PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex c);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl, Py_buffer, Py_bufferP)
+    pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
@@ -351,7 +351,7 @@
         else:
             #do not call twice
             return
-        if self.releasebufferproc:        
+        if self.releasebufferproc:
             func_target = rffi.cast(releasebufferproc, self.releasebufferproc)
             with lltype.scoped_alloc(Py_buffer) as pybuf:
                 pybuf.c_buf = self.ptr
@@ -416,7 +416,7 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self, 
+        buf = CPyBuffer(space, ptr[0], size, w_self,
                                releasebuffer=releasebuffer)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
@@ -522,7 +522,7 @@
     w_type = space.gettypeobject(typedef)
 
     header = pypy_decl
-    if mangle_name('', typedef.name) is None:
+    if not (name.startswith('Py') or name.startswith('_Py')):
         header = None
     handled = False
     # unary functions
diff --git a/pypy/module/cpyext/src/complexobject.c b/pypy/module/cpyext/src/complexobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/complexobject.c
@@ -0,0 +1,16 @@
+
+#include "Python.h"
+
+Py_complex
+PyComplex_AsCComplex(PyObject *obj)
+{
+    Py_complex result;
+    _PyComplex_AsCComplex(obj, &result);
+    return result;
+}
+
+PyObject *
+PyComplex_FromCComplex(Py_complex c)
+{
+    return _PyComplex_FromCComplex(&c);
+}
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -2,7 +2,6 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import executioncontext
-from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
@@ -14,9 +13,7 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
-        if space.config.translation.gc != "boehm":
-            pyobj_dealloc_action = PyObjDeallocAction(space)
-            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+        self.builder = None
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -57,22 +54,40 @@
                         "Function returned an error result without setting an "
                         "exception")
 
-    def build_api(self, space):
+    def setup_rawrefcount(self):
+        space = self.space
+        if not self.space.config.translating:
+            def dealloc_trigger():
+                from pypy.module.cpyext.pyobject import PyObject, decref
+                print 'dealloc_trigger...'
+                while True:
+                    ob = rawrefcount.next_dead(PyObject)
+                    if not ob:
+                        break
+                    print 'deallocating PyObject', ob
+                    decref(space, ob)
+                print 'dealloc_trigger DONE'
+                return "RETRY"
+            rawrefcount.init(dealloc_trigger)
+        else:
+            if space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(space)
+                space.actionflag.register_periodic_action(action,
+                    use_bytecode_counter=True)
+            else:
+                pyobj_dealloc_action = PyObjDeallocAction(space)
+                self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+
+    def build_api(self):
         """NOT_RPYTHON
         This function is called when at object space creation,
         and drives the compilation of the cpyext library
         """
         from pypy.module.cpyext import api
-        state = self.space.fromcache(State)
         if not self.space.config.translating:
-            state.api_lib = str(api.build_bridge(self.space))
+            self.api_lib = str(api.build_bridge(self.space))
         else:
             api.setup_library(self.space)
-            #
-            if self.space.config.translation.gc == "boehm":
-                action = BoehmPyObjDeallocAction(self.space)
-                self.space.actionflag.register_periodic_action(action,
-                    use_bytecode_counter=True)
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -84,17 +99,17 @@
 
     def startup(self, space):
         "This function is called when the program really starts"
-
         from pypy.module.cpyext.typeobject import setup_new_method_def
         from pypy.module.cpyext.api import INIT_FUNCTIONS
-        from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
             if space.config.translation.gc != "boehm":
+                # This must be called in RPython, the untranslated version
+                # does something different. Sigh.
                 rawrefcount.init(
                     llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
                     self.dealloc_trigger))
-            init_static_data_translated(space)
+            self.builder.attach_all(space)
 
         setup_new_method_def(space)
 
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,9 +89,9 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest1'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
                 == ('Py_ssize_t', 'Py_ssize_t arg0'))
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest2'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
                 == ('Py_ssize_t *', 'Py_ssize_t *arg0'))
 
         PyPy_TypedefTest1(space, 0)
@@ -100,7 +100,7 @@
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
 
- at pytest.mark.skipif(os.environ.get('USER')=='root', 
+ at pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
     api.copy_header_files(tmpdir, True)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -27,8 +27,9 @@
 
 class TestApi:
     def test_signature(self):
-        assert 'PyModule_Check' in api.FUNCTIONS
-        assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
+        common_functions = api.FUNCTIONS_BY_HEADER[api.pypy_decl]
+        assert 'PyModule_Check' in common_functions
+        assert common_functions['PyModule_Check'].argtypes == [api.PyObject]
 
 
 class SpaceCompiler(SystemCompilationInfo):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -87,7 +87,7 @@
     py_getsetdef.c_set = rffi.cast(setter, 0)
     py_getsetdef.c_closure = rffi.cast(rffi.VOIDP, 0)
     return py_getsetdef
-    
+
 
 class W_MemberDescr(GetSetProperty):
     name = 'member_descriptor'
@@ -711,7 +711,7 @@
 
     pto.c_tp_free = llslot(space, PyObject_Free)
     pto.c_tp_alloc = llslot(space, PyType_GenericAlloc)
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder
     if ((pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE) != 0
             and builder.cpyext_type_init is None):
             # this ^^^ is not None only during startup of cpyext.  At that
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -548,6 +548,10 @@
 
         sub = self._op_val(space, w_old)
         by = self._op_val(space, w_new)
+        # the following two lines are for being bug-to-bug compatible
+        # with CPython: see issue #2448
+        if count >= 0 and len(input) == 0:
+            return self._empty()
         try:
             res = replace(input, sub, by, count)
         except OverflowError:
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -809,6 +809,16 @@
         s = b"a" * (2**16)
         raises(OverflowError, s.replace, b"", s)
 
+    def test_replace_issue2448(self):
+        # CPython's replace() method has a bug that makes
+        #   ''.replace('', 'x')  gives a different answer than
+        #   ''.replace('', 'x', 1000).  This is the case in all
+        # known versions, at least until 2.7.13.  Some people
+        # call that a feature on the CPython issue report and
+        # the discussion dies out, so it might never be fixed.
+        assert ''.replace('', 'x') == 'x'
+        assert ''.replace('', 'x', 1000) == ''
+
     def test_getslice(self):
         assert "foobar".__getslice__(4, 4321) == "ar"
         s = b"abc"
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1770,11 +1770,11 @@
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
-            pinned_obj_size = llarena.getfakearenaaddress(cur) - prev
+            free_range_size = llarena.getfakearenaaddress(cur) - prev
             if self.gc_nursery_debug:
-                llarena.arena_reset(prev, pinned_obj_size, 3)
+                llarena.arena_reset(prev, free_range_size, 3)
             else:
-                llarena.arena_reset(prev, pinned_obj_size, 0)
+                llarena.arena_reset(prev, free_range_size, 0)
             #
             # clean up object's flags
             obj = cur + size_gc_header
@@ -1784,7 +1784,7 @@
             nursery_barriers.append(cur)
             #
             # update 'prev' to the end of the 'cur' object
-            prev = prev + pinned_obj_size + \
+            prev = prev + free_range_size + \
                 (size_gc_header + self.get_size(obj))
         #
         # reset everything after the last pinned object till the end of the arena

From pypy.commits at gmail.com  Fri Jan  6 09:06:01 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 06 Jan 2017 06:06:01 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <586fa449.c515c20a.a3d17.1c51@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89396:d1617cb7e740
Date: 2017-01-06 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/d1617cb7e740/

Log:	merge default into branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -87,3 +87,7 @@
 Implement StringBuffer.get_raw_address (missing feature for the buffer protocol).
 More generally it is now possible to obtain the address of any object (if it
 is readonly) without pinning it.
+
+.. branch: cpyext-cleanup
+
+Refactor cpyext initialisation.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -219,7 +219,7 @@
             # cpyext types that may have only old buffer interface
             w_impl = space.lookup(self, '__wbuffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self, 
+            w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
@@ -665,9 +665,6 @@
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
-        if self.config.objspace.usemodules.cpyext:
-            from pypy.module.cpyext.state import State
-            self.fromcache(State).build_api(self)
         self.getbuiltinmodule('sys')
         self.getbuiltinmodule('imp')
         self.getbuiltinmodule('__builtin__')
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3419,17 +3419,22 @@
     BCharA = new_array_type(BCharP, None)
     p1 = from_buffer(BCharA, b"foo")
     assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # Python 2 only
         contents = from_buffer(BCharA, buffer(b"foo"))
+        assert len(contents) == len(p1)
         for i in range(len(contents)):
             assert contents[i] == p1[i]
-        p4 = from_buffer(BCharA, b"f\x00\x00\x00o\x00\x00\x00o\x00\x00\x00")
+        p4 = buffer(u+"foo")
         contents = from_buffer(BCharA, buffer(u+"foo"))
+        assert len(contents) == len(p4)
         for i in range(len(contents)):
             assert contents[i] == p4[i]
     try:
@@ -3438,6 +3443,7 @@
         pass
     else:
         contents = from_buffer(BCharA, memoryview(b"foo"))
+        assert len(contents) == len(p1)
         for i in range(len(contents)):
             assert contents[i] == p1[i]
 
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -1,5 +1,4 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.interpreter import gateway
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
 
@@ -13,12 +12,17 @@
 
     atexit_funcs = []
 
+    def setup_after_space_initialization(self):
+        state = self.space.fromcache(State)
+        state.setup_rawrefcount()
+        state.build_api()
+
     def startup(self, space):
         space.fromcache(State).startup(space)
         method = pypy.module.cpyext.typeobject.get_new_method_def(space)
         w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap(''))
         space.appexec([space.type(w_obj)], """(methodtype):
-            from pickle import Pickler 
+            from pickle import Pickler
             Pickler.dispatch[methodtype] = Pickler.save_global
         """)
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1,5 +1,6 @@
 import ctypes
 import sys, os
+from collections import defaultdict
 
 import py
 
@@ -72,7 +73,6 @@
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
-VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list')
 CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
                                        hints={'nolength': True}),
                           use_cache=False)
@@ -365,8 +365,8 @@
         func_name = func.func_name
         if header is not None:
             c_name = None
-            assert func_name not in FUNCTIONS, (
-                "%s already registered" % func_name)
+            if func_name in FUNCTIONS_BY_HEADER[header]:
+                raise ValueError("%s already registered" % func_name)
         else:
             c_name = func_name
         api_function = ApiFunction(argtypes, restype, func, error,
@@ -464,9 +464,7 @@
             return res
 
         if header is not None:
-            if header == DEFAULT_HEADER:
-                FUNCTIONS[func_name] = api_function
-            FUNCTIONS_BY_HEADER.setdefault(header, {})[func_name] = api_function
+            FUNCTIONS_BY_HEADER[header][func_name] = api_function
         INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
         return unwrapper  # used in 'normal' RPython code.
     return decorate
@@ -490,8 +488,7 @@
     GLOBALS[name] = (typ, expr)
 
 INTERPLEVEL_API = {}
-FUNCTIONS = {}
-FUNCTIONS_BY_HEADER = {}
+FUNCTIONS_BY_HEADER = defaultdict(dict)
 
 # These are C symbols which cpyext will export, but which are defined in .c
 # files somewhere in the implementation of cpyext (rather than being defined in
@@ -527,6 +524,8 @@
     'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor',
     'PyCapsule_SetContext', 'PyCapsule_Import', 'PyCapsule_Type', '_Py_get_capsule_type',
 
+    'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
+
     'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
 
     'PyOS_getsig', 'PyOS_setsig',
@@ -573,7 +572,9 @@
     # PyExc_AttributeError, PyExc_OverflowError, PyExc_ImportError,
     # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError,
     # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ...
-    for exc_name in exceptions.Module.interpleveldefs.keys():
+    global all_exceptions
+    all_exceptions = list(exceptions.Module.interpleveldefs)
+    for exc_name in all_exceptions:
         register_global('PyExc_' + exc_name,
             'PyTypeObject*',
             'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, ))
@@ -617,14 +618,6 @@
                              % (cpyname, ))
 build_exported_objects()
 
-def get_structtype_for_ctype(ctype):
-    from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
-    from pypy.module.cpyext.cdatetime import PyDateTime_CAPI
-    from pypy.module.cpyext.intobject import PyIntObject
-    return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr,
-            "PyIntObject*": PyIntObject,
-            "PyDateTime_CAPI*": lltype.Ptr(PyDateTime_CAPI)}[ctype]
-
 # Note: as a special case, "PyObject" is the pointer type in RPython,
 # corresponding to "PyObject *" in C.  We do that only for PyObject.
 # For example, "PyTypeObject" is the struct type even in RPython.
@@ -673,12 +666,6 @@
 # a pointer to PyObject
 PyObjectP = rffi.CArrayPtr(PyObject)
 
-VA_TP_LIST = {}
-#{'int': lltype.Signed,
-#              'PyObject*': PyObject,
-#              'PyObject**': PyObjectP,
-#              'int*': rffi.INTP}
-
 def configure_types():
     for config in (CConfig, CConfig2):
         for name, TYPE in rffi_platform.configure(config).iteritems():
@@ -953,21 +940,8 @@
     wrapper_second_level._dont_inline_ = True
     return wrapper_second_level
 
-def process_va_name(name):
-    return name.replace('*', '_star')
 
-def setup_va_functions(eci):
-    for name, TP in VA_TP_LIST.iteritems():
-        name_no_star = process_va_name(name)
-        func = rffi.llexternal('pypy_va_get_%s' % name_no_star, [VA_LIST_P],
-                               TP, compilation_info=eci)
-        globals()['va_get_%s' % name_no_star] = func
-
-def setup_init_functions(eci, translating):
-    if translating:
-        prefix = 'PyPy'
-    else:
-        prefix = 'cpyexttest'
+def setup_init_functions(eci, prefix):
     # jump through hoops to avoid releasing the GIL during initialization
     # of the cpyext module.  The C functions are called with no wrapper,
     # but must not do anything like calling back PyType_Ready().  We
@@ -1029,23 +1003,27 @@
 # Do not call this more than once per process
 def build_bridge(space):
     "NOT_RPYTHON"
-    from pypy.module.cpyext.pyobject import make_ref
     from rpython.translator.c.database import LowLevelDatabase
     use_micronumpy = setup_micronumpy(space)
     db = LowLevelDatabase()
-    prefix ='cpyexttest'
+    prefix = 'cpyexttest'
 
-    functions = generate_decls_and_callbacks(db, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
 
     # Structure declaration code
+    functions = []
     members = []
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                # added only for the macro, not the decl
-                continue
             restype, args = c_function_signature(db, func)
+            callargs = ', '.join('arg%d' % (i,)
+                                for i in range(len(func.argtypes)))
+            if func.restype is lltype.Void:
+                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+            else:
+                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
             members.append('%s (*%s)(%s);' % (restype, name, args))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
@@ -1057,15 +1035,8 @@
     """ % dict(members=structmembers)
 
     global_objects = []
-    for name, (typ, expr) in GLOBALS.iteritems():
-        if '#' in name:
-            continue
-        if typ == 'PyDateTime_CAPI*':
-            continue
-        elif name.startswith('PyExc_'):
-            global_objects.append('%s _%s;' % (typ[:-1], name))
-        else:
-            global_objects.append('%s %s = NULL;' % (typ, name))
+    for name in all_exceptions:
+        global_objects.append('PyTypeObject _PyExc_%s;' % name)
     global_code = '\n'.join(global_objects)
 
     prologue = ("#include <Python.h>\n"
@@ -1082,84 +1053,64 @@
             '\n' +
             '\n'.join(functions))
 
-    eci = build_eci(True, code, use_micronumpy)
+    eci = build_eci(code, use_micronumpy, translating=False)
     eci = eci.compile_shared_lib(
         outputfilename=str(udir / "module_cache" / "pypyapi"))
+    space.fromcache(State).install_dll(eci)
     modulename = py.path.local(eci.libraries[-1])
 
-    def dealloc_trigger():
-        from pypy.module.cpyext.pyobject import decref
-        print 'dealloc_trigger...'
-        while True:
-            ob = rawrefcount.next_dead(PyObject)
-            if not ob:
-                break
-            print 'deallocating PyObject', ob
-            decref(space, ob)
-        print 'dealloc_trigger DONE'
-        return "RETRY"
-    rawrefcount.init(dealloc_trigger)
-
     run_bootstrap_functions(space)
 
     # load the bridge, and init structure
     bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
 
-    space.fromcache(State).install_dll(eci)
+    # populate static data
+    builder = space.fromcache(State).builder = TestingObjBuilder()
+    for name, (typ, expr) in GLOBALS.iteritems():
+        if '#' in name:
+            name, header = name.split('#')
+            assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
+            isptr = False
+        elif name.startswith('PyExc_'):
+            isptr = False
+        elif typ == 'PyDateTime_CAPI*':
+            isptr = True
+        else:
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
 
-    # populate static data
-    builder = space.fromcache(StaticObjectBuilder)
-    for name, (typ, expr) in GLOBALS.iteritems():
         from pypy.module import cpyext    # for the eval() below
         w_obj = eval(expr)
-        if '#' in name:
-            name = name.split('#')[0]
-            isptr = False
-        else:
-            isptr = True
-        if name.startswith('PyExc_'):
-            isptr = False
-
         INTERPLEVEL_API[name] = w_obj
 
-        name = name.replace('Py', prefix)
+        mname = mangle_name(prefix, name)
         if isptr:
-            ptr = ctypes.c_void_p.in_dll(bridge, name)
-            if typ == 'PyObject*':
-                value = make_ref(space, w_obj)
-            elif typ == 'PyDateTime_CAPI*':
-                value = w_obj
-            else:
-                assert False, "Unknown static pointer: %s %s" % (typ, name)
+            assert typ == 'PyDateTime_CAPI*'
+            value = w_obj
+            ptr = ctypes.c_void_p.in_dll(bridge, mname)
             ptr.value = ctypes.cast(ll2ctypes.lltype2ctypes(value),
                                     ctypes.c_void_p).value
-        elif typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*'):
-            if name.startswith('PyPyExc_') or name.startswith('cpyexttestExc_'):
+        else:
+            if name.startswith('PyExc_'):
                 # we already have the pointer
-                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, in_dll)
             else:
                 # we have a structure, get its address
-                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, ctypes.pointer(in_dll))
             builder.prepare(py_obj, w_obj)
-        else:
-            assert False, "Unknown static object: %s %s" % (typ, name)
-    builder.attach_all()
+    builder.attach_all(space)
 
     pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
 
     # implement structure initialization code
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if name.startswith('cpyext_') or func is None: # XXX hack
-                continue
             pypyAPI[structindex[name]] = ctypes.cast(
                 ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
                 ctypes.c_void_p)
-    setup_va_functions(eci)
 
-    setup_init_functions(eci, translating=False)
+    setup_init_functions(eci, prefix)
     return modulename.new(ext='')
 
 def attach_recusively(space, static_pyobjs, static_objs_w, attached_objs, i):
@@ -1191,9 +1142,8 @@
     attached_objs.append(i)
 
 
-class StaticObjectBuilder:
-    def __init__(self, space):
-        self.space = space
+class StaticObjectBuilder(object):
+    def __init__(self):
         self.static_pyobjs = []
         self.static_objs_w = []
         self.cpyext_type_init = None
@@ -1209,12 +1159,11 @@
         self.static_pyobjs.append(py_obj)
         self.static_objs_w.append(w_obj)
 
-    def attach_all(self):
+    def attach_all(self, space):
         # this is RPython, called once in pypy-c when it imports cpyext
         from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
         from pypy.module.cpyext.pyobject import track_reference
         #
-        space = self.space
         static_pyobjs = self.get_static_pyobjs()
         static_objs_w = self.static_objs_w
         for i in range(len(static_objs_w)):
@@ -1230,6 +1179,12 @@
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
+class TestingObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used in tests."""
+
+class TranslationObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used during translation."""
+
 
 def mangle_name(prefix, name):
     if name.startswith('Py'):
@@ -1237,23 +1192,26 @@
     elif name.startswith('_Py'):
         return '_' + prefix + name[3:]
     else:
-        return None
+        raise ValueError("Error converting '%s'" % name)
 
-def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
+def write_header(header_name, decls):
+    lines = [
+        '#define Signed   long           /* xxx temporary fix */',
+        '#define Unsigned unsigned long  /* xxx temporary fix */',
+        '',] + decls + [
+        '',
+        '#undef Signed    /* xxx temporary fix */',
+        '#undef Unsigned  /* xxx temporary fix */',
+        '']
+    decl_h = udir.join(header_name)
+    decl_h.write('\n'.join(lines))
+
+def generate_decls_and_callbacks(db, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
-    export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
-    for name in export_symbols:
-        if '#' in name:
-            name, header = name.split('#')
-        else:
-            header = pypy_decl
+    for name in SYMBOLS_C:
         newname = mangle_name(prefix, name)
-        assert newname, name
-        if header == pypy_decl:
-            pypy_macros.append('#define %s %s' % (name, newname))
-        if name.startswith("PyExc_"):
-            pypy_macros.append('#define _%s _%s' % (name, newname))
+        pypy_macros.append('#define %s %s' % (name, newname))
 
     # Generate defines
     for macro_name, size in [
@@ -1273,50 +1231,18 @@
     pypy_macros_h = udir.join('pypy_macros.h')
     pypy_macros_h.write('\n'.join(pypy_macros))
 
-    # implement function callbacks and generate function decls
-    functions = []
-    decls = {}
-    pypy_decls = decls[pypy_decl] = []
-    pypy_decls.append('#define Signed   long           /* xxx temporary fix */\n')
-    pypy_decls.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-
+    # generate function decls
+    decls = defaultdict(list)
     for decl in FORWARD_DECLS:
-        pypy_decls.append("%s;" % (decl,))
+        decls[pypy_decl].append("%s;" % (decl,))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
-        if header_name not in decls:
-            header = decls[header_name] = []
-            header.append('#define Signed   long           /* xxx temporary fix */\n')
-            header.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-        else:
-            header = decls[header_name]
-
+        header = decls[header_name]
         for name, func in sorted(header_functions.iteritems()):
-            if not func:
-                continue
-            if header == DEFAULT_HEADER:
-                _name = name
-            else:
-                # this name is not included in pypy_macros.h
-                _name = mangle_name(prefix, name)
-                assert _name is not None, 'error converting %s' % name
-                header.append("#define %s %s" % (name, _name))
+            _name = mangle_name(prefix, name)
+            header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, _name, args))
-            if api_struct:
-                callargs = ', '.join('arg%d' % (i,)
-                                    for i in range(len(func.argtypes)))
-                if func.restype is lltype.Void:
-                    body = "{ _pypyAPI.%s(%s); }" % (_name, callargs)
-                else:
-                    body = "{ return _pypyAPI.%s(%s); }" % (_name, callargs)
-                functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
-    for name in VA_TP_LIST:
-        name_no_star = process_va_name(name)
-        header = ('%s pypy_va_get_%s(va_list* vp)' %
-                  (name, name_no_star))
-        pypy_decls.append('RPY_EXTERN ' + header + ';')
-        functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
+            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
 
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
@@ -1325,19 +1251,11 @@
         elif name.startswith('PyExc_'):
             typ = 'PyObject*'
             header = pypy_decl
-        if header != pypy_decl:
-            decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
+        decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
         decls[header].append('PyAPI_DATA(%s) %s;' % (typ, name))
 
-    for header_name in FUNCTIONS_BY_HEADER.keys():
-        header = decls[header_name]
-        header.append('#undef Signed    /* xxx temporary fix */\n')
-        header.append('#undef Unsigned  /* xxx temporary fix */\n')
-
     for header_name, header_decls in decls.iteritems():
-        decl_h = udir.join(header_name)
-        decl_h.write('\n'.join(header_decls))
-    return functions
+        write_header(header_name, header_decls)
 
 separate_module_files = [source_dir / "varargwrapper.c",
                          source_dir / "pyerrors.c",
@@ -1349,6 +1267,7 @@
                          source_dir / "pythonrun.c",
                          source_dir / "sysmodule.c",
                          source_dir / "bufferobject.c",
+                         source_dir / "complexobject.c",
                          source_dir / "cobject.c",
                          source_dir / "structseq.c",
                          source_dir / "capsule.c",
@@ -1358,14 +1277,16 @@
                          source_dir / "pymem.c",
                          ]
 
-def build_eci(building_bridge, code, use_micronumpy=False):
+def build_eci(code, use_micronumpy=False, translating=False):
     "NOT_RPYTHON"
     # Build code and get pointer to the structure
     kwds = {}
 
     compile_extra=['-DPy_BUILD_CORE']
 
-    if building_bridge:
+    if translating:
+        kwds["includes"] = ['Python.h'] # this is our Python.h
+    else:
         if sys.platform == "win32":
             # '%s' undefined; assuming extern returning int
             compile_extra.append("/we4013")
@@ -1375,8 +1296,6 @@
         elif sys.platform.startswith('linux'):
             compile_extra.append("-Werror=implicit-function-declaration")
             compile_extra.append('-g')
-    else:
-        kwds["includes"] = ['Python.h'] # this is our Python.h
 
     # Generate definitions for global structures
     structs = ["#include <Python.h>"]
@@ -1427,9 +1346,7 @@
         return use_micronumpy
     # import registers api functions by side-effect, we also need HEADER
     from pypy.module.cpyext.ndarrayobject import HEADER
-    global FUNCTIONS_BY_HEADER, separate_module_files
-    for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
-        FUNCTIONS_BY_HEADER.setdefault(HEADER, {})[func_name] = None
+    global separate_module_files
     register_global("PyArray_Type",
         'PyTypeObject*',  "space.gettypeobject(W_NDimArray.typedef)",
         header=HEADER)
@@ -1443,21 +1360,19 @@
     db = LowLevelDatabase()
     prefix = 'PyPy'
 
-    functions = generate_decls_and_callbacks(db, api_struct=False, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
+
     code = "#include <Python.h>\n"
     if use_micronumpy:
         code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
-    code  += "\n".join(functions)
 
-    eci = build_eci(False, code, use_micronumpy)
-
+    eci = build_eci(code, use_micronumpy, translating=True)
     space.fromcache(State).install_dll(eci)
 
     run_bootstrap_functions(space)
-    setup_va_functions(eci)
 
     # emit uninitialized static data
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder = TranslationObjBuilder()
     lines = ['PyObject *pypy_static_pyobjs[] = {\n']
     include_lines = ['RPY_EXTERN PyObject *pypy_static_pyobjs[];\n']
     for name, (typ, expr) in sorted(GLOBALS.items()):
@@ -1465,17 +1380,15 @@
             name, header = name.split('#')
             assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
             typ = typ[:-1]
-            if header != pypy_decl:
-                # since the #define is not in pypy_macros, do it here
-                mname = mangle_name(prefix, name)
-                include_lines.append('#define %s %s\n' % (name, mname))
+            mname = mangle_name(prefix, name)
+            include_lines.append('#define %s %s\n' % (name, mname))
         elif name.startswith('PyExc_'):
             typ = 'PyTypeObject'
             name = '_' + name
         elif typ == 'PyDateTime_CAPI*':
             continue
         else:
-            assert False, "Unknown static data: %s %s" % (typ, name)
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
 
         from pypy.module import cpyext     # for the eval() below
         w_obj = eval(expr)
@@ -1495,20 +1408,15 @@
 
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                continue
-            newname = mangle_name('PyPy', name) or name
+            newname = mangle_name(prefix, name)
             deco = entrypoint_lowlevel("cpyext", func.argtypes, newname,
                                         relax=True)
             deco(func.get_wrapper(space))
 
-    setup_init_functions(eci, translating=True)
+    setup_init_functions(eci, prefix)
     trunk_include = pypydir.dirpath() / 'include'
     copy_header_files(trunk_include, use_micronumpy)
 
-def init_static_data_translated(space):
-    builder = space.fromcache(StaticObjectBuilder)
-    builder.attach_all()
 
 def _load_from_cffi(space, name, path, initptr):
     from pypy.module._cffi_backend import cffi1_module
diff --git a/pypy/module/cpyext/include/complexobject.h b/pypy/module/cpyext/include/complexobject.h
--- a/pypy/module/cpyext/include/complexobject.h
+++ b/pypy/module/cpyext/include/complexobject.h
@@ -16,23 +16,8 @@
     Py_complex cval;
 } PyComplexObject;
 
-/* generated function */
-PyAPI_FUNC(int) _PyComplex_AsCComplex(PyObject *, Py_complex *);
-PyAPI_FUNC(PyObject *) _PyComplex_FromCComplex(Py_complex *);
-
-Py_LOCAL_INLINE(Py_complex) PyComplex_AsCComplex(PyObject *obj)
-{
-    Py_complex result;
-    _PyComplex_AsCComplex(obj, &result);
-    return result;
-}
-
-// shmuller 2013/07/30: Make a function, since macro will fail in C++ due to 
-//                      const correctness if called with "const Py_complex"
-//#define PyComplex_FromCComplex(c) _PyComplex_FromCComplex(&c)
-Py_LOCAL_INLINE(PyObject *) PyComplex_FromCComplex(Py_complex c) {
-    return _PyComplex_FromCComplex(&c);
-}
+PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *obj);
+PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex c);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl, Py_buffer, Py_bufferP)
+    pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
@@ -352,7 +352,7 @@
         else:
             #do not call twice
             return
-        if self.releasebufferproc:        
+        if self.releasebufferproc:
             func_target = rffi.cast(releasebufferproc, self.releasebufferproc)
             with lltype.scoped_alloc(Py_buffer) as pybuf:
                 pybuf.c_buf = self.ptr
@@ -417,7 +417,7 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self, 
+        buf = CPyBuffer(space, ptr[0], size, w_self,
                                releasebuffer=releasebuffer)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
@@ -527,7 +527,7 @@
     w_type = space.gettypeobject(typedef)
 
     header = pypy_decl
-    if mangle_name('', typedef.name) is None:
+    if not (name.startswith('Py') or name.startswith('_Py')):
         header = None
     handled = False
     # unary functions
diff --git a/pypy/module/cpyext/src/complexobject.c b/pypy/module/cpyext/src/complexobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/complexobject.c
@@ -0,0 +1,16 @@
+
+#include "Python.h"
+
+Py_complex
+PyComplex_AsCComplex(PyObject *obj)
+{
+    Py_complex result;
+    _PyComplex_AsCComplex(obj, &result);
+    return result;
+}
+
+PyObject *
+PyComplex_FromCComplex(Py_complex c)
+{
+    return _PyComplex_FromCComplex(&c);
+}
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -2,7 +2,6 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import executioncontext
-from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
@@ -14,9 +13,7 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
-        if space.config.translation.gc != "boehm":
-            pyobj_dealloc_action = PyObjDeallocAction(space)
-            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+        self.builder = None
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -57,22 +54,40 @@
                         "Function returned an error result without setting an "
                         "exception")
 
-    def build_api(self, space):
+    def setup_rawrefcount(self):
+        space = self.space
+        if not self.space.config.translating:
+            def dealloc_trigger():
+                from pypy.module.cpyext.pyobject import PyObject, decref
+                print 'dealloc_trigger...'
+                while True:
+                    ob = rawrefcount.next_dead(PyObject)
+                    if not ob:
+                        break
+                    print 'deallocating PyObject', ob
+                    decref(space, ob)
+                print 'dealloc_trigger DONE'
+                return "RETRY"
+            rawrefcount.init(dealloc_trigger)
+        else:
+            if space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(space)
+                space.actionflag.register_periodic_action(action,
+                    use_bytecode_counter=True)
+            else:
+                pyobj_dealloc_action = PyObjDeallocAction(space)
+                self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+
+    def build_api(self):
         """NOT_RPYTHON
         This function is called when at object space creation,
         and drives the compilation of the cpyext library
         """
         from pypy.module.cpyext import api
-        state = self.space.fromcache(State)
         if not self.space.config.translating:
-            state.api_lib = str(api.build_bridge(self.space))
+            self.api_lib = str(api.build_bridge(self.space))
         else:
             api.setup_library(self.space)
-            #
-            if self.space.config.translation.gc == "boehm":
-                action = BoehmPyObjDeallocAction(self.space)
-                self.space.actionflag.register_periodic_action(action,
-                    use_bytecode_counter=True)
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -84,17 +99,17 @@
 
     def startup(self, space):
         "This function is called when the program really starts"
-
         from pypy.module.cpyext.typeobject import setup_new_method_def
         from pypy.module.cpyext.api import INIT_FUNCTIONS
-        from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
             if space.config.translation.gc != "boehm":
+                # This must be called in RPython, the untranslated version
+                # does something different. Sigh.
                 rawrefcount.init(
                     llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
                     self.dealloc_trigger))
-            init_static_data_translated(space)
+            self.builder.attach_all(space)
 
         setup_new_method_def(space)
 
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,9 +89,9 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest1'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
                 == ('Py_ssize_t', 'Py_ssize_t arg0'))
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest2'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
                 == ('Py_ssize_t *', 'Py_ssize_t *arg0'))
 
         PyPy_TypedefTest1(space, 0)
@@ -100,7 +100,7 @@
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
 
- at pytest.mark.skipif(os.environ.get('USER')=='root', 
+ at pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
     api.copy_header_files(tmpdir, True)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -27,8 +27,9 @@
 
 class TestApi:
     def test_signature(self):
-        assert 'PyModule_Check' in api.FUNCTIONS
-        assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
+        common_functions = api.FUNCTIONS_BY_HEADER[api.pypy_decl]
+        assert 'PyModule_Check' in common_functions
+        assert common_functions['PyModule_Check'].argtypes == [api.PyObject]
 
 
 class SpaceCompiler(SystemCompilationInfo):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -87,7 +87,7 @@
     py_getsetdef.c_set = rffi.cast(setter, 0)
     py_getsetdef.c_closure = rffi.cast(rffi.VOIDP, 0)
     return py_getsetdef
-    
+
 
 class W_MemberDescr(GetSetProperty):
     name = 'member_descriptor'
@@ -740,7 +740,7 @@
 
     pto.c_tp_free = llslot(space, PyObject_Free)
     pto.c_tp_alloc = llslot(space, PyType_GenericAlloc)
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder
     if ((pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE) != 0
             and builder.cpyext_type_init is None):
             # this ^^^ is not None only during startup of cpyext.  At that
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -548,6 +548,10 @@
 
         sub = self._op_val(space, w_old)
         by = self._op_val(space, w_new)
+        # the following two lines are for being bug-to-bug compatible
+        # with CPython: see issue #2448
+        if count >= 0 and len(input) == 0:
+            return self._empty()
         try:
             res = replace(input, sub, by, count)
         except OverflowError:
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -809,6 +809,16 @@
         s = b"a" * (2**16)
         raises(OverflowError, s.replace, b"", s)
 
+    def test_replace_issue2448(self):
+        # CPython's replace() method has a bug that makes
+        #   ''.replace('', 'x')  gives a different answer than
+        #   ''.replace('', 'x', 1000).  This is the case in all
+        # known versions, at least until 2.7.13.  Some people
+        # call that a feature on the CPython issue report and
+        # the discussion dies out, so it might never be fixed.
+        assert ''.replace('', 'x') == 'x'
+        assert ''.replace('', 'x', 1000) == ''
+
     def test_getslice(self):
         assert "foobar".__getslice__(4, 4321) == "ar"
         s = b"abc"
diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -106,8 +106,9 @@
         else:
             name = 'unnamed' % id(ssarepr)
         i = 1
-        # escape <lambda> names for windows
-        name = name.replace('<lambda>', '_(lambda)_')
+        # escape names like <lambda> for windows by removing any strange
+        # character; then make sure the names are not too long
+        name = ''.join(c for c in name if c.isalnum() or c == '_')[:60]
         extra = ''
         while dir.join(name+extra).check():
             i += 1
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4613,3 +4613,10 @@
         self.check_operations_history(guard_nonnull=0, guard_nonnull_class=0,
                                       guard_class=2,
                                       assert_not_none=2) # before optimization
+
+    def test_call_time_clock(self):
+        import time
+        def g():
+            time.clock()
+            return 0
+        self.interp_operations(g, [])
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1770,11 +1770,11 @@
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
-            pinned_obj_size = llarena.getfakearenaaddress(cur) - prev
+            free_range_size = llarena.getfakearenaaddress(cur) - prev
             if self.gc_nursery_debug:
-                llarena.arena_reset(prev, pinned_obj_size, 3)
+                llarena.arena_reset(prev, free_range_size, 3)
             else:
-                llarena.arena_reset(prev, pinned_obj_size, 0)
+                llarena.arena_reset(prev, free_range_size, 0)
             #
             # clean up object's flags
             obj = cur + size_gc_header
@@ -1784,7 +1784,7 @@
             nursery_barriers.append(cur)
             #
             # update 'prev' to the end of the 'cur' object
-            prev = prev + pinned_obj_size + \
+            prev = prev + free_range_size + \
                 (size_gc_header + self.get_size(obj))
         #
         # reset everything after the last pinned object till the end of the arena
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1320,7 +1320,8 @@
         def __init__(self, string):
             self.ptr = str2charp(string, track_allocation=False)
         def __del__(self):
-            free_charp(self.ptr, track_allocation=False)
+            if free_charp is not None:    # CPython shutdown
+                free_charp(self.ptr, track_allocation=False)
 
     TEST_RAW_ADDR_KEEP_ALIVE = {}
 

From pypy.commits at gmail.com  Fri Jan  6 09:22:58 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 06 Jan 2017 06:22:58 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Replace body of @cpython_api
 with a _create_api_func() function
Message-ID: <586fa842.145e1c0a.65985.3e43@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89397:de8be6201edb
Date: 2017-01-06 14:15 +0000
http://bitbucket.org/pypy/pypy/changeset/de8be6201edb/

Log:	Replace body of @cpython_api with a _create_api_func() function

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -344,6 +344,14 @@
     - set `gil` to "acquire", "release" or "around" to acquire the GIL,
       release the GIL, or both
     """
+    def decorate(func):
+        return _create_api_func(func, argtypes, restype, error, header, gil,
+            result_borrowed, result_is_ll)
+    return decorate
+
+def _create_api_func(
+        func, argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
+        gil=None, result_borrowed=False, result_is_ll=False):
     if isinstance(restype, lltype.Typedef):
         real_restype = restype.OF
     else:
@@ -359,7 +367,7 @@
     expect_integer = (isinstance(real_restype, lltype.Primitive) and
                       rffi.cast(restype, 0) == 0)
 
-    def decorate(func):
+    if True:  # preserve indentation
         func._always_inline_ = 'try'
         func_name = func.func_name
         if header is not None:
@@ -466,7 +474,7 @@
             FUNCTIONS_BY_HEADER[header][func_name] = api_function
         INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
         return unwrapper  # used in 'normal' RPython code.
-    return decorate
+
 
 def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')

From pypy.commits at gmail.com  Fri Jan  6 09:23:00 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 06 Jan 2017 06:23:00 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: deindent
Message-ID: <586fa844.8f95c20a.1cc9c.1363@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89398:52133eacf24f
Date: 2017-01-06 14:15 +0000
http://bitbucket.org/pypy/pypy/changeset/52133eacf24f/

Log:	deindent

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -367,113 +367,112 @@
     expect_integer = (isinstance(real_restype, lltype.Primitive) and
                       rffi.cast(restype, 0) == 0)
 
-    if True:  # preserve indentation
-        func._always_inline_ = 'try'
-        func_name = func.func_name
-        if header is not None:
-            c_name = None
-            if func_name in FUNCTIONS_BY_HEADER[header]:
-                raise ValueError("%s already registered" % func_name)
-        else:
-            c_name = func_name
-        api_function = ApiFunction(argtypes, restype, func, error,
-                                   c_name=c_name, gil=gil,
-                                   result_borrowed=result_borrowed,
-                                   result_is_ll=result_is_ll)
-        func.api_func = api_function
+    func._always_inline_ = 'try'
+    func_name = func.func_name
+    if header is not None:
+        c_name = None
+        if func_name in FUNCTIONS_BY_HEADER[header]:
+            raise ValueError("%s already registered" % func_name)
+    else:
+        c_name = func_name
+    api_function = ApiFunction(argtypes, restype, func, error,
+                                c_name=c_name, gil=gil,
+                                result_borrowed=result_borrowed,
+                                result_is_ll=result_is_ll)
+    func.api_func = api_function
 
-        if error is _NOT_SPECIFIED:
-            raise ValueError("function %s has no return value for exceptions"
-                             % func)
-        names = api_function.argnames
-        types_names_enum_ui = unrolling_iterable(enumerate(
-            zip(api_function.argtypes,
-                [tp_name.startswith("w_") for tp_name in names])))
+    if error is _NOT_SPECIFIED:
+        raise ValueError("function %s has no return value for exceptions"
+                            % func)
+    names = api_function.argnames
+    types_names_enum_ui = unrolling_iterable(enumerate(
+        zip(api_function.argtypes,
+            [tp_name.startswith("w_") for tp_name in names])))
 
-        @specialize.ll()
-        def unwrapper(space, *args):
-            from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj
-            from pypy.module.cpyext.pyobject import from_ref, as_pyobj
-            newargs = ()
-            keepalives = ()
-            assert len(args) == len(api_function.argtypes)
-            for i, (ARG, is_wrapped) in types_names_enum_ui:
-                input_arg = args[i]
-                if is_PyObject(ARG) and not is_wrapped:
-                    # build a 'PyObject *' (not holding a reference)
-                    if not is_pyobj(input_arg):
-                        keepalives += (input_arg,)
-                        arg = rffi.cast(ARG, as_pyobj(space, input_arg))
-                    else:
-                        arg = rffi.cast(ARG, input_arg)
-                elif ARG == rffi.VOIDP and not is_wrapped:
-                    # unlike is_PyObject case above, we allow any kind of
-                    # argument -- just, if it's an object, we assume the
-                    # caller meant for it to become a PyObject*.
-                    if input_arg is None or isinstance(input_arg, W_Root):
-                        keepalives += (input_arg,)
-                        arg = rffi.cast(ARG, as_pyobj(space, input_arg))
-                    else:
-                        arg = rffi.cast(ARG, input_arg)
-                elif (is_PyObject(ARG) or ARG == rffi.VOIDP) and is_wrapped:
-                    # build a W_Root, possibly from a 'PyObject *'
-                    if is_pyobj(input_arg):
-                        arg = from_ref(space, input_arg)
-                    else:
-                        arg = input_arg
+    @specialize.ll()
+    def unwrapper(space, *args):
+        from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj
+        from pypy.module.cpyext.pyobject import from_ref, as_pyobj
+        newargs = ()
+        keepalives = ()
+        assert len(args) == len(api_function.argtypes)
+        for i, (ARG, is_wrapped) in types_names_enum_ui:
+            input_arg = args[i]
+            if is_PyObject(ARG) and not is_wrapped:
+                # build a 'PyObject *' (not holding a reference)
+                if not is_pyobj(input_arg):
+                    keepalives += (input_arg,)
+                    arg = rffi.cast(ARG, as_pyobj(space, input_arg))
+                else:
+                    arg = rffi.cast(ARG, input_arg)
+            elif ARG == rffi.VOIDP and not is_wrapped:
+                # unlike is_PyObject case above, we allow any kind of
+                # argument -- just, if it's an object, we assume the
+                # caller meant for it to become a PyObject*.
+                if input_arg is None or isinstance(input_arg, W_Root):
+                    keepalives += (input_arg,)
+                    arg = rffi.cast(ARG, as_pyobj(space, input_arg))
+                else:
+                    arg = rffi.cast(ARG, input_arg)
+            elif (is_PyObject(ARG) or ARG == rffi.VOIDP) and is_wrapped:
+                # build a W_Root, possibly from a 'PyObject *'
+                if is_pyobj(input_arg):
+                    arg = from_ref(space, input_arg)
+                else:
+                    arg = input_arg
 
-                        ## ZZZ: for is_pyobj:
-                        ## try:
-                        ##     arg = from_ref(space,
-                        ##                rffi.cast(PyObject, input_arg))
-                        ## except TypeError, e:
-                        ##     err = oefmt(space.w_TypeError,
-                        ##                 "could not cast arg to PyObject")
-                        ##     if not catch_exception:
-                        ##         raise err
-                        ##     state = space.fromcache(State)
-                        ##     state.set_exception(err)
-                        ##     if is_PyObject(restype):
-                        ##         return None
-                        ##     else:
-                        ##         return api_function.error_value
-                else:
-                    # arg is not declared as PyObject, no magic
-                    arg = input_arg
-                newargs += (arg, )
-            try:
-                return func(space, *newargs)
-            finally:
-                keepalive_until_here(*keepalives)
+                    ## ZZZ: for is_pyobj:
+                    ## try:
+                    ##     arg = from_ref(space,
+                    ##                rffi.cast(PyObject, input_arg))
+                    ## except TypeError, e:
+                    ##     err = oefmt(space.w_TypeError,
+                    ##                 "could not cast arg to PyObject")
+                    ##     if not catch_exception:
+                    ##         raise err
+                    ##     state = space.fromcache(State)
+                    ##     state.set_exception(err)
+                    ##     if is_PyObject(restype):
+                    ##         return None
+                    ##     else:
+                    ##         return api_function.error_value
+            else:
+                # arg is not declared as PyObject, no magic
+                arg = input_arg
+            newargs += (arg, )
+        try:
+            return func(space, *newargs)
+        finally:
+            keepalive_until_here(*keepalives)
 
-        unwrapper.func = func
-        unwrapper.api_func = api_function
+    unwrapper.func = func
+    unwrapper.api_func = api_function
 
-        # ZZZ is this whole logic really needed???  It seems to be only
-        # for RPython code calling PyXxx() functions directly.  I would
-        # think that usually directly calling the function is clean
-        # enough now
-        def unwrapper_catch(space, *args):
-            try:
-                res = unwrapper(space, *args)
-            except OperationError as e:
-                if not hasattr(api_function, "error_value"):
-                    raise
-                state = space.fromcache(State)
-                state.set_exception(e)
-                if is_PyObject(restype):
-                    return None
-                else:
-                    return api_function.error_value
-            got_integer = isinstance(res, (int, long, float))
-            assert got_integer == expect_integer, (
-                'got %r not integer' % (res,))
-            return res
+    # ZZZ is this whole logic really needed???  It seems to be only
+    # for RPython code calling PyXxx() functions directly.  I would
+    # think that usually directly calling the function is clean
+    # enough now
+    def unwrapper_catch(space, *args):
+        try:
+            res = unwrapper(space, *args)
+        except OperationError as e:
+            if not hasattr(api_function, "error_value"):
+                raise
+            state = space.fromcache(State)
+            state.set_exception(e)
+            if is_PyObject(restype):
+                return None
+            else:
+                return api_function.error_value
+        got_integer = isinstance(res, (int, long, float))
+        assert got_integer == expect_integer, (
+            'got %r not integer' % (res,))
+        return res
 
-        if header is not None:
-            FUNCTIONS_BY_HEADER[header][func_name] = api_function
-        INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
-        return unwrapper  # used in 'normal' RPython code.
+    if header is not None:
+        FUNCTIONS_BY_HEADER[header][func_name] = api_function
+    INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
+    return unwrapper  # used in 'normal' RPython code.
 
 
 def cpython_struct(name, fields, forward=None, level=1):

From pypy.commits at gmail.com  Fri Jan  6 09:36:04 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 06 Jan 2017 06:36:04 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Do not expose unexported
 functions to INTERPLEVEL_API either
Message-ID: <586fab54.d3811c0a.b1516.3e2c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89399:8db0229e854d
Date: 2017-01-06 14:35 +0000
http://bitbucket.org/pypy/pypy/changeset/8db0229e854d/

Log:	Do not expose unexported functions to INTERPLEVEL_API either

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -471,7 +471,7 @@
 
     if header is not None:
         FUNCTIONS_BY_HEADER[header][func_name] = api_function
-    INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
+        INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
     return unwrapper  # used in 'normal' RPython code.
 
 

From pypy.commits at gmail.com  Fri Jan  6 11:23:08 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 06 Jan 2017 08:23:08 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-from: close branch
Message-ID: <586fc46c.c11d1c0a.f79a8.6f70@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: cpyext-from
Changeset: r89400:8e6bf680169d
Date: 2017-01-06 14:59 +0100
http://bitbucket.org/pypy/pypy/changeset/8e6bf680169d/

Log:	close branch

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -211,7 +211,7 @@
     #_dup_Py_buffer(space, view, buf)
     view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
     # the docs say that in PyMemoryView_FromBuffer (and thus FromObject)
-    # this object must be NULL
+    # this object must be NULL in Python 3.5
     view.c_obj = make_ref(space, w_obj)
 
     # XXX what about w_mview.base = w_obj (see cpython 2.7 implementation)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -36,6 +36,7 @@
                 str2 = w_other.as_str()
                 return space.wrap(getattr(operator, name)(str1, str2))
 
+            import pdb; pdb.set_trace()
             try:
                 buf = space.buffer_w(w_other, space.BUF_CONTIG_RO)
             except OperationError as e:

From pypy.commits at gmail.com  Fri Jan  6 11:23:10 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 06 Jan 2017 08:23:10 -0800 (PST)
Subject: [pypy-commit] pypy default: merge cpyext-from2
Message-ID: <586fc46e.cf3fc20a.aa7e8.279e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r89401:0615cd861fd0
Date: 2017-01-06 17:20 +0100
http://bitbucket.org/pypy/pypy/changeset/0615cd861fd0/

Log:	merge cpyext-from2

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -35,8 +35,24 @@
     """
     Fills a newly allocated PyMemoryViewObject with the given W_MemoryView object.
     """
+    assert isinstance(w_obj, W_MemoryView)
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
-    py_obj.c_view.c_obj = rffi.cast(PyObject, 0)
+    view = py_obj.c_view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return
+    fill_Py_buffer(space, w_obj.buf, view)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_userdata)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s),
+                                             track_allocation=False))
+        rffi.setintfield(view, 'c_readonly', 1)
 
 def memory_realize(space, py_obj):
     """
@@ -88,29 +104,13 @@
     return ret
 
 @cpython_api([PyObject], Py_bufferP, error=CANNOT_FAIL)
-def PyMemoryView_GET_BUFFER(space, w_obj):
+def PyMemoryView_GET_BUFFER(space, pyobj):
     """Return a pointer to the buffer-info structure wrapped by the given
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
-    if not isinstance(w_obj, W_MemoryView):
-        return lltype.nullptr(Py_buffer)
-    py_memobj = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_obj)) # no inc_ref
-    view = py_memobj.c_view
-    ndim = w_obj.buf.getndim()
-    if ndim >= Py_MAX_NDIMS:
-        # XXX warn?
-        return view
-    fill_Py_buffer(space, w_obj.buf, view)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-        #view.c_obj = make_ref(space, w_obj) # NO - this creates a ref cycle!
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
-    except ValueError:
-        w_s = w_obj.descr_tobytes(space)
-        view.c_obj = make_ref(space, w_s)
-        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
-        rffi.setintfield(view, 'c_readonly', 1)
-    return view
+    # XXX move to a c-macro
+    py_memobj = rffi.cast(PyMemoryViewObject, pyobj)
+    return py_memobj.c_view
 
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
@@ -202,9 +202,11 @@
         return (_IsCContiguous(view) or _IsFortranContiguous(view))
     return 0
 
- at cpython_api([PyObject], PyObject)
+ at cpython_api([PyObject], PyObject, result_is_ll=True)
 def PyMemoryView_FromObject(space, w_obj):
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    w_memview = space.call_method(space.builtin, "memoryview", w_obj)
+    py_memview = make_ref(space, w_memview, w_obj)
+    return py_memview
 
 @cpython_api([Py_bufferP], PyObject)
 def PyMemoryView_FromBuffer(space, view):
@@ -212,6 +214,7 @@
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
+    assert view.c_obj
     w_obj = from_ref(space, view.c_obj)
     if isinstance(w_obj, W_MemoryView):
         return w_obj
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -4,6 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
+from pypy.module.cpyext.pyobject import from_ref
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
@@ -11,7 +12,7 @@
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
-        w_view = api.PyMemoryView_FromObject(w_hello)
+        w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
         assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
@@ -19,7 +20,7 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_memoryview = from_ref(space, api.PyMemoryView_FromObject(w_buf))
         view = api.PyMemoryView_GET_BUFFER(w_memoryview)
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)

From pypy.commits at gmail.com  Fri Jan  6 11:23:12 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 06 Jan 2017 08:23:12 -0800 (PST)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <586fc470.e337c20a.c90e9.5d16@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r89402:61294de6f130
Date: 2017-01-06 17:22 +0100
http://bitbucket.org/pypy/pypy/changeset/61294de6f130/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -91,3 +91,8 @@
 .. branch: cpyext-cleanup
 
 Refactor cpyext initialisation.
+
+.. branch: cpyext-from2
+
+Fix a test failure introduced by strbuf-as-buffer
+

From pypy.commits at gmail.com  Fri Jan  6 11:36:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jan 2017 08:36:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Update the unraisable-exception output to
 include a full traceback
Message-ID: <586fc7a2.54161c0a.87599.7742@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89403:671ae80283ca
Date: 2017-01-06 17:35 +0100
http://bitbucket.org/pypy/pypy/changeset/671ae80283ca/

Log:	Update the unraisable-exception output to include a full traceback

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -248,6 +248,10 @@
 
     def write_unraisable(self, space, where, w_object=None,
                          with_traceback=False, extra_line=''):
+        # Note: since Python 3.5, unraisable exceptions are always
+        # printed with a traceback.  Setting 'with_traceback=False'
+        # only asks for a different format, starting with the message
+        # "Exception Xxx ignored".
         if w_object is None:
             objrepr = ''
         else:
@@ -257,31 +261,32 @@
                 objrepr = "<object repr() failed>"
         #
         try:
-            if with_traceback:
-                try:
-                    self.normalize_exception(space)
-                except OperationError:
-                    pass
-                w_t = self.w_type
-                w_v = self.get_w_value(space)
-                w_tb = space.wrap(self.get_traceback())
-                space.appexec([space.wrap(where),
-                               space.wrap(objrepr),
-                               space.wrap(extra_line),
-                               w_t, w_v, w_tb],
-                """(where, objrepr, extra_line, t, v, tb):
-                    import sys, traceback
-                    if where or objrepr:
-                        sys.stderr.write('From %s%s:\\n' % (where, objrepr))
-                    if extra_line:
-                        sys.stderr.write(extra_line)
-                    traceback.print_exception(t, v, tb)
-                """)
+            try:
+                self.normalize_exception(space)
+            except OperationError:
+                pass
+            w_t = self.w_type
+            w_v = self.get_w_value(space)
+            w_tb = space.wrap(self.get_traceback())
+            if where or objrepr:
+                if with_traceback:
+                    first_line = 'From %s%s:\n' % (where, objrepr)
+                else:
+                    first_line = 'Exception ignored in: %s%s\n' % (
+                        where, objrepr)
             else:
-                msg = 'Exception %s in %s%s ignored\n' % (
-                    self.errorstr(space, use_repr=True), where, objrepr)
-                space.call_method(space.sys.get('stderr'), 'write',
-                                  space.wrap(msg))
+                first_line = ''
+            space.appexec([space.wrap(first_line),
+                           space.wrap(extra_line),
+                           w_t, w_v, w_tb],
+            """(first_line, extra_line, t, v, tb):
+                import sys
+                sys.stderr.write(first_line)
+                if extra_line:
+                    sys.stderr.write(extra_line)
+                import traceback
+                traceback.print_exception(t, v, tb)
+            """)
         except OperationError:
             pass   # ignored
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -574,9 +574,13 @@
             if self.gc_disabled(w_obj):
                 return
             try:
-                space.get_and_call_function(w_del, w_obj)
+                w_impl = space.get(w_del, w_obj)
             except Exception as e:
                 report_error(space, e, "method __del__ of ", w_obj)
+            try:
+                space.call_function(w_impl)
+            except Exception as e:
+                report_error(space, e, '', w_impl)
 
         # Call the RPython-level _finalize_() method.
         try:
diff --git a/pypy/objspace/std/test/test_userobject.py b/pypy/objspace/std/test/test_userobject.py
--- a/pypy/objspace/std/test/test_userobject.py
+++ b/pypy/objspace/std/test/test_userobject.py
@@ -163,29 +163,38 @@
         from io import StringIO
         class A(object):
             def __del__(self):
-                yaddadlaouti
+                raise ValueError('foo bar')
         prev = sys.stderr
         try:
             sys.stderr = StringIO()
             A()
             gc.collect()
             res = sys.stderr.getvalue()
+            sys.stderr = StringIO()
             A()
             gc.collect()
             res2 = sys.stderr.getvalue()
+            A.__del__ = lambda a, b, c: None  # will get "not enough arguments"
+            sys.stderr = StringIO()
+            A()
+            gc.collect()
+            res3 = sys.stderr.getvalue()
         finally:
             sys.stderr = prev
-        assert res.startswith('Exception')
-        assert 'NameError' in res
-        assert 'yaddadlaouti' in res
-        assert 'ignored' in res
-        assert res.count('\n') == 1    # a single line
-        assert res2.count('\n') == 2   # two lines
-        line2 = res2.split('\n')[1]
-        assert line2.startswith('Exception')
-        assert 'NameError' in line2
-        assert 'yaddadlaouti' in line2
-        assert 'ignored' in line2
+        def check_tb(x, traceback=True):
+            print('----\n%s----\n' % (x,))
+            assert x.startswith('Exception ignored in: <bound method ')
+            if traceback:
+                assert '>\nTraceback (most recent call last):\n  File "' in x
+                assert " in __del__\n" in x
+                assert x.endswith("\nValueError: foo bar\n")
+            else:
+                assert 'Traceback' not in x
+                assert '  File' not in x
+                assert '>\nTypeError: <lambda>() missing 2 required pos' in x
+        check_tb(res)
+        check_tb(res2)
+        check_tb(res3, traceback=False)
 
     def test_instance_overrides_meth(self):
         class C(object):

From pypy.commits at gmail.com  Fri Jan  6 12:12:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jan 2017 09:12:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: translation fix
Message-ID: <586fcfea.08301c0a.8588a.86d2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89404:593191898bc9
Date: 2017-01-06 17:22 +0000
http://bitbucket.org/pypy/pypy/changeset/593191898bc9/

Log:	translation fix

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -577,10 +577,11 @@
                 w_impl = space.get(w_del, w_obj)
             except Exception as e:
                 report_error(space, e, "method __del__ of ", w_obj)
-            try:
-                space.call_function(w_impl)
-            except Exception as e:
-                report_error(space, e, '', w_impl)
+            else:
+                try:
+                    space.call_function(w_impl)
+                except Exception as e:
+                    report_error(space, e, '', w_impl)
 
         # Call the RPython-level _finalize_() method.
         try:

From pypy.commits at gmail.com  Fri Jan  6 12:55:27 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 06 Jan 2017 09:55:27 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Don't pass the header to
 _create_api_func()
Message-ID: <586fda0f.08301c0a.8588a.9b4a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89405:4c1556231a40
Date: 2017-01-06 17:34 +0000
http://bitbucket.org/pypy/pypy/changeset/4c1556231a40/

Log:	Don't pass the header to _create_api_func()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -344,13 +344,54 @@
     - set `gil` to "acquire", "release" or "around" to acquire the GIL,
       release the GIL, or both
     """
+    if isinstance(restype, lltype.Typedef):
+        real_restype = restype.OF
+    else:
+        real_restype = restype
+    expect_integer = (isinstance(real_restype, lltype.Primitive) and
+                      rffi.cast(restype, 0) == 0)
     def decorate(func):
-        return _create_api_func(func, argtypes, restype, error, header, gil,
-            result_borrowed, result_is_ll)
+        if header is not None:
+            if func.__name__ in FUNCTIONS_BY_HEADER[header]:
+                raise ValueError("%s already registered" % func.__name__)
+        if header is not None:
+            c_name = None
+        else:
+            c_name = func.__name__
+
+        unwrapper = _create_api_func(
+            func, argtypes, restype, error, c_name, gil, result_borrowed,
+            result_is_ll)
+
+        # ZZZ is this whole logic really needed???  It seems to be only
+        # for RPython code calling PyXxx() functions directly.  I would
+        # think that usually directly calling the function is clean
+        # enough now
+        def unwrapper_catch(space, *args):
+            try:
+                res = unwrapper(space, *args)
+            except OperationError as e:
+                if not hasattr(unwrapper.api_func, "error_value"):
+                    raise
+                state = space.fromcache(State)
+                state.set_exception(e)
+                if is_PyObject(restype):
+                    return None
+                else:
+                    return unwrapper.api_func.error_value
+            got_integer = isinstance(res, (int, long, float))
+            assert got_integer == expect_integer, (
+                'got %r not integer' % (res,))
+            return res
+
+        if header is not None:
+            FUNCTIONS_BY_HEADER[header][func.__name__] = unwrapper.api_func
+            INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
+        return unwrapper
     return decorate
 
 def _create_api_func(
-        func, argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
+        func, argtypes, restype, error=_NOT_SPECIFIED, c_name=None,
         gil=None, result_borrowed=False, result_is_ll=False):
     if isinstance(restype, lltype.Typedef):
         real_restype = restype.OF
@@ -364,17 +405,8 @@
             error = CANNOT_FAIL
     if type(error) is int:
         error = rffi.cast(real_restype, error)
-    expect_integer = (isinstance(real_restype, lltype.Primitive) and
-                      rffi.cast(restype, 0) == 0)
 
     func._always_inline_ = 'try'
-    func_name = func.func_name
-    if header is not None:
-        c_name = None
-        if func_name in FUNCTIONS_BY_HEADER[header]:
-            raise ValueError("%s already registered" % func_name)
-    else:
-        c_name = func_name
     api_function = ApiFunction(argtypes, restype, func, error,
                                 c_name=c_name, gil=gil,
                                 result_borrowed=result_borrowed,
@@ -447,32 +479,7 @@
 
     unwrapper.func = func
     unwrapper.api_func = api_function
-
-    # ZZZ is this whole logic really needed???  It seems to be only
-    # for RPython code calling PyXxx() functions directly.  I would
-    # think that usually directly calling the function is clean
-    # enough now
-    def unwrapper_catch(space, *args):
-        try:
-            res = unwrapper(space, *args)
-        except OperationError as e:
-            if not hasattr(api_function, "error_value"):
-                raise
-            state = space.fromcache(State)
-            state.set_exception(e)
-            if is_PyObject(restype):
-                return None
-            else:
-                return api_function.error_value
-        got_integer = isinstance(res, (int, long, float))
-        assert got_integer == expect_integer, (
-            'got %r not integer' % (res,))
-        return res
-
-    if header is not None:
-        FUNCTIONS_BY_HEADER[header][func_name] = api_function
-        INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
-    return unwrapper  # used in 'normal' RPython code.
+    return unwrapper
 
 
 def cpython_struct(name, fields, forward=None, level=1):

From pypy.commits at gmail.com  Fri Jan  6 12:55:29 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 06 Jan 2017 09:55:29 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Don't store 'api_function' on
 the hidden 'func', it's already on 'unwrapper'; kill dead code
Message-ID: <586fda11.cb911c0a.87fc9.fe42@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89406:a004964c95d4
Date: 2017-01-06 17:54 +0000
http://bitbucket.org/pypy/pypy/changeset/a004964c95d4/

Log:	Don't store 'api_function' on the hidden 'func', it's already on
	'unwrapper'; kill dead code

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -411,11 +411,6 @@
                                 c_name=c_name, gil=gil,
                                 result_borrowed=result_borrowed,
                                 result_is_ll=result_is_ll)
-    func.api_func = api_function
-
-    if error is _NOT_SPECIFIED:
-        raise ValueError("function %s has no return value for exceptions"
-                            % func)
     names = api_function.argnames
     types_names_enum_ui = unrolling_iterable(enumerate(
         zip(api_function.argtypes,

From pypy.commits at gmail.com  Sat Jan  7 04:22:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 07 Jan 2017 01:22:42 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe: Start, mostly just playing around
Message-ID: <5870b362.4c9d1c0a.2a4b4.a391@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe
Changeset: r89408:c17b7079746f
Date: 2017-01-07 10:20 +0100
http://bitbucket.org/pypy/pypy/changeset/c17b7079746f/

Log:	Start, mostly just playing around

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -190,6 +190,11 @@
 FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
 NURSARRAY = lltype.Array(llmemory.Address)
 
+GCTL = lltype.Struct('GCThreadLocal',
+                     ('nursery_free', llmemory.Address),
+                     ('nursery_top', llmemory.Address),
+                     hints={'thread_local': True})
+
 # ____________________________________________________________
 
 class IncrementalMiniMarkGC(MovingGCBase):
@@ -269,12 +274,23 @@
         "card_page_indices": 128,
 
         # Objects whose total size is at least 'large_object' bytes are
-        # allocated out of the nursery immediately, as old objects.  The
-        # minimal allocated size of the nursery is 2x the following
-        # number (by default, at least 132KB on 32-bit and 264KB on 64-bit).
-        "large_object": (16384+512)*WORD,
+        # allocated out of the nursery immediately, as old objects.
+        "large_object": 13000,
+
+        # Thread-local Block size: the nursery is divided into blocks of
+        # at most this size each, and allocations go on in a
+        # thread-local manner inside each block.  "large_object" must be
+        # significantly smaller, but at the same time the total nursery
+        # size must be many times bigger than "tl_block_size"; the minimum
+        # allocated nursery size is 2 times "tl_block_size".
+        # "cache_line_min" is used to round the actual thread-local
+        # blocks to a cache line, to avoid pointless cache conflicts.
+        "tl_block_size": 32768,
+        "cache_line_min": 256,
         }
 
+    tl = lltype.malloc(GCTL, flavor='raw', immortal=True)
+
     def __init__(self, config,
                  read_from_env=False,
                  nursery_size=32*WORD,
@@ -286,6 +302,8 @@
                  growth_rate_max=2.5,   # for tests
                  card_page_indices=0,
                  large_object=8*WORD,
+                 tl_block_size=9*WORD,
+                 cache_line_min=1*WORD,
                  ArenaCollectionClass=None,
                  **kwds):
         "NOT_RPYTHON"
@@ -313,10 +331,12 @@
         # 'large_object' limit how big objects can be in the nursery, so
         # it gives a lower bound on the allowed size of the nursery.
         self.nonlarge_max = large_object - 1
+        self.tl_block_size = tl_block_size
+        self.cache_line_min = cache_line_min
         #
         self.nursery      = llmemory.NULL
-        self.nursery_free = llmemory.NULL
-        self.nursery_top  = llmemory.NULL
+        self.tl.nursery_free = llmemory.NULL
+        self.tl.nursery_top  = llmemory.NULL
         self.debug_tiny_nursery = -1
         self.debug_rotating_nurseries = lltype.nullptr(NURSARRAY)
         self.extra_threshold = 0
@@ -437,7 +457,7 @@
         else:
             #
             defaultsize = self.nursery_size
-            minsize = 2 * (self.nonlarge_max + 1)
+            minsize = 2 * self.tl_block_size
             self.nursery_size = minsize
             self.allocate_nursery()
             #
@@ -513,6 +533,13 @@
             # Estimate this number conservatively
             bigobj = self.nonlarge_max + 1
             self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2)
+        #
+        # Round up
+        ll_assert((self.cache_line_min & (self.cache_line_min - 1)) == 0,
+                  "cache_line_min is not a power a two")
+        self.tl_block_size = ((self.tl_block_size + self.cache_line_min - 1)
+                              & ~(self.cache_line_min - 1))
+
 
     def _nursery_memory_size(self):
         extra = self.nonlarge_max + 1
@@ -532,10 +559,6 @@
         debug_start("gc-set-nursery-size")
         debug_print("nursery size:", self.nursery_size)
         self.nursery = self._alloc_nursery()
-        # the current position in the nursery:
-        self.nursery_free = self.nursery
-        # the end of the nursery:
-        self.nursery_top = self.nursery + self.nursery_size
         # initialize the threshold
         self.min_heap_size = max(self.min_heap_size, self.nursery_size *
                                               self.major_collection_threshold)
@@ -608,7 +631,6 @@
             #
             llarena.arena_protect(newnurs, self._nursery_memory_size(), False)
             self.nursery = newnurs
-            self.nursery_top = self.nursery + self.nursery_size
             debug_print("switching from nursery", oldnurs,
                         "to nursery", self.nursery,
                         "size", self.nursery_size)
@@ -651,10 +673,10 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.nursery_free
+            result = self.tl.nursery_free
             ll_assert(result != llmemory.NULL, "uninitialized nursery")
-            self.nursery_free = new_free = result + totalsize
-            if new_free > self.nursery_top:
+            self.tl.nursery_free = new_free = result + totalsize
+            if new_free > self.tl.nursery_top:
                 result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
@@ -711,10 +733,10 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.nursery_free
+            result = self.tl.nursery_free
             ll_assert(result != llmemory.NULL, "uninitialized nursery")
-            self.nursery_free = new_free = result + totalsize
-            if new_free > self.nursery_top:
+            self.tl.nursery_free = new_free = result + totalsize
+            if new_free > self.tl.nursery_top:
                 result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
@@ -802,13 +824,14 @@
         Otherwise do a minor collection, and possibly some steps of a
         major collection, and finally reserve totalsize bytes.
         """
-
         minor_collection_count = 0
+        must_downgrade_gil = False
         while True:
-            self.nursery_free = llmemory.NULL      # debug: don't use me
+            self.tl.nursery_free = llmemory.NULL      # debug: don't use me
             # note: no "raise MemoryError" between here and the next time
             # we initialize nursery_free!
 
+            self._gc_lock()
             if self.nursery_barriers.non_empty():
                 # Pinned object in front of nursery_top. Try reserving totalsize
                 # by jumping into the next, yet unused, area inside the
@@ -822,7 +845,7 @@
                 #     v     v    v  jump over this
                 # +---------+--------+--------+--------+-----------+ }
                 # | used    | pinned | empty  | pinned |  empty    | }- nursery
-                # +---------+--------+--------+--------+-----------+ }
+                # +---------+--------B--------B--------B-----------B }
                 #                       ^- try reserving totalsize in here next
                 #
                 # All pinned objects are represented by entries in
@@ -833,15 +856,21 @@
                 # totalsize) starts at the end of the pinned object and ends at
                 # nursery's end.
                 #
-                # find the size of the pinned object after nursery_top
-                size_gc_header = self.gcheaderbuilder.size_gc_header
-                pinned_obj_size = size_gc_header + self.get_size(
-                        self.nursery_top + size_gc_header)
+                # In the diagram above, self.nursery_barriers contains
+                # four addresses which match the four "B".
                 #
                 # update used nursery space to allocate objects
-                self.nursery_free = self.nursery_top + pinned_obj_size
-                self.nursery_top = self.nursery_barriers.popleft()
+                self.tl.nursery_free = self.nursery_barriers.popleft()
+                self.tl.nursery_top = self.nursery_barriers.popleft()
+                self._gc_unlock()
+                prev_gil = False
             else:
+                self._gc_unlock()
+                if not llop.gil_is_exclusive(lltype.Bool):
+                    ll_assert(not must_downgrade_gil,
+                              "collect_and_reverse: bad gil state")
+                    must_downgrade_gil = llop.gil_wait(lltype.Bool)
+                    continue      # waited, maybe the situation changed
                 minor_collection_count += 1
                 if minor_collection_count == 1:
                     self.minor_collection_with_major_progress()
@@ -868,16 +897,18 @@
             # Tried to do something about nursery_free overflowing
             # nursery_top before this point. Try to reserve totalsize now.
             # If this succeeds break out of loop.
-            result = self.nursery_free
-            if self.nursery_free + totalsize <= self.nursery_top:
-                self.nursery_free = result + totalsize
-                ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow")
+            result = self.tl.nursery_free
+            if self.tl.nursery_free + totalsize <= self.tl.nursery_top:
+                self.tl.nursery_free = result + totalsize
+                ll_assert(self.tl.nursery_free <= self.tl.nursery_top, "nursery overflow")
                 break
             #
+        if must_downgrade_gil:
+            llop.gil_downgrade(lltype.Void)
         #
         if self.debug_tiny_nursery >= 0:   # for debugging
-            if self.nursery_top - self.nursery_free > self.debug_tiny_nursery:
-                self.nursery_free = self.nursery_top - self.debug_tiny_nursery
+            if self.tl.nursery_top - self.tl.nursery_free > self.debug_tiny_nursery:
+                self.tl.nursery_free = self.tl.nursery_top - self.debug_tiny_nursery
         #
         return result
     collect_and_reserve._dont_inline_ = True
@@ -1037,7 +1068,7 @@
         if self.next_major_collection_threshold < 0:
             # cannot trigger a full collection now, but we can ensure
             # that one will occur very soon
-            self.nursery_free = self.nursery_top
+            self.tl.nursery_free = self.tl.nursery_top
 
     def can_optimize_clean_setarrayitems(self):
         if self.card_page_indices > 0:
@@ -1144,7 +1175,7 @@
         # Check if the object at 'addr' is young.
         if not self.is_valid_gc_object(addr):
             return False     # filter out tagged pointers explicitly.
-        if self.nursery <= addr < self.nursery_top:
+        if self.nursery <= addr < self.nursery + self.nursery_size:
             return True      # addr is in the nursery
         # Else, it may be in the set 'young_rawmalloced_objects'
         return (bool(self.young_rawmalloced_objects) and
@@ -1756,51 +1787,82 @@
         # pointer.
         size_gc_header = self.gcheaderbuilder.size_gc_header
         nursery_barriers = self.AddressDeque()
-        prev = self.nursery
-        self.surviving_pinned_objects.sort()
+        if self.surviving_pinned_objects.non_empty():
+            self.surviving_pinned_objects.sort()
+            next_pinned_object = self.surviving_pinned_objects.pop()
+        else:
+            next_pinned_object = llmemory.NULL
         ll_assert(
             self.pinned_objects_in_nursery == \
             self.surviving_pinned_objects.length(),
             "pinned_objects_in_nursery != surviving_pinned_objects.length()")
-        while self.surviving_pinned_objects.non_empty():
+
+        # The following loop divides the nursery into small blocks whose
+        # size is generally about 'self.tl_block_size', but skipping
+        # over any pinned object.  Depending on the position of pinned
+        # objects, it is possible that one or two of these blocks are
+        # unusable because they are too small, but it should not matter.
+        prev = self.nursery
+        full_end = self.nursery + self.nursery_size
+
+        while True:
+            # Round up 'prev' to a multiple of 'cache_line_min'
+            prev_num = llmemory.cast_adr_to_int(prev)
+            prev += (-prev_num) & (self.cache_line_min - 1)
             #
-            cur = self.surviving_pinned_objects.pop()
-            ll_assert(
-                cur >= prev, "pinned objects encountered in backwards order")
+            # Compute the next TL block limit as 'cur1' and 'cur2'.
+            # These two addresses are normally equal to each other,
+            # but if there is a pinned object, then 'cur1' is the
+            # start of the pinned object and 'cur2' the end.
             #
-            # clear the arena between the last pinned object (or arena start)
-            # and the pinned object
-            pinned_obj_size = llarena.getfakearenaaddress(cur) - prev
+            if full_end - prev <= block_size:
+                cur1 = full_end
+            else:
+                cur1 = prev + block_size
+            #
+            if next_pinned_object and next_pinned_object <= cur1:
+                cur1 = next_pinned_object
+                if self.surviving_pinned_objects.non_empty():
+                    next_pinned_object = self.surviving_pinned_objects.pop()
+                else:
+                    next_pinned_object = llmemory.NULL
+                ll_assert(cur1 >= prev,
+                          "pinned objects encountered in backwards order")
+                # clean up object's flags
+                obj = cur1 + size_gc_header
+                self.header(obj).tid &= ~GCFLAG_VISITED
+                # set up 'cur1' and 'cur2'
+                cur1 = llarena.getfakearenaaddress(cur1)
+                cur2 = cur1 + (size_gc_header + self.get_size(obj))
+            else:
+                # no pinned object in this TL block.
+                cur2 = cur1
+            #
+            # clear this block in the arena
+            free_range_size = cur1 - prev
             if self.gc_nursery_debug:
-                llarena.arena_reset(prev, pinned_obj_size, 3)
+                llarena.arena_reset(prev, free_range_size, 3)
             else:
-                llarena.arena_reset(prev, pinned_obj_size, 0)
-            #
-            # clean up object's flags
-            obj = cur + size_gc_header
-            self.header(obj).tid &= ~GCFLAG_VISITED
+                llarena.arena_reset(prev, free_range_size, 0)
             #
             # create a new nursery barrier for the pinned object
-            nursery_barriers.append(cur)
+            nursery_barriers.append(cur1)    # pinned object
+            if cur1 == full_end:
+                break
+            nursery_barriers.append(cur2)    # end of pinned object
             #
-            # update 'prev' to the end of the 'cur' object
-            prev = prev + pinned_obj_size + \
-                (size_gc_header + self.get_size(obj))
+            # update 'prev' for the next iteration
+            prev = cur2
         #
-        # reset everything after the last pinned object till the end of the arena
+        ll_assert(not next_pinned_object, "bad pinned object location")
         if self.gc_nursery_debug:
-            llarena.arena_reset(prev, self.nursery + self.nursery_size - prev, 3)
             if not nursery_barriers.non_empty():   # no pinned objects
                 self.debug_rotate_nursery()
-        else:
-            llarena.arena_reset(prev, self.nursery + self.nursery_size - prev, 0)
-        #
-        # always add the end of the nursery to the list
-        nursery_barriers.append(self.nursery + self.nursery_size)
         #
         self.nursery_barriers = nursery_barriers
         self.surviving_pinned_objects.delete()
         #
+        XXX must clear out the other threads nursery_free/nursery_top
         self.nursery_free = self.nursery
         self.nursery_top = self.nursery_barriers.popleft()
         #
diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py
--- a/rpython/translator/exceptiontransform.py
+++ b/rpython/translator/exceptiontransform.py
@@ -452,7 +452,9 @@
     def setup_excdata(self):
         EXCDATA = lltype.Struct('ExcData',
             ('exc_type',  self.lltype_of_exception_type),
-            ('exc_value', self.lltype_of_exception_value))
+            ('exc_value', self.lltype_of_exception_value),
+            #('have_debug_prints', lltype.Signed),
+            hints={'thread_local': True})
         self.EXCDATA = EXCDATA
 
         exc_data = lltype.malloc(EXCDATA, immortal=True)

From pypy.commits at gmail.com  Sat Jan  7 04:22:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 07 Jan 2017 01:22:40 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe: Experimental: playing with a
 no-gil version of RPython
Message-ID: <5870b360.0d1a1c0a.d4e06.a384@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe
Changeset: r89407:6cea2be6e621
Date: 2017-01-05 15:22 +0100
http://bitbucket.org/pypy/pypy/changeset/6cea2be6e621/

Log:	Experimental: playing with a no-gil version of RPython


From pypy.commits at gmail.com  Sat Jan  7 04:59:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 07 Jan 2017 01:59:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: zlib: CPython issue27164
Message-ID: <5870bbf6.42061c0a.18fe5.b280@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89409:ee9a45377582
Date: 2017-01-07 10:56 +0100
http://bitbucket.org/pypy/pypy/changeset/ee9a45377582/

Log:	zlib: CPython issue27164

diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -122,9 +122,7 @@
         ZLibObject.__init__(self, space)
         try:
             self.stream = rzlib.deflateInit(level, method, wbits,
-                                            memLevel, strategy)
-            if zdict is not None:
-                rzlib.deflateSetDictionary(self.stream, zdict)
+                                            memLevel, strategy, zdict=zdict)
         except rzlib.RZlibError as e:
             raise zlib_error(space, e.msg)
         except ValueError:
@@ -242,7 +240,7 @@
         self.unconsumed_tail = ''
         self.eof = False
         try:
-            self.stream = rzlib.inflateInit(wbits)
+            self.stream = rzlib.inflateInit(wbits, zdict=zdict)
         except rzlib.RZlibError as e:
             raise zlib_error(space, e.msg)
         except ValueError:
diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py
--- a/pypy/module/zlib/test/test_zlib.py
+++ b/pypy/module/zlib/test/test_zlib.py
@@ -320,3 +320,13 @@
     def test_version(self):
         zlib = self.zlib
         assert zlib.ZLIB_VERSION[0] == zlib.ZLIB_RUNTIME_VERSION[0]
+
+    # CPython issue27164
+    def test_decompress_raw_with_dictionary(self):
+        zlib = self.zlib
+        zdict = b'abcdefghijklmnopqrstuvwxyz'
+        co = zlib.compressobj(wbits=-zlib.MAX_WBITS, zdict=zdict)
+        comp = co.compress(zdict) + co.flush()
+        dco = zlib.decompressobj(wbits=-zlib.MAX_WBITS, zdict=zdict)
+        uncomp = dco.decompress(comp) + dco.flush()
+        assert zdict == uncomp
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -51,6 +51,7 @@
     voidpf = rffi_platform.SimpleType('voidpf', rffi.VOIDP)
 
     ZLIB_VERSION = rffi_platform.DefinedConstantString('ZLIB_VERSION')
+    ZLIB_VERNUM = rffi_platform.DefinedConstantInteger('ZLIB_VERNUM')
 
 for _name in constantnames:
     setattr(SimpleCConfig, _name, rffi_platform.ConstantInteger(_name))
@@ -63,6 +64,7 @@
 Bytefp = lltype.Ptr(lltype.Array(Bytef, hints={'nolength': True}))
 
 ZLIB_VERSION = config['ZLIB_VERSION']
+ZLIB_VERNUM = config['ZLIB_VERNUM']
 
 for _name in constantnames:
     globals()[_name] = config[_name]
@@ -261,7 +263,7 @@
 
 def deflateInit(level=Z_DEFAULT_COMPRESSION, method=Z_DEFLATED,
                 wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL,
-                strategy=Z_DEFAULT_STRATEGY):
+                strategy=Z_DEFAULT_STRATEGY, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     compress data.
@@ -270,6 +272,12 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
     if err == Z_OK:
+        if zdict is not None:
+            try:
+                deflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:
@@ -290,7 +298,7 @@
     lltype.free(stream, flavor='raw')
 
 
-def inflateInit(wbits=MAX_WBITS):
+def inflateInit(wbits=MAX_WBITS, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     decompress data.
@@ -299,6 +307,17 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _inflateInit2(stream, wbits)
     if err == Z_OK:
+        if zdict is not None and wbits < 0:
+            try:
+                if ZLIB_VERNUM is None or ZLIB_VERNUM < 0x1221:
+                    raise RZlibError("zlib version %s does not allow raw "
+                                     "inflate with dictionary" %
+                                       ZLIB_VERSION if ZLIB_VERSION is not None
+                                       else "<unknown>")
+                inflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:

From pypy.commits at gmail.com  Sat Jan  7 04:59:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 07 Jan 2017 01:59:20 -0800 (PST)
Subject: [pypy-commit] pypy default: Backport ee9a45377582
Message-ID: <5870bbf8.6a5cc20a.d0b08.7425@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89410:f3abf62ba364
Date: 2017-01-07 10:57 +0100
http://bitbucket.org/pypy/pypy/changeset/f3abf62ba364/

Log:	Backport ee9a45377582

diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -51,6 +51,7 @@
     voidpf = rffi_platform.SimpleType('voidpf', rffi.VOIDP)
 
     ZLIB_VERSION = rffi_platform.DefinedConstantString('ZLIB_VERSION')
+    ZLIB_VERNUM = rffi_platform.DefinedConstantInteger('ZLIB_VERNUM')
 
 for _name in constantnames:
     setattr(SimpleCConfig, _name, rffi_platform.ConstantInteger(_name))
@@ -63,6 +64,7 @@
 Bytefp = lltype.Ptr(lltype.Array(Bytef, hints={'nolength': True}))
 
 ZLIB_VERSION = config['ZLIB_VERSION']
+ZLIB_VERNUM = config['ZLIB_VERNUM']
 
 for _name in constantnames:
     globals()[_name] = config[_name]
@@ -261,7 +263,7 @@
 
 def deflateInit(level=Z_DEFAULT_COMPRESSION, method=Z_DEFLATED,
                 wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL,
-                strategy=Z_DEFAULT_STRATEGY):
+                strategy=Z_DEFAULT_STRATEGY, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     compress data.
@@ -270,6 +272,12 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
     if err == Z_OK:
+        if zdict is not None:
+            try:
+                deflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:
@@ -290,7 +298,7 @@
     lltype.free(stream, flavor='raw')
 
 
-def inflateInit(wbits=MAX_WBITS):
+def inflateInit(wbits=MAX_WBITS, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     decompress data.
@@ -299,6 +307,17 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _inflateInit2(stream, wbits)
     if err == Z_OK:
+        if zdict is not None and wbits < 0:
+            try:
+                if ZLIB_VERNUM is None or ZLIB_VERNUM < 0x1221:
+                    raise RZlibError("zlib version %s does not allow raw "
+                                     "inflate with dictionary" %
+                                       ZLIB_VERSION if ZLIB_VERSION is not None
+                                       else "<unknown>")
+                inflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:

From pypy.commits at gmail.com  Sun Jan  8 09:57:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 06:57:06 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix the exception class we get
Message-ID: <58725342.4dd41c0a.82ec4.a6b4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89411:2cf5804b5f91
Date: 2017-01-08 15:24 +0100
http://bitbucket.org/pypy/pypy/changeset/2cf5804b5f91/

Log:	Fix the exception class we get

diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -69,12 +69,12 @@
     # Validate category
     try:
         if not space.abstract_issubclass_w(w_category, space.w_Warning):
-            raise oefmt(space.w_ValueError,
+            raise oefmt(space.w_TypeError,
                         "category is not a subclass of Warning")
     except OperationError as e:
         if e.async(space):
             raise
-        raise oefmt(space.w_ValueError,
+        raise oefmt(space.w_TypeError,
                     "category must be a Warning subclass, not '%T'",
                     w_category)
 
diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -80,3 +80,10 @@
         _warnings.warn('test', UserWarning)
         globals()['__file__'] = None
         _warnings.warn('test', UserWarning)
+
+    def test_bad_category(self):
+        import _warnings
+        raises(TypeError, _warnings.warn, "text", 123)
+        class Foo:
+            pass
+        raises(TypeError, _warnings.warn, "text", Foo)

From pypy.commits at gmail.com  Sun Jan  8 09:57:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 06:57:08 -0800 (PST)
Subject: [pypy-commit] pypy default: Add an XXX about .decode('utf-8') in
 RPython
Message-ID: <58725344.0209c20a.3e8d9.ad05@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89412:b144076bf8d4
Date: 2017-01-08 15:54 +0100
http://bitbucket.org/pypy/pypy/changeset/b144076bf8d4/

Log:	Add an XXX about .decode('utf-8') in RPython

diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -33,6 +33,10 @@
             value, len(value), 'strict', final=True,
             errorhandler=self.ll_raise_unicode_exception_decode,
             allow_surrogates=False, result=result)
+        # XXX should it really be 'allow_surrogates=False'?  In RPython,
+        # unicode.decode('utf-8') happily accepts surrogates.  This
+        # makes it hard to test untranslated (it's the cause of a
+        # failure in lib-python's test_warnings on PyPy3, for example)
         return self.ll.llunicode(result.build())
 
     @staticmethod

From pypy.commits at gmail.com  Sun Jan  8 09:57:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 06:57:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Trying it out in this order:
 PYTHONWARNINGS should have lower priority than explicit -W options
Message-ID: <58725346.12921c0a.5fd96.b2cc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89413:62f33b906529
Date: 2017-01-08 15:56 +0100
http://bitbucket.org/pypy/pypy/changeset/62f33b906529/

Log:	Trying it out in this order: PYTHONWARNINGS should have lower
	priority than explicit -W options

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -586,7 +586,7 @@
 
     pythonwarnings = readenv and os.getenv('PYTHONWARNINGS')
     if pythonwarnings:
-        warnoptions.extend(pythonwarnings.split(','))
+        warnoptions = pythonwarnings.split(',') + warnoptions
     if warnoptions:
         sys.warnoptions[:] = warnoptions
         from warnings import _processoptions

From pypy.commits at gmail.com  Sun Jan  8 09:59:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 06:59:36 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix the check for "is a built-in function"
Message-ID: <587253d8.2105c30a.98f04.2463@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89414:6f5a8cd7e3b2
Date: 2017-01-08 15:59 +0100
http://bitbucket.org/pypy/pypy/changeset/6f5a8cd7e3b2/

Log:	Fix the check for "is a built-in function"

diff --git a/lib-python/3/test/test_warnings/__init__.py b/lib-python/3/test/test_warnings/__init__.py
--- a/lib-python/3/test/test_warnings/__init__.py
+++ b/lib-python/3/test/test_warnings/__init__.py
@@ -561,8 +561,9 @@
     # As an early adopter, we sanity check the
     # test.support.import_fresh_module utility function
     def test_accelerated(self):
+        import types
         self.assertFalse(original_warnings is self.module)
-        self.assertFalse(hasattr(self.module.warn, '__code__'))
+        self.assertIs(type(self.module.warn), types.BuiltinFunctionType)
 
 class PyWarnTests(WarnTests, unittest.TestCase):
     module = py_warnings
@@ -570,8 +571,9 @@
     # As an early adopter, we sanity check the
     # test.support.import_fresh_module utility function
     def test_pure_python(self):
+        import types
         self.assertFalse(original_warnings is self.module)
-        self.assertTrue(hasattr(self.module.warn, '__code__'))
+        self.assertIs(type(self.module.warn), types.FunctionType)
 
 
 class WCmdLineTests(BaseTest):

From pypy.commits at gmail.com  Sun Jan  8 10:02:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 07:02:27 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: missing gc.collect()
Message-ID: <58725483.08301c0a.8588a.ae20@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89415:4c3fa9a270fd
Date: 2017-01-08 16:01 +0100
http://bitbucket.org/pypy/pypy/changeset/4c3fa9a270fd/

Log:	missing gc.collect()

diff --git a/lib-python/3/test/test_warnings/__init__.py b/lib-python/3/test/test_warnings/__init__.py
--- a/lib-python/3/test/test_warnings/__init__.py
+++ b/lib-python/3/test/test_warnings/__init__.py
@@ -1003,12 +1003,11 @@
     def __del__(self):
         warn("test")
 
-a=A()
+A()
+import gc; gc.collect()
         """
         rc, out, err = assert_python_ok("-c", code)
-        # note: "__main__" filename is not correct, it should be the name
-        # of the script
-        self.assertEqual(err, b'__main__:7: UserWarning: test')
+        self.assertEqual(err, b'-c:7: UserWarning: test')
 
     def test_late_resource_warning(self):
         # Issue #21925: Emitting a ResourceWarning late during the Python

From pypy.commits at gmail.com  Sun Jan  8 10:04:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 07:04:37 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: this test is really CPython-only
Message-ID: <58725505.973f1c0a.1c20.b2e0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89416:52da0cf18848
Date: 2017-01-08 16:03 +0100
http://bitbucket.org/pypy/pypy/changeset/52da0cf18848/

Log:	this test is really CPython-only

diff --git a/lib-python/3/test/test_warnings/__init__.py b/lib-python/3/test/test_warnings/__init__.py
--- a/lib-python/3/test/test_warnings/__init__.py
+++ b/lib-python/3/test/test_warnings/__init__.py
@@ -1009,6 +1009,7 @@
         rc, out, err = assert_python_ok("-c", code)
         self.assertEqual(err, b'-c:7: UserWarning: test')
 
+    @support.cpython_only
     def test_late_resource_warning(self):
         # Issue #21925: Emitting a ResourceWarning late during the Python
         # shutdown must be logged.

From pypy.commits at gmail.com  Sun Jan  8 10:19:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 07:19:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Issue #17032: The "global" in the
 "NameError: global name 'x' is not defined"
Message-ID: <58725898.6737c20a.6877a.90af@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89417:cbb0db3f773d
Date: 2017-01-08 16:19 +0100
http://bitbucket.org/pypy/pypy/changeset/cbb0db3f773d/

Log:	Issue #17032: The "global" in the "NameError: global name 'x' is not
	defined" error message has been removed.

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -923,8 +923,10 @@
 
     @dont_inline
     def _load_global_failed(self, w_varname):
+        # CPython Issue #17032: The "global" in the "NameError: global
+        # name 'x' is not defined" error message has been removed.
         raise oefmt(self.space.w_NameError,
-                    "global name %R is not defined", w_varname)
+                    "name %R is not defined", w_varname)
 
     @always_inline
     def LOAD_GLOBAL(self, nameindex, next_instr):

From pypy.commits at gmail.com  Sun Jan  8 10:30:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 07:30:06 -0800 (PST)
Subject: [pypy-commit] pypy default: list.insert() in RPython should not
 force the list in the JIT, at least
Message-ID: <58725afe.54161c0a.87599.bc27@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89418:9c99515a58a3
Date: 2017-01-08 16:29 +0100
http://bitbucket.org/pypy/pypy/changeset/9c99515a58a3/

Log:	list.insert() in RPython should not force the list in the JIT, at
	least if the index is constant

diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -210,7 +210,6 @@
     return rlist.ll_pop(rlist.dum_checkidx, l, index)
 _ll_2_list_append = rlist.ll_append
 _ll_2_list_extend = rlist.ll_extend
-_ll_3_list_insert = rlist.ll_insert_nonneg
 _ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly
 _ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop
 _ll_2_list_inplace_mul = rlist.ll_inplace_mul
diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -212,6 +212,8 @@
                 s += lst[0]
                 lst.pop()
                 lst.append(1)
+                lst.insert(0, 5)
+                lst.insert(1, 6)
                 s *= lst.pop()
             return s
         res = self.meta_interp(f, [15], listops=True)
diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py
--- a/rpython/rtyper/rlist.py
+++ b/rpython/rtyper/rlist.py
@@ -588,6 +588,7 @@
     l.ll_setitem_fast(length, newitem)
 
 # this one is for the special case of insert(0, x)
+ at jit.look_inside_iff(lambda l,n: jit.isvirtual(l))
 def ll_prepend(l, newitem):
     length = l.ll_length()
     l._ll_resize_ge(length+1)           # see "a note about overflows" above
@@ -597,7 +598,6 @@
         l.ll_setitem_fast(dst, l.ll_getitem_fast(src))
         dst = src
     l.ll_setitem_fast(0, newitem)
-ll_prepend.oopspec = 'list.insert(l, 0, newitem)'
 
 def ll_concat(RESLIST, l1, l2):
     len1 = l1.ll_length()
@@ -612,6 +612,7 @@
     return l
 # no oopspec -- the function is inlined by the JIT
 
+ at jit.look_inside_iff(lambda l,i,n: jit.isvirtual(l) and jit.isconstant(i))
 def ll_insert_nonneg(l, index, newitem):
     length = l.ll_length()
     ll_assert(0 <= index, "negative list insertion index")
@@ -623,7 +624,6 @@
         l.ll_setitem_fast(dst, l.ll_getitem_fast(src))
         dst = src
     l.ll_setitem_fast(index, newitem)
-ll_insert_nonneg.oopspec = 'list.insert(l, index, newitem)'
 
 def ll_pop_nonneg(func, l, index):
     ll_assert(index >= 0, "unexpectedly negative list pop index")

From pypy.commits at gmail.com  Sun Jan  8 10:45:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 07:45:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <58725eae.c515c20a.a3d17.9f3b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89419:de42c521f47a
Date: 2017-01-08 16:45 +0100
http://bitbucket.org/pypy/pypy/changeset/de42c521f47a/

Log:	fix

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -32,7 +32,7 @@
         w_sys.flush_std_files(space)
 
         msg = space.bytes_w(space.call_function(w_read))
-        assert 'Exception OSError' in msg
+        assert 'Exception ignored in:' in msg and '\nOSError' in msg
     finally:
         space.setattr(w_sys, space.wrap('stdout'), w_sys.get('__stdout__'))
         space.setattr(w_sys, space.wrap('stderr'), w_sys.get('__stderr__'))

From pypy.commits at gmail.com  Sun Jan  8 10:59:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 07:59:40 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add an extra test
Message-ID: <587261ec.2350c20a.e5711.8d88@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89420:ea5c8eea66d5
Date: 2017-01-08 16:55 +0100
http://bitbucket.org/pypy/pypy/changeset/ea5c8eea66d5/

Log:	add an extra test

diff --git a/lib-python/3/test/test_decimal.py b/lib-python/3/test/test_decimal.py
--- a/lib-python/3/test/test_decimal.py
+++ b/lib-python/3/test/test_decimal.py
@@ -4039,6 +4039,11 @@
         self.assertRaises(TypeError, Context, flags=(0,1))
         self.assertRaises(TypeError, Context, traps=(1,0))
 
+    def test_context_from_signaldict(self):
+        ctx = self.decimal.Context()
+        ctx2 = self.decimal.Context(flags=ctx.flags)
+        assert ctx.flags == ctx2.flags
+
 class CContextInputValidation(ContextInputValidation):
     decimal = C
 class PyContextInputValidation(ContextInputValidation):

From pypy.commits at gmail.com  Sun Jan  8 10:59:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 07:59:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Pass test_invalid_context
Message-ID: <587261ed.e7b1c20a.678f4.7c4f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89421:feb960899f2a
Date: 2017-01-08 16:59 +0100
http://bitbucket.org/pypy/pypy/changeset/feb960899f2a/

Log:	Pass test_invalid_context

diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -1086,26 +1086,30 @@
 
         if traps is None:
             ctx.traps = dc.traps
-        elif not isinstance(traps, dict):
+        elif isinstance(traps, list):
             ctx.traps = 0
             for signal in traps:
                 ctx.traps |= _SIGNALS[signal]
-        else:
+        elif isinstance(traps, dict):
             ctx.traps = 0
             for signal, value in traps.items():
                 if value:
                     ctx.traps |= _SIGNALS[signal]
+        else:
+            self.traps = traps
 
         if flags is None:
             ctx.status = 0
-        elif not isinstance(flags, dict):
+        elif isinstance(flags, list):
             ctx.status = 0
             for signal in flags:
                 ctx.status |= _SIGNALS[signal]
-        else:
+        elif isinstance(flags, dict):
             for signal, value in flags.items():
                 if value:
                     ctx.status |= _SIGNALS[signal]
+        else:
+            self.flags = flags
 
     def clear_flags(self):
         self._ctx.status = 0

From pypy.commits at gmail.com  Sun Jan  8 11:05:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 08:05:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Generalize the test to accept both
 ValueError and OverflowError for some cases
Message-ID: <58726355.cf3fc20a.aa7e8.827e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89422:5169bbf58c6c
Date: 2017-01-08 17:05 +0100
http://bitbucket.org/pypy/pypy/changeset/5169bbf58c6c/

Log:	Generalize the test to accept both ValueError and OverflowError for
	some cases

diff --git a/lib-python/3/test/test_decimal.py b/lib-python/3/test/test_decimal.py
--- a/lib-python/3/test/test_decimal.py
+++ b/lib-python/3/test/test_decimal.py
@@ -4808,15 +4808,24 @@
         self.assertRaises(OverflowError, Context, Emax=int_max+1)
         self.assertRaises(OverflowError, Context, Emin=-int_max-2)
         self.assertRaises(OverflowError, Context, clamp=int_max+1)
-        self.assertRaises(OverflowError, Context, capitals=int_max+1)
+        self.assertRaises((OverflowError, ValueError),
+                                         Context, capitals=int_max+1)
 
         # OverflowError, general ValueError
         for attr in ('prec', 'Emin', 'Emax', 'capitals', 'clamp'):
-            self.assertRaises(OverflowError, setattr, c, attr, int_max+1)
-            self.assertRaises(OverflowError, setattr, c, attr, -int_max-2)
+            if attr == 'capitals':
+                err = (OverflowError, ValueError)
+            else:
+                err = OverflowError
+            self.assertRaises(err, setattr, c, attr, int_max+1)
+            self.assertRaises(err, setattr, c, attr, -int_max-2)
             if sys.platform != 'win32':
-                self.assertRaises(ValueError, setattr, c, attr, int_max)
-                self.assertRaises(ValueError, setattr, c, attr, -int_max-1)
+                if attr == 'clamp':
+                    err = (ValueError, OverflowError)
+                else:
+                    err = ValueError
+                self.assertRaises(err, setattr, c, attr, int_max)
+                self.assertRaises(err, setattr, c, attr, -int_max-1)
 
         # OverflowError: _unsafe_setprec, _unsafe_setemin, _unsafe_setemax
         if C.MAX_PREC == 425000000:
@@ -4845,8 +4854,9 @@
             self.assertRaises(ValueError, setattr, c, attr, 2)
             self.assertRaises(TypeError, setattr, c, attr, [1,2,3])
             if HAVE_CONFIG_64:
-                self.assertRaises(ValueError, setattr, c, attr, 2**32)
-                self.assertRaises(ValueError, setattr, c, attr, 2**32+1)
+                err = (ValueError, OverflowError)
+                self.assertRaises(err, setattr, c, attr, 2**32)
+                self.assertRaises(err, setattr, c, attr, 2**32+1)
 
         # Invalid local context
         self.assertRaises(TypeError, exec, 'with localcontext("xyz"): pass',

From pypy.commits at gmail.com  Sun Jan  8 11:12:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 08:12:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <587264d9.92ae1c0a.76308.b60b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89423:2e6e0d9eba8a
Date: 2017-01-08 17:06 +0100
http://bitbucket.org/pypy/pypy/changeset/2e6e0d9eba8a/

Log:	fix test

diff --git a/lib-python/3/test/test_decimal.py b/lib-python/3/test/test_decimal.py
--- a/lib-python/3/test/test_decimal.py
+++ b/lib-python/3/test/test_decimal.py
@@ -4185,7 +4185,9 @@
 
         x = [s for s in dir(C.Context()) if '__' in s or not s.startswith('_')]
         y = [s for s in dir(P.Context()) if '__' in s or not s.startswith('_')]
-        self.assertEqual(set(x) - set(y), set())
+        extra = set(x) - set(y)
+        extra.discard('__slots__')
+        self.assertEqual(extra, set())
 
     def test_decimal_attributes(self):
 

From pypy.commits at gmail.com  Sun Jan  8 11:12:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 08:12:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix for test_inspect_module
Message-ID: <587264db.0777c20a.c29ab.9dd3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89424:acb109bf07ca
Date: 2017-01-08 17:11 +0100
http://bitbucket.org/pypy/pypy/changeset/acb109bf07ca/

Log:	Fix for test_inspect_module

diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -102,9 +102,10 @@
 # Default context
 
 import threading
-local = threading.local()
+__local = threading.local()
+del threading
 
-def getcontext(*, _local=local):
+def getcontext():
     """Returns this thread's context.
     
     If this thread does not yet have a context, returns
@@ -112,10 +113,10 @@
     New contexts are copies of DefaultContext.
     """
     try:
-        return _local.__decimal_context__
+        return __local.__decimal_context__
     except AttributeError:
         context = Context()
-        _local.__decimal_context__ = context
+        __local.__decimal_context__ = context
         return context
 
 def _getcontext(context=None):
@@ -125,17 +126,14 @@
         raise TypeError
     return context
 
-def setcontext(context, *, _local=local):
+def setcontext(context):
     """Set this thread's context to context."""
     if context in (DefaultContext, BasicContext, ExtendedContext):
         context = context.copy()
         context.clear_flags()
     if not isinstance(context, Context):
         raise TypeError
-    _local.__decimal_context__ = context
-
-
-del local, threading
+    __local.__decimal_context__ = context
 
 def localcontext(ctx=None):
     """Return a context manager for a copy of the supplied context.
@@ -1042,7 +1040,9 @@
 
     __slots__ = ('_ctx', '_capitals')
 
-    def __new__(cls, *args, **kwargs):
+    def __new__(cls, prec=None, rounding=None, Emin=None, Emax=None,
+                capitals=None, clamp=None, flags=None, traps=None):
+        # NOTE: the arguments are ignored here, they are used in __init__()
         self = object.__new__(cls)
         self._ctx = ctx = _ffi.new("struct mpd_context_t*")
         # Default context

From pypy.commits at gmail.com  Sun Jan  8 11:29:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 08:29:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Rename argument, as per test_inspect_types
Message-ID: <587268f9.973f1c0a.1c20.cf72@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89426:eed5601c6e65
Date: 2017-01-08 17:29 +0100
http://bitbucket.org/pypy/pypy/changeset/eed5601c6e65/

Log:	Rename argument, as per test_inspect_types

diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -1260,8 +1260,8 @@
     def create_decimal(self, num="0"):
         return Decimal._from_object(num, self, exact=False)
 
-    def create_decimal_from_float(self, value):
-        return Decimal._from_float(value, self, exact=False)
+    def create_decimal_from_float(self, f):
+        return Decimal._from_float(f, self, exact=False)
 
     # operations
     def _convert_unaryop(self, a, *, strict=True):

From pypy.commits at gmail.com  Sun Jan  8 11:29:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 08:29:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix test_inspect_types (shows real
 failures now)
Message-ID: <587268f7.4dd41c0a.82ec4.c5ba@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89425:b7c079c171ab
Date: 2017-01-08 17:27 +0100
http://bitbucket.org/pypy/pypy/changeset/b7c079c171ab/

Log:	Fix test_inspect_types (shows real failures now)

diff --git a/lib-python/3/test/test_decimal.py b/lib-python/3/test/test_decimal.py
--- a/lib-python/3/test/test_decimal.py
+++ b/lib-python/3/test/test_decimal.py
@@ -35,7 +35,7 @@
 from test.support import (run_unittest, run_doctest, is_resource_enabled,
                           requires_IEEE_754, requires_docstrings)
 from test.support import (check_warnings, import_fresh_module, TestFailed,
-                          run_with_locale, cpython_only)
+                          run_with_locale, cpython_only, check_impl_detail)
 import random
 import time
 import warnings
@@ -5452,6 +5452,7 @@
 
         POS = inspect._ParameterKind.POSITIONAL_ONLY
         POS_KWD = inspect._ParameterKind.POSITIONAL_OR_KEYWORD
+        KWONLY = inspect._ParameterKind.KEYWORD_ONLY
 
         # Type heuristic (type annotations would help!):
         pdict = {C: {'other': C.Decimal(1),
@@ -5489,6 +5490,8 @@
                     args.append(pdict[module][name])
                 elif param.kind == POS_KWD:
                     kwargs[name] = pdict[module][name]
+                elif param.kind == KWONLY:
+                    pass
                 else:
                     raise TestFailed("unexpected parameter kind")
             return args, kwargs
@@ -5517,15 +5520,26 @@
                     p_names = list(p_sig.parameters.keys())
                     c_names = [tr(x) for x in c_sig.parameters.keys()]
 
+                    p_kind = [x.kind for x in p_sig.parameters.values()]
+                    c_kind = [x.kind for x in c_sig.parameters.values()]
+
+                    if check_impl_detail(pypy=True):
+                        # PyPy only: _decimal.py has some methods with
+                        # an extra keyword-only argument 'strict', which
+                        # we ignore here
+                        if c_names[-1:] == ['strict'] and c_kind[-1] == KWONLY:
+                            del c_names[-1]
+                            del c_kind[-1]
+
                     self.assertEqual(c_names, p_names,
                                      msg="parameter name mismatch in %s" % p_func)
 
-                    p_kind = [x.kind for x in p_sig.parameters.values()]
-                    c_kind = [x.kind for x in c_sig.parameters.values()]
-
                     # 'self' parameter:
                     self.assertIs(p_kind[0], POS_KWD)
-                    self.assertIs(c_kind[0], POS)
+                    if check_impl_detail(cpython=True):
+                        self.assertIs(c_kind[0], POS)
+                    else:
+                        self.assertIs(c_kind[0], POS_KWD)
 
                     # remaining parameters:
                     if ty == 'Decimal':
diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -1264,7 +1264,7 @@
         return Decimal._from_float(value, self, exact=False)
 
     # operations
-    def _convert_unaryop(self, a, strict=True):
+    def _convert_unaryop(self, a, *, strict=True):
         if isinstance(a, Decimal):
             return a
         elif isinstance(a, int):
@@ -1274,7 +1274,7 @@
         else:
             return NotImplemented
 
-    def _convert_binop(self, a, b, strict=True):
+    def _convert_binop(self, a, b, *, strict=True):
         a = self._convert_unaryop(a, strict=strict)
         b = self._convert_unaryop(b, strict=strict)
         if b is NotImplemented:
@@ -1441,7 +1441,7 @@
             _mpdec.mpd_qfinalize(result._mpd, ctx, status_ptr)
         return result
 
-    def divmod(self, a, b, strict=True):
+    def divmod(self, a, b, *, strict=True):
         a, b = self._convert_binop(a, b, strict=strict)
         if a is NotImplemented:
             return NotImplemented
@@ -1452,7 +1452,7 @@
                                ctx, status_ptr)
         return q, r
 
-    def power(self, a, b, modulo=None, strict=True):
+    def power(self, a, b, modulo=None, *, strict=True):
         a, b = self._convert_binop(a, b, strict=strict)
         if a is NotImplemented:
             return NotImplemented

From pypy.commits at gmail.com  Sun Jan  8 11:58:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 08:58:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add _testcapi.raise_signal(). Hack to
 have it (and awaitType) correctly
Message-ID: <58726fa8.8838c20a.ed9d4.5a1f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89428:69efed9f9864
Date: 2017-01-08 17:57 +0100
http://bitbucket.org/pypy/pypy/changeset/69efed9f9864/

Log:	Add _testcapi.raise_signal(). Hack to have it (and awaitType)
	correctly present even if cpyext is enabled.

diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -25,3 +25,14 @@
         self._iterator = iterator
     def __await__(self):
         return self._iterator
+
+def raise_signal(signum):
+    import _signal, _thread
+    _signal.pthread_kill(_thread.get_ident(), signum)
+
+
+# the hacks above have replaced this module with another, so we need
+# to push the extra names into this other module too...
+import _testcapi
+_testcapi.awaitType = awaitType
+_testcapi.raise_signal = raise_signal

From pypy.commits at gmail.com  Sun Jan  8 11:58:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 08:58:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Complain in signal.set_wakeup_fd() if the
 fd is blocking
Message-ID: <58726fa6.e4361c0a.f7c9e.b21d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89427:3da675aed49d
Date: 2017-01-08 17:42 +0100
http://bitbucket.org/pypy/pypy/changeset/3da675aed49d/

Log:	Complain in signal.set_wakeup_fd() if the fd is blocking

diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -6,12 +6,12 @@
 import errno
 
 from pypy.interpreter.error import (
-    OperationError, exception_from_saved_errno, oefmt)
+    OperationError, exception_from_saved_errno, oefmt, wrap_oserror)
 from pypy.interpreter.executioncontext import (AsyncAction, AbstractActionFlag,
     PeriodicAsyncAction)
 from pypy.interpreter.gateway import unwrap_spec
 
-from rpython.rlib import jit, rgc
+from rpython.rlib import jit, rgc, rposix, rposix_stat
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import intmask, widen
 from rpython.rlib.rsignal import *
@@ -241,7 +241,7 @@
 
 
 @jit.dont_look_inside
- at unwrap_spec(fd=int)
+ at unwrap_spec(fd="c_int")
 def set_wakeup_fd(space, fd):
     """Sets the fd to be written to (with the signal number) when a signal
     comes in.  Returns the old fd.  A library can use this to
@@ -254,11 +254,19 @@
                     "set_wakeup_fd only works in main thread or with "
                     "__pypy__.thread.enable_signals()")
     if fd != -1:
+        if not rposix.is_valid_fd(fd):
+            raise oefmt(space.w_ValueError, "invalid fd")
         try:
             os.fstat(fd)
+            flags = rposix.get_status_flags(fd)
         except OSError as e:
             if e.errno == errno.EBADF:
                 raise oefmt(space.w_ValueError, "invalid fd")
+            raise wrap_oserror(space, e, eintr_retry=False)
+        if flags & rposix.O_NONBLOCK == 0:
+            raise oefmt(space.w_ValueError,
+                        "the fd %d must be in non-blocking mode", fd)
+
     old_fd = pypysig_set_wakeup_fd(fd, False)
     return space.wrap(intmask(old_fd))
 

From pypy.commits at gmail.com  Sun Jan  8 15:16:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 12:16:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: TypeError => OverflowError
Message-ID: <58729e0e.61c9c20a.d7016.5efd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89429:ddb87455245b
Date: 2017-01-08 21:13 +0100
http://bitbucket.org/pypy/pypy/changeset/ddb87455245b/

Log:	TypeError => OverflowError

diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -741,7 +741,7 @@
     def get_buffer_size(self, space):
         return space.wrap(self.buffer_size)
     def set_buffer_size(self, space, w_value):
-        value = space.getindex_w(w_value, space.w_TypeError)
+        value = space.getindex_w(w_value, space.w_OverflowError)
         if value <= 0:
             raise oefmt(space.w_ValueError,
                         "buffer_size must be greater than zero")
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -89,7 +89,7 @@
         p = pyexpat.ParserCreate()
         p.buffer_size = 150
         assert p.buffer_size == 150
-        raises((ValueError, TypeError),
+        raises(OverflowError,
                setattr, p, 'buffer_size', sys.maxsize + 1)
 
     def test_encoding_xml(self):

From pypy.commits at gmail.com  Sun Jan  8 15:22:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 12:22:06 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix a division by zero
Message-ID: <58729f6e.86cbc20a.960e5.49b0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89430:f2d9207e657b
Date: 2017-01-08 21:21 +0100
http://bitbucket.org/pypy/pypy/changeset/f2d9207e657b/

Log:	Fix a division by zero

diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -107,6 +107,10 @@
 class W_UnpackIter(W_Root):
     def __init__(self, space, w_struct, w_buffer):
         buf = space.buffer_w(w_buffer, space.BUF_SIMPLE)
+        if w_struct.size <= 0:
+            raise oefmt(get_error(space),
+                "cannot iteratively unpack with a struct of length %d",
+                w_struct.size)
         if buf.getlength() % w_struct.size != 0:
             raise oefmt(get_error(space),
                 "iterative unpacking requires a bytes length multiple of %d",
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -423,6 +423,12 @@
         raises(struct.error, s.iter_unpack, b'123')
         raises(struct.error, struct.iter_unpack, 'h', b'12345')
 
+    def test_iter_unpack_empty_struct(self):
+        struct = self.struct
+        s = struct.Struct('')
+        raises(struct.error, s.iter_unpack, b'')
+        raises(struct.error, s.iter_unpack, b'?')
+
     def test___float__(self):
         class MyFloat(object):
             def __init__(self, x):

From pypy.commits at gmail.com  Sun Jan  8 15:24:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 08 Jan 2017 12:24:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Skip tests that are really about
 subinterpreters
Message-ID: <5872a015.8b9a1c0a.b98fd.2bee@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89431:ce8c2c3d808f
Date: 2017-01-08 21:24 +0100
http://bitbucket.org/pypy/pypy/changeset/ce8c2c3d808f/

Log:	Skip tests that are really about subinterpreters

diff --git a/lib-python/3/test/test_threading.py b/lib-python/3/test/test_threading.py
--- a/lib-python/3/test/test_threading.py
+++ b/lib-python/3/test/test_threading.py
@@ -836,6 +836,7 @@
 
 class SubinterpThreadingTests(BaseTestCase):
 
+    @cpython_only
     def test_threads_join(self):
         # Non-daemon threads should be joined at subinterpreter shutdown
         # (issue #18808)
@@ -859,6 +860,7 @@
         # The thread was joined properly.
         self.assertEqual(os.read(r, 1), b"x")
 
+    @cpython_only
     def test_threads_join_2(self):
         # Same as above, but a delay gets introduced after the thread's
         # Python code returned but before the thread state is deleted.

From pypy.commits at gmail.com  Sun Jan  8 20:13:13 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 08 Jan 2017 17:13:13 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Move C code generation code
 to methods of ApiFunction
Message-ID: <5872e3a9.518a1c0a.b8be2.202a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89432:deeb13497468
Date: 2017-01-09 01:12 +0000
http://bitbucket.org/pypy/pypy/changeset/deeb13497468/

Log:	Move C code generation code to methods of ApiFunction

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -329,6 +329,47 @@
             wrapper.c_name = cpyext_namespace.uniquename(self.c_name)
         return wrapper
 
+    def get_c_restype(self, c_writer):
+        return c_writer.gettype(self.restype).replace('@', '').strip()
+
+    def get_c_args(self, c_writer):
+        args = []
+        for i, argtype in enumerate(self.argtypes):
+            if argtype is CONST_STRING:
+                arg = 'const char *@'
+            elif argtype is CONST_STRINGP:
+                arg = 'const char **@'
+            elif argtype is CONST_WSTRING:
+                arg = 'const wchar_t *@'
+            else:
+                arg = c_writer.gettype(argtype)
+            arg = arg.replace('@', 'arg%d' % (i,)).strip()
+            args.append(arg)
+        args = ', '.join(args) or "void"
+        return args
+
+    def get_api_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "PyAPI_FUNC({restype}) {name}({args});".format(**locals())
+
+    def get_ptr_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "{restype} (*{name})({args});".format(**locals())
+
+    def get_ctypes_impl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        callargs = ', '.join('arg%d' % (i,)
+                            for i in range(len(self.argtypes)))
+        if self.restype is lltype.Void:
+            body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+        else:
+            body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+        return '%s %s(%s)\n%s' % (restype, name, args, body)
+
+
 DEFAULT_HEADER = 'pypy_decl.h'
 def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
                 gil=None, result_borrowed=False, result_is_ll=False):
@@ -987,23 +1028,6 @@
     for func in BOOTSTRAP_FUNCTIONS:
         func(space)
 
-def c_function_signature(db, func):
-    restype = db.gettype(func.restype).replace('@', '').strip()
-    args = []
-    for i, argtype in enumerate(func.argtypes):
-        if argtype is CONST_STRING:
-            arg = 'const char *@'
-        elif argtype is CONST_STRINGP:
-            arg = 'const char **@'
-        elif argtype is CONST_WSTRING:
-            arg = 'const wchar_t *@'
-        else:
-            arg = db.gettype(argtype)
-        arg = arg.replace('@', 'arg%d' % (i,)).strip()
-        args.append(arg)
-    args = ', '.join(args) or "void"
-    return restype, args
-
 #_____________________________________________________
 # Build the bridge DLL, Allow extension DLLs to call
 # back into Pypy space functions
@@ -1023,15 +1047,8 @@
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            restype, args = c_function_signature(db, func)
-            callargs = ', '.join('arg%d' % (i,)
-                                for i in range(len(func.argtypes)))
-            if func.restype is lltype.Void:
-                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
-            else:
-                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
-            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
-            members.append('%s (*%s)(%s);' % (restype, name, args))
+            functions.append(func.get_ctypes_impl(name, db))
+            members.append(func.get_ptr_decl(name, db))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
     struct_declaration_code = """\
@@ -1226,8 +1243,7 @@
         for name, func in sorted(header_functions.iteritems()):
             _name = mangle_name(prefix, name)
             header.append("#define %s %s" % (name, _name))
-            restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
+            header.append(func.get_api_decl(name, db))
 
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,10 +89,10 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
-                == ('Py_ssize_t', 'Py_ssize_t arg0'))
-        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
-                == ('Py_ssize_t *', 'Py_ssize_t *arg0'))
+        assert PyPy_TypedefTest1.api_func.get_c_restype(db) == 'Py_ssize_t'
+        assert PyPy_TypedefTest1.api_func.get_c_args(db) == 'Py_ssize_t arg0'
+        assert PyPy_TypedefTest2.api_func.get_c_restype(db) == 'Py_ssize_t *'
+        assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Py_ssize_t *arg0'
 
         PyPy_TypedefTest1(space, 0)
         ppos = lltype.malloc(api.Py_ssize_tP.TO, 1, flavor='raw')

From pypy.commits at gmail.com  Mon Jan  9 05:04:53 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 02:04:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merged default
Message-ID: <58736045.2669c20a.2af85.c573@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89433:8f58de1af422
Date: 2017-01-09 11:01 +0100
http://bitbucket.org/pypy/pypy/changeset/8f58de1af422/

Log:	merged default

diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -32,7 +32,7 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises((TypeError, ValueError),
+        self.assertRaises(TypeError,
                           (c_char * 16).from_buffer, "a" * 16)
 
     def test_fom_buffer_with_offset(self):
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -428,11 +428,6 @@
   ``datetime.date`` is the superclass of ``datetime.datetime``).
   Anyway, the proper fix is arguably to use a regular method call in
   the first place: ``datetime.date.today().strftime(...)``
-
-* the ``__dict__`` attribute of new-style classes returns a normal dict, as
-  opposed to a dict proxy like in CPython. Mutating the dict will change the
-  type and vice versa. For builtin types, a dictionary will be returned that
-  cannot be changed (but still looks and behaves like a normal dictionary).
   
 * some functions and attributes of the ``gc`` module behave in a
   slightly different way: for example, ``gc.enable`` and
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -90,3 +90,18 @@
 
 Support translations of cpyext with the Boehm GC (for special cases like
 revdb).
+
+.. branch: strbuf-as-buffer
+
+Implement StringBuffer.get_raw_address (missing feature for the buffer protocol).
+More generally it is now possible to obtain the address of any object (if it
+is readonly) without pinning it.
+
+.. branch: cpyext-cleanup
+
+Refactor cpyext initialisation.
+
+.. branch: cpyext-from2
+
+Fix a test failure introduced by strbuf-as-buffer
+
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -662,9 +662,6 @@
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
-        if self.config.objspace.usemodules.cpyext:
-            from pypy.module.cpyext.state import State
-            self.fromcache(State).build_api(self)
         self.getbuiltinmodule('sys')
         self.getbuiltinmodule('_imp')
         self.getbuiltinmodule('_frozen_importlib')
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -4,6 +4,7 @@
 
 from rpython.rlib.buffer import Buffer
 from pypy.interpreter.gateway import unwrap_spec
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
 
 
 class ByteBuffer(Buffer):
@@ -22,6 +23,8 @@
     def setitem(self, index, char):
         self.data[index] = char
 
+    def get_raw_address(self):
+        return nonmoving_raw_ptr_for_resizable_list(self.data)
 
 @unwrap_spec(length=int)
 def bytebuffer(space, length):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -136,6 +136,9 @@
     return _from_buffer(space, w_ctype, w_x)
 
 def _from_buffer(space, w_ctype, w_x):
+    if space.isinstance_w(w_x, space.w_unicode):
+        raise oefmt(space.w_TypeError,
+                        "from_buffer() cannot return the address a unicode")
     buf = _fetch_as_read_buffer(space, w_x)
     if space.isinstance_w(w_x, space.w_str):
         _cdata = get_raw_address_of_string(space, w_x)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3427,16 +3427,26 @@
     except ImportError:
         pass
     else:
-        # from_buffer(buffer(b"foo")) does not work, because it's not
-        # implemented on pypy; only from_buffer(b"foo") works.
-        py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
-        py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
+        # Python 2 only
+        contents = from_buffer(BCharA, buffer(b"foo"))
+        assert len(contents) == len(p1)
+        for i in range(len(contents)):
+            assert contents[i] == p1[i]
+        p4 = buffer(u+"foo")
+        contents = from_buffer(BCharA, buffer(u+"foo"))
+        assert len(contents) == len(p4)
+        for i in range(len(contents)):
+            assert contents[i] == p4[i]
     try:
         from __builtin__ import memoryview
     except ImportError:
         pass
     else:
-        py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo"))
+        contents = from_buffer(BCharA, memoryview(b"foo"))
+        assert len(contents) == len(p1)
+        for i in range(len(contents)):
+            assert contents[i] == p1[i]
+
 
 def test_from_buffer_bytearray():
     a = bytearray(b"xyz")
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -494,7 +494,7 @@
         while True:
             try:
                 addr = self.addr_from_object(space, w_addr)
-                count = self.sock.sendto(data, flags, addr)
+                count = self.sock.sendto(data, len(data), flags, addr)
                 break
             except SocketError as e:
                 converted_error(space, e, eintr_retry=True)
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -1,5 +1,4 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.interpreter import gateway
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
 
@@ -13,12 +12,17 @@
 
     atexit_funcs = []
 
+    def setup_after_space_initialization(self):
+        state = self.space.fromcache(State)
+        state.setup_rawrefcount()
+        state.build_api()
+
     def startup(self, space):
         space.fromcache(State).startup(space)
         method = pypy.module.cpyext.typeobject.get_new_method_def(space)
         w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap(''))
         space.appexec([space.type(w_obj)], """(methodtype):
-            from pickle import Pickler 
+            from pickle import Pickler
             Pickler.dispatch[methodtype] = Pickler.save_global
         """)
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1,5 +1,6 @@
 import ctypes
 import sys, os
+from collections import defaultdict
 
 import py
 
@@ -70,7 +71,6 @@
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
-VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list')
 CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
                                        hints={'nolength': True}),
                           use_cache=False)
@@ -367,8 +367,8 @@
         func_name = func.func_name
         if header is not None:
             c_name = None
-            assert func_name not in FUNCTIONS, (
-                "%s already registered" % func_name)
+            if func_name in FUNCTIONS_BY_HEADER[header]:
+                raise ValueError("%s already registered" % func_name)
         else:
             c_name = func_name
         api_function = ApiFunction(argtypes, restype, func, error,
@@ -466,9 +466,7 @@
             return res
 
         if header is not None:
-            if header == DEFAULT_HEADER:
-                FUNCTIONS[func_name] = api_function
-            FUNCTIONS_BY_HEADER.setdefault(header, {})[func_name] = api_function
+            FUNCTIONS_BY_HEADER[header][func_name] = api_function
         INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
         return unwrapper  # used in 'normal' RPython code.
     return decorate
@@ -492,8 +490,7 @@
     GLOBALS[name] = (typ, expr)
 
 INTERPLEVEL_API = {}
-FUNCTIONS = {}
-FUNCTIONS_BY_HEADER = {}
+FUNCTIONS_BY_HEADER = defaultdict(dict)
 
 # These are C symbols which cpyext will export, but which are defined in .c
 # files somewhere in the implementation of cpyext (rather than being defined in
@@ -528,6 +525,8 @@
     'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor',
     'PyCapsule_SetContext', 'PyCapsule_Import', 'PyCapsule_Type', '_Py_get_capsule_type',
 
+    'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
+
     'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
 
     'PyOS_getsig', 'PyOS_setsig',
@@ -574,7 +573,9 @@
     # PyExc_AttributeError, PyExc_OverflowError, PyExc_ImportError,
     # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError,
     # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ...
-    for exc_name in exceptions.Module.interpleveldefs.keys():
+    global all_exceptions
+    all_exceptions = list(exceptions.Module.interpleveldefs)
+    for exc_name in all_exceptions:
         if exc_name in ('EnvironmentError', 'IOError'):
             # FIXME: aliases of OSError cause a clash of names via
             # export_struct
@@ -619,12 +620,6 @@
                              % (cpyname, ))
 build_exported_objects()
 
-def get_structtype_for_ctype(ctype):
-    from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
-    from pypy.module.cpyext.cdatetime import PyDateTime_CAPI
-    return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr,
-            "PyDateTime_CAPI*": lltype.Ptr(PyDateTime_CAPI)}[ctype]
-
 # Note: as a special case, "PyObject" is the pointer type in RPython,
 # corresponding to "PyObject *" in C.  We do that only for PyObject.
 # For example, "PyTypeObject" is the struct type even in RPython.
@@ -673,12 +668,6 @@
 # a pointer to PyObject
 PyObjectP = rffi.CArrayPtr(PyObject)
 
-VA_TP_LIST = {}
-#{'int': lltype.Signed,
-#              'PyObject*': PyObject,
-#              'PyObject**': PyObjectP,
-#              'int*': rffi.INTP}
-
 def configure_types():
     for config in (CConfig, CConfig2):
         for name, TYPE in rffi_platform.configure(config).iteritems():
@@ -952,21 +941,8 @@
     wrapper_second_level._dont_inline_ = True
     return wrapper_second_level
 
-def process_va_name(name):
-    return name.replace('*', '_star')
 
-def setup_va_functions(eci):
-    for name, TP in VA_TP_LIST.iteritems():
-        name_no_star = process_va_name(name)
-        func = rffi.llexternal('pypy_va_get_%s' % name_no_star, [VA_LIST_P],
-                               TP, compilation_info=eci)
-        globals()['va_get_%s' % name_no_star] = func
-
-def setup_init_functions(eci, translating):
-    if translating:
-        prefix = 'PyPy'
-    else:
-        prefix = 'cpyexttest'
+def setup_init_functions(eci, prefix):
     # jump through hoops to avoid releasing the GIL during initialization
     # of the cpyext module.  The C functions are called with no wrapper,
     # but must not do anything like calling back PyType_Ready().  We
@@ -1028,23 +1004,27 @@
 # Do not call this more than once per process
 def build_bridge(space):
     "NOT_RPYTHON"
-    from pypy.module.cpyext.pyobject import make_ref
     from rpython.translator.c.database import LowLevelDatabase
     use_micronumpy = setup_micronumpy(space)
     db = LowLevelDatabase()
-    prefix ='cpyexttest'
+    prefix = 'cpyexttest'
 
-    functions = generate_decls_and_callbacks(db, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
 
     # Structure declaration code
+    functions = []
     members = []
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                # added only for the macro, not the decl
-                continue
             restype, args = c_function_signature(db, func)
+            callargs = ', '.join('arg%d' % (i,)
+                                for i in range(len(func.argtypes)))
+            if func.restype is lltype.Void:
+                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+            else:
+                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
             members.append('%s (*%s)(%s);' % (restype, name, args))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
@@ -1056,15 +1036,8 @@
     """ % dict(members=structmembers)
 
     global_objects = []
-    for name, (typ, expr) in GLOBALS.iteritems():
-        if '#' in name:
-            continue
-        if typ == 'PyDateTime_CAPI*':
-            continue
-        elif name.startswith('PyExc_'):
-            global_objects.append('%s _%s;' % (typ[:-1], name))
-        else:
-            global_objects.append('%s %s = NULL;' % (typ, name))
+    for name in all_exceptions:
+        global_objects.append('PyTypeObject _PyExc_%s;' % name)
     global_code = '\n'.join(global_objects)
 
     prologue = ("#include <Python.h>\n"
@@ -1081,90 +1054,69 @@
             '\n' +
             '\n'.join(functions))
 
-    eci = build_eci(True, code, use_micronumpy)
+    eci = build_eci(code, use_micronumpy, translating=False)
     eci = eci.compile_shared_lib(
         outputfilename=str(udir / "module_cache" / "pypyapi"))
+    space.fromcache(State).install_dll(eci)
     modulename = py.path.local(eci.libraries[-1])
 
-    def dealloc_trigger():
-        from pypy.module.cpyext.pyobject import decref
-        print 'dealloc_trigger...'
-        while True:
-            ob = rawrefcount.next_dead(PyObject)
-            if not ob:
-                break
-            print 'deallocating PyObject', ob
-            decref(space, ob)
-        print 'dealloc_trigger DONE'
-        return "RETRY"
-    rawrefcount.init(dealloc_trigger)
-
     run_bootstrap_functions(space)
 
     # load the bridge, and init structure
     bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
 
-    space.fromcache(State).install_dll(eci)
+    # populate static data
+    builder = space.fromcache(State).builder = TestingObjBuilder()
+    for name, (typ, expr) in GLOBALS.iteritems():
+        if '#' in name:
+            name, header = name.split('#')
+            assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
+            isptr = False
+        elif name.startswith('PyExc_'):
+            isptr = False
+        elif typ == 'PyDateTime_CAPI*':
+            isptr = True
+        else:
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
 
-    # populate static data
-    builder = space.fromcache(StaticObjectBuilder)
-    for name, (typ, expr) in GLOBALS.iteritems():
         from pypy.module import cpyext    # for the eval() below
         w_obj = eval(expr)
-        if '#' in name:
-            name = name.split('#')[0]
-            isptr = False
-        else:
-            isptr = True
-        if name.startswith('PyExc_'):
-            isptr = False
-
         INTERPLEVEL_API[name] = w_obj
 
-        name = name.replace('Py', prefix)
+        mname = mangle_name(prefix, name)
         if isptr:
-            ptr = ctypes.c_void_p.in_dll(bridge, name)
-            if typ == 'PyObject*':
-                value = make_ref(space, w_obj)
-            elif typ == 'PyDateTime_CAPI*':
-                value = w_obj
-            else:
-                assert False, "Unknown static pointer: %s %s" % (typ, name)
+            assert typ == 'PyDateTime_CAPI*'
+            value = w_obj
+            ptr = ctypes.c_void_p.in_dll(bridge, mname)
             ptr.value = ctypes.cast(ll2ctypes.lltype2ctypes(value),
                                     ctypes.c_void_p).value
         elif typ in ('PyObject*', 'PyTypeObject*'):
             if name.startswith('PyPyExc_') or name.startswith('cpyexttestExc_'):
                 # we already have the pointer
-                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, in_dll)
             else:
                 # we have a structure, get its address
-                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, name)
+                in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, ctypes.pointer(in_dll))
             builder.prepare(py_obj, w_obj)
-        else:
-            assert False, "Unknown static object: %s %s" % (typ, name)
-    builder.attach_all()
+    builder.attach_all(space)
 
     pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
 
     # implement structure initialization code
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if name.startswith('cpyext_') or func is None: # XXX hack
-                continue
             pypyAPI[structindex[name]] = ctypes.cast(
                 ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
                 ctypes.c_void_p)
-    setup_va_functions(eci)
 
-    setup_init_functions(eci, translating=False)
+    setup_init_functions(eci, prefix)
     return modulename.new(ext='')
 
 
-class StaticObjectBuilder:
-    def __init__(self, space):
-        self.space = space
+class StaticObjectBuilder(object):
+    def __init__(self):
         self.static_pyobjs = []
         self.static_objs_w = []
         self.cpyext_type_init = None
@@ -1180,13 +1132,12 @@
         self.static_pyobjs.append(py_obj)
         self.static_objs_w.append(w_obj)
 
-    def attach_all(self):
+    def attach_all(self, space):
         # this is RPython, called once in pypy-c when it imports cpyext
         from pypy.module.cpyext.pyobject import get_typedescr, make_ref
         from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
         from pypy.module.cpyext.pyobject import track_reference
         #
-        space = self.space
         static_pyobjs = self.get_static_pyobjs()
         static_objs_w = self.static_objs_w
         for i in range(len(static_objs_w)):
@@ -1207,6 +1158,12 @@
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
+class TestingObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used in tests."""
+
+class TranslationObjBuilder(StaticObjectBuilder):
+    """The StaticObjectBuilder used during translation."""
+
 
 def mangle_name(prefix, name):
     if name.startswith('Py'):
@@ -1214,23 +1171,26 @@
     elif name.startswith('_Py'):
         return '_' + prefix + name[3:]
     else:
-        return None
+        raise ValueError("Error converting '%s'" % name)
 
-def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
+def write_header(header_name, decls):
+    lines = [
+        '#define Signed   long           /* xxx temporary fix */',
+        '#define Unsigned unsigned long  /* xxx temporary fix */',
+        '',] + decls + [
+        '',
+        '#undef Signed    /* xxx temporary fix */',
+        '#undef Unsigned  /* xxx temporary fix */',
+        '']
+    decl_h = udir.join(header_name)
+    decl_h.write('\n'.join(lines))
+
+def generate_decls_and_callbacks(db, prefix=''):
     "NOT_RPYTHON"
     pypy_macros = []
-    export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
-    for name in export_symbols:
-        if '#' in name:
-            name, header = name.split('#')
-        else:
-            header = pypy_decl
+    for name in SYMBOLS_C:
         newname = mangle_name(prefix, name)
-        assert newname, name
-        if header == pypy_decl:
-            pypy_macros.append('#define %s %s' % (name, newname))
-        if name.startswith("PyExc_"):
-            pypy_macros.append('#define _%s _%s' % (name, newname))
+        pypy_macros.append('#define %s %s' % (name, newname))
 
     # Generate defines
     for macro_name, size in [
@@ -1250,50 +1210,18 @@
     pypy_macros_h = udir.join('pypy_macros.h')
     pypy_macros_h.write('\n'.join(pypy_macros))
 
-    # implement function callbacks and generate function decls
-    functions = []
-    decls = {}
-    pypy_decls = decls[pypy_decl] = []
-    pypy_decls.append('#define Signed   long           /* xxx temporary fix */\n')
-    pypy_decls.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-
+    # generate function decls
+    decls = defaultdict(list)
     for decl in FORWARD_DECLS:
-        pypy_decls.append("%s;" % (decl,))
+        decls[pypy_decl].append("%s;" % (decl,))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
-        if header_name not in decls:
-            header = decls[header_name] = []
-            header.append('#define Signed   long           /* xxx temporary fix */\n')
-            header.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
-        else:
-            header = decls[header_name]
-
+        header = decls[header_name]
         for name, func in sorted(header_functions.iteritems()):
-            if not func:
-                continue
-            if header == DEFAULT_HEADER:
-                _name = name
-            else:
-                # this name is not included in pypy_macros.h
-                _name = mangle_name(prefix, name)
-                assert _name is not None, 'error converting %s' % name
-                header.append("#define %s %s" % (name, _name))
+            _name = mangle_name(prefix, name)
+            header.append("#define %s %s" % (name, _name))
             restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, _name, args))
-            if api_struct:
-                callargs = ', '.join('arg%d' % (i,)
-                                    for i in range(len(func.argtypes)))
-                if func.restype is lltype.Void:
-                    body = "{ _pypyAPI.%s(%s); }" % (_name, callargs)
-                else:
-                    body = "{ return _pypyAPI.%s(%s); }" % (_name, callargs)
-                functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
-    for name in VA_TP_LIST:
-        name_no_star = process_va_name(name)
-        header = ('%s pypy_va_get_%s(va_list* vp)' %
-                  (name, name_no_star))
-        pypy_decls.append('RPY_EXTERN ' + header + ';')
-        functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
+            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
 
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
@@ -1302,19 +1230,11 @@
         elif name.startswith('PyExc_'):
             typ = 'PyObject*'
             header = pypy_decl
-        if header != pypy_decl:
-            decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
+        decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
         decls[header].append('PyAPI_DATA(%s) %s;' % (typ, name))
 
-    for header_name in FUNCTIONS_BY_HEADER.keys():
-        header = decls[header_name]
-        header.append('#undef Signed    /* xxx temporary fix */\n')
-        header.append('#undef Unsigned  /* xxx temporary fix */\n')
-
     for header_name, header_decls in decls.iteritems():
-        decl_h = udir.join(header_name)
-        decl_h.write('\n'.join(header_decls))
-    return functions
+        write_header(header_name, header_decls)
 
 separate_module_files = [source_dir / "varargwrapper.c",
                          source_dir / "pyerrors.c",
@@ -1335,14 +1255,16 @@
                          source_dir / "bytesobject.c",
                          ]
 
-def build_eci(building_bridge, code, use_micronumpy=False):
+def build_eci(code, use_micronumpy=False, translating=False):
     "NOT_RPYTHON"
     # Build code and get pointer to the structure
     kwds = {}
 
     compile_extra=['-DPy_BUILD_CORE']
 
-    if building_bridge:
+    if translating:
+        kwds["includes"] = ['Python.h'] # this is our Python.h
+    else:
         if sys.platform == "win32":
             # '%s' undefined; assuming extern returning int
             compile_extra.append("/we4013")
@@ -1352,8 +1274,6 @@
         elif sys.platform.startswith('linux'):
             compile_extra.append("-Werror=implicit-function-declaration")
             compile_extra.append('-g')
-    else:
-        kwds["includes"] = ['Python.h'] # this is our Python.h
 
     # Generate definitions for global structures
     structs = ["#include <Python.h>"]
@@ -1407,9 +1327,7 @@
         return use_micronumpy
     # import registers api functions by side-effect, we also need HEADER
     from pypy.module.cpyext.ndarrayobject import HEADER
-    global FUNCTIONS_BY_HEADER, separate_module_files
-    for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
-        FUNCTIONS_BY_HEADER.setdefault(HEADER, {})[func_name] = None
+    global separate_module_files
     register_global("PyArray_Type",
         'PyTypeObject*',  "space.gettypeobject(W_NDimArray.typedef)",
         header=HEADER)
@@ -1423,21 +1341,19 @@
     db = LowLevelDatabase()
     prefix = 'PyPy'
 
-    functions = generate_decls_and_callbacks(db, api_struct=False, prefix=prefix)
+    generate_decls_and_callbacks(db, prefix=prefix)
+
     code = "#include <Python.h>\n"
     if use_micronumpy:
         code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
-    code  += "\n".join(functions)
 
-    eci = build_eci(False, code, use_micronumpy)
-
+    eci = build_eci(code, use_micronumpy, translating=True)
     space.fromcache(State).install_dll(eci)
 
     run_bootstrap_functions(space)
-    setup_va_functions(eci)
 
     # emit uninitialized static data
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder = TranslationObjBuilder()
     lines = ['PyObject *pypy_static_pyobjs[] = {\n']
     include_lines = ['RPY_EXTERN PyObject *pypy_static_pyobjs[];\n']
     for name, (typ, expr) in sorted(GLOBALS.items()):
@@ -1445,17 +1361,15 @@
             name, header = name.split('#')
             assert typ in ('PyObject*', 'PyTypeObject*')
             typ = typ[:-1]
-            if header != pypy_decl:
-                # since the #define is not in pypy_macros, do it here
-                mname = mangle_name(prefix, name)
-                include_lines.append('#define %s %s\n' % (name, mname))
+            mname = mangle_name(prefix, name)
+            include_lines.append('#define %s %s\n' % (name, mname))
         elif name.startswith('PyExc_'):
             typ = 'PyTypeObject'
             name = '_' + name
         elif typ == 'PyDateTime_CAPI*':
             continue
         else:
-            assert False, "Unknown static data: %s %s" % (typ, name)
+            raise ValueError("Unknown static data: %s %s" % (typ, name))
 
         from pypy.module import cpyext     # for the eval() below
         w_obj = eval(expr)
@@ -1475,20 +1389,15 @@
 
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            if not func:
-                continue
-            newname = mangle_name('PyPy', name) or name
+            newname = mangle_name(prefix, name)
             deco = entrypoint_lowlevel("cpyext", func.argtypes, newname,
                                         relax=True)
             deco(func.get_wrapper(space))
 
-    setup_init_functions(eci, translating=True)
+    setup_init_functions(eci, prefix)
     trunk_include = pypydir.dirpath() / 'include'
     copy_header_files(trunk_include, use_micronumpy)
 
-def init_static_data_translated(space):
-    builder = space.fromcache(StaticObjectBuilder)
-    builder.attach_all()
 
 def _load_from_cffi(space, name, path, initptr):
     from pypy.module._cffi_backend import cffi1_module
diff --git a/pypy/module/cpyext/include/complexobject.h b/pypy/module/cpyext/include/complexobject.h
--- a/pypy/module/cpyext/include/complexobject.h
+++ b/pypy/module/cpyext/include/complexobject.h
@@ -16,23 +16,8 @@
     Py_complex cval;
 } PyComplexObject;
 
-/* generated function */
-PyAPI_FUNC(int) _PyComplex_AsCComplex(PyObject *, Py_complex *);
-PyAPI_FUNC(PyObject *) _PyComplex_FromCComplex(Py_complex *);
-
-Py_LOCAL_INLINE(Py_complex) PyComplex_AsCComplex(PyObject *obj)
-{
-    Py_complex result;
-    _PyComplex_AsCComplex(obj, &result);
-    return result;
-}
-
-// shmuller 2013/07/30: Make a function, since macro will fail in C++ due to 
-//                      const correctness if called with "const Py_complex"
-//#define PyComplex_FromCComplex(c) _PyComplex_FromCComplex(&c)
-Py_LOCAL_INLINE(PyObject *) PyComplex_FromCComplex(Py_complex c) {
-    return _PyComplex_FromCComplex(&c);
-}
+PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *obj);
+PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex c);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -37,8 +37,24 @@
     """
     Fills a newly allocated PyMemoryViewObject with the given W_MemoryView object.
     """
+    assert isinstance(w_obj, W_MemoryView)
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
-    py_obj.c_view.c_obj = rffi.cast(PyObject, 0)
+    view = py_obj.c_view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return
+    fill_Py_buffer(space, w_obj.buf, view)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_userdata)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s),
+                                             track_allocation=False))
+        rffi.setintfield(view, 'c_readonly', 1)
 
 def memory_realize(space, py_obj):
     """
@@ -79,29 +95,13 @@
     return generic_cpy_call(space, pb.c_bf_getbuffer, exporter, view, flags)
 
 @cpython_api([PyObject], Py_bufferP, error=CANNOT_FAIL)
-def PyMemoryView_GET_BUFFER(space, w_obj):
+def PyMemoryView_GET_BUFFER(space, pyobj):
     """Return a pointer to the buffer-info structure wrapped by the given
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
-    if not isinstance(w_obj, W_MemoryView):
-        return lltype.nullptr(Py_buffer)
-    py_memobj = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_obj)) # no inc_ref
-    view = py_memobj.c_view
-    ndim = w_obj.buf.getndim()
-    if ndim >= Py_MAX_NDIMS:
-        # XXX warn?
-        return view
-    fill_Py_buffer(space, w_obj.buf, view)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-        #view.c_obj = make_ref(space, w_obj) # NO - this creates a ref cycle!
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
-    except ValueError:
-        w_s = w_obj.descr_tobytes(space)
-        view.c_obj = make_ref(space, w_s)
-        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
-        rffi.setintfield(view, 'c_readonly', 1)
-    return view
+    # XXX move to a c-macro
+    py_memobj = rffi.cast(PyMemoryViewObject, pyobj)
+    return py_memobj.c_view
 
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
@@ -193,9 +193,11 @@
         return (_IsCContiguous(view) or _IsFortranContiguous(view))
     return 0
 
- at cpython_api([PyObject], PyObject)
+ at cpython_api([PyObject], PyObject, result_is_ll=True)
 def PyMemoryView_FromObject(space, w_obj):
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    w_memview = space.call_method(space.builtin, "memoryview", w_obj)
+    py_memview = make_ref(space, w_memview, w_obj)
+    return py_memview
 
 @cpython_api([Py_bufferP], PyObject)
 def PyMemoryView_FromBuffer(space, view):
@@ -203,6 +205,7 @@
     The memoryview object then owns the buffer represented by view, which
     means you shouldn't try to call PyBuffer_Release() yourself: it
     will be done on deallocation of the memoryview object."""
+	assert view.c_obj
     w_obj = from_ref(space, view.c_obj)
     if isinstance(w_obj, W_MemoryView):
         return w_obj
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t,
-    Py_buffer, mangle_name, pypy_decl)
+    pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
@@ -342,7 +342,7 @@
         else:
             #do not call twice
             return
-        if self.releasebufferproc:        
+        if self.releasebufferproc:
             func_target = rffi.cast(releasebufferproc, self.releasebufferproc)
             with lltype.scoped_alloc(Py_buffer) as pybuf:
                 pybuf.c_buf = self.ptr
@@ -407,7 +407,7 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self, 
+        buf = CPyBuffer(space, ptr[0], size, w_self,
                                releasebuffer=releasebuffer)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
@@ -516,7 +516,7 @@
     w_type = space.gettypeobject(typedef)
 
     header = pypy_decl
-    if mangle_name('', typedef.name) is None:
+    if not (name.startswith('Py') or name.startswith('_Py')):
         header = None
     handled = False
     # unary functions
diff --git a/pypy/module/cpyext/src/complexobject.c b/pypy/module/cpyext/src/complexobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/complexobject.c
@@ -0,0 +1,16 @@
+
+#include "Python.h"
+
+Py_complex
+PyComplex_AsCComplex(PyObject *obj)
+{
+    Py_complex result;
+    _PyComplex_AsCComplex(obj, &result);
+    return result;
+}
+
+PyObject *
+PyComplex_FromCComplex(Py_complex c)
+{
+    return _PyComplex_FromCComplex(&c);
+}
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -2,7 +2,6 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import executioncontext
-from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
@@ -14,9 +13,7 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CWCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
-        if space.config.translation.gc != "boehm":
-            pyobj_dealloc_action = PyObjDeallocAction(space)
-            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+        self.builder = None
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -57,22 +54,40 @@
                         "Function returned an error result without setting an "
                         "exception")
 
-    def build_api(self, space):
+    def setup_rawrefcount(self):
+        space = self.space
+        if not self.space.config.translating:
+            def dealloc_trigger():
+                from pypy.module.cpyext.pyobject import PyObject, decref
+                print 'dealloc_trigger...'
+                while True:
+                    ob = rawrefcount.next_dead(PyObject)
+                    if not ob:
+                        break
+                    print 'deallocating PyObject', ob
+                    decref(space, ob)
+                print 'dealloc_trigger DONE'
+                return "RETRY"
+            rawrefcount.init(dealloc_trigger)
+        else:
+            if space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(space)
+                space.actionflag.register_periodic_action(action,
+                    use_bytecode_counter=True)
+            else:
+                pyobj_dealloc_action = PyObjDeallocAction(space)
+                self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+
+    def build_api(self):
         """NOT_RPYTHON
         This function is called when at object space creation,
         and drives the compilation of the cpyext library
         """
         from pypy.module.cpyext import api
-        state = self.space.fromcache(State)
         if not self.space.config.translating:
-            state.api_lib = str(api.build_bridge(self.space))
+            self.api_lib = str(api.build_bridge(self.space))
         else:
             api.setup_library(self.space)
-            #
-            if self.space.config.translation.gc == "boehm":
-                action = BoehmPyObjDeallocAction(self.space)
-                self.space.actionflag.register_periodic_action(action,
-                    use_bytecode_counter=True)
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -84,17 +99,17 @@
 
     def startup(self, space):
         "This function is called when the program really starts"
-
         from pypy.module.cpyext.typeobject import setup_new_method_def
         from pypy.module.cpyext.api import INIT_FUNCTIONS
-        from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
             if space.config.translation.gc != "boehm":
+                # This must be called in RPython, the untranslated version
+                # does something different. Sigh.
                 rawrefcount.init(
                     llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
                     self.dealloc_trigger))
-            init_static_data_translated(space)
+            self.builder.attach_all(space)
 
         setup_new_method_def(space)
 
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,9 +89,9 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest1'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
                 == ('Py_ssize_t', 'Py_ssize_t arg0'))
-        assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest2'])
+        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
                 == ('Py_ssize_t *', 'Py_ssize_t *arg0'))
 
         PyPy_TypedefTest1(space, 0)
@@ -100,7 +100,7 @@
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
 
- at pytest.mark.skipif(os.environ.get('USER')=='root', 
+ at pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
     api.copy_header_files(tmpdir, True)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -27,8 +27,9 @@
 
 class TestApi:
     def test_signature(self):
-        assert 'PyModule_Check' in api.FUNCTIONS
-        assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
+        common_functions = api.FUNCTIONS_BY_HEADER[api.pypy_decl]
+        assert 'PyModule_Check' in common_functions
+        assert common_functions['PyModule_Check'].argtypes == [api.PyObject]
 
 
 class SpaceCompiler(SystemCompilationInfo):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -4,15 +4,15 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
+from pypy.module.cpyext.pyobject import from_ref
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
-        # implemented as a C macro
-        #assert api.PyObject_CheckBuffer(w_hello)
-        w_view = api.PyMemoryView_FromObject(w_hello)
+        assert api.PyObject_CheckBuffer(w_hello)
+        w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
         assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
@@ -20,7 +20,7 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_memoryview = from_ref(space, api.PyMemoryView_FromObject(w_buf))
         view = api.PyMemoryView_GET_BUFFER(w_memoryview)
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)
@@ -69,7 +69,9 @@
         assert y.format == 'i'
         assert y.shape == (10,)
         assert len(y) == 10
-        assert y[3] == 3
+        s = y[3]
+        assert len(s) == struct.calcsize('i')
+        assert s == struct.pack('i', 3)
 
     def test_buffer_protocol_capi(self):
         foo = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -86,7 +86,7 @@
     py_getsetdef.c_set = rffi.cast(setter, 0)
     py_getsetdef.c_closure = rffi.cast(rffi.VOIDP, 0)
     return py_getsetdef
-    
+
 
 class W_MemberDescr(GetSetProperty):
     name = 'member_descriptor'
@@ -619,7 +619,7 @@
 
     pto.c_tp_free = llslot(space, PyObject_Free)
     pto.c_tp_alloc = llslot(space, PyType_GenericAlloc)
-    builder = space.fromcache(StaticObjectBuilder)
+    builder = space.fromcache(State).builder
     if ((pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE) != 0
             and builder.cpyext_type_init is None):
             # this ^^^ is not None only during startup of cpyext.  At that
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -541,6 +541,10 @@
 
         sub = self._op_val(space, w_old)
         by = self._op_val(space, w_new)
+        # the following two lines are for being bug-to-bug compatible
+        # with CPython: see issue #2448
+        if count >= 0 and len(input) == 0:
+            return self._empty()
         try:
             res = replace(input, sub, by, count)
         except OverflowError:
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -874,6 +874,16 @@
         s = b"a" * (2**16)
         raises(OverflowError, s.replace, b"", s)
 
+    def test_replace_issue2448(self):
+        # CPython's replace() method has a bug that makes
+        #   ''.replace('', 'x')  gives a different answer than
+        #   ''.replace('', 'x', 1000).  This is the case in all
+        # known versions, at least until 2.7.13.  Some people
+        # call that a feature on the CPython issue report and
+        # the discussion dies out, so it might never be fixed.
+        assert ''.replace('', 'x') == 'x'
+        assert ''.replace('', 'x', 1000) == ''
+
     def test_getslice(self):
         s = b"abc"
         assert s[:] == b"abc"
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -185,9 +185,10 @@
         assert m[2] == 1
 
     def test_pypy_raw_address_base(self):
-        raises(ValueError, memoryview(b"foobar")._pypy_raw_address)
-        a = memoryview(bytearray(b"foobar"))._pypy_raw_address()
+        a = memoryview("foobar")._pypy_raw_address()
         assert a != 0
+        b = memoryview(bytearray("foobar"))._pypy_raw_address()
+        assert b != 0
 
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -106,8 +106,9 @@
         else:
             name = 'unnamed' % id(ssarepr)
         i = 1
-        # escape <lambda> names for windows
-        name = name.replace('<lambda>', '_(lambda)_')
+        # escape names like <lambda> for windows by removing any strange
+        # character; then make sure the names are not too long
+        name = ''.join(c for c in name if c.isalnum() or c == '_')[:60]
         extra = ''
         while dir.join(name+extra).check():
             i += 1
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -210,7 +210,6 @@
     return rlist.ll_pop(rlist.dum_checkidx, l, index)
 _ll_2_list_append = rlist.ll_append
 _ll_2_list_extend = rlist.ll_extend
-_ll_3_list_insert = rlist.ll_insert_nonneg
 _ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly
 _ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop
 _ll_2_list_inplace_mul = rlist.ll_inplace_mul
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4613,3 +4613,10 @@
         self.check_operations_history(guard_nonnull=0, guard_nonnull_class=0,
                                       guard_class=2,
                                       assert_not_none=2) # before optimization
+
+    def test_call_time_clock(self):
+        import time
+        def g():
+            time.clock()
+            return 0
+        self.interp_operations(g, [])
diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -212,6 +212,8 @@
                 s += lst[0]
                 lst.pop()
                 lst.append(1)
+                lst.insert(0, 5)
+                lst.insert(1, 6)
                 s *= lst.pop()
             return s
         res = self.meta_interp(f, [15], listops=True)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -158,7 +158,11 @@
 # record that ignore_finalizer() has been called
 GCFLAG_IGNORE_FINALIZER = first_gcflag << 10
 
-_GCFLAG_FIRST_UNUSED = first_gcflag << 11    # the first unused bit
+# shadow objects can have its memory initialized when it is created.
+# It does not need an additional copy in trace out
+GCFLAG_SHADOW_INITIALIZED   = first_gcflag << 11
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 12    # the first unused bit
 
 
 # States for the incremental GC
@@ -729,6 +733,16 @@
         obj = self.external_malloc(typeid, length, alloc_young=True)
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
+    def move_out_of_nursery(self, obj):
+        # called twice, it should return the same shadow object,
+        # and not creating another shadow object
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW:
+            shadow = self.nursery_objects_shadows.get(obj)
+            ll_assert(shadow != llmemory.NULL,
+                      "GCFLAG_HAS_SHADOW but no shadow found")
+            return shadow
+
+        return self._allocate_shadow(obj, copy=True)
 
     def collect(self, gen=2):
         """Do a minor (gen=0), start a major (gen=1), or do a full
@@ -1756,11 +1770,11 @@
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
-            pinned_obj_size = llarena.getfakearenaaddress(cur) - prev
+            free_range_size = llarena.getfakearenaaddress(cur) - prev
             if self.gc_nursery_debug:
-                llarena.arena_reset(prev, pinned_obj_size, 3)
+                llarena.arena_reset(prev, free_range_size, 3)
             else:
-                llarena.arena_reset(prev, pinned_obj_size, 0)
+                llarena.arena_reset(prev, free_range_size, 0)
             #
             # clean up object's flags
             obj = cur + size_gc_header
@@ -1770,7 +1784,7 @@
             nursery_barriers.append(cur)
             #
             # update 'prev' to the end of the 'cur' object
-            prev = prev + pinned_obj_size + \
+            prev = prev + free_range_size + \
                 (size_gc_header + self.get_size(obj))
         #
         # reset everything after the last pinned object till the end of the arena
@@ -1982,6 +1996,9 @@
                 and self.young_rawmalloced_objects.contains(obj)):
                 self._visit_young_rawmalloced_object(obj)
             return
+        # copy the contents of the object? usually yes, but not for some
+        # shadow objects
+        copy = True
         #
         size_gc_header = self.gcheaderbuilder.size_gc_header
         if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
@@ -2037,13 +2054,18 @@
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
             # copied to the shadow itself.
             self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+            tid = self.header(obj).tid
+            if (tid & GCFLAG_SHADOW_INITIALIZED) != 0:
+                copy = False
+                self.header(obj).tid &= ~GCFLAG_SHADOW_INITIALIZED
             #
             totalsize = size_gc_header + self.get_size(obj)
             self.nursery_surviving_size += raw_malloc_usage(totalsize)
         #
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.
-        llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
+        if copy:
+            llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
         #
         # Set the old object's tid to -42 (containing all flags) and
         # replace the old object's content with the target address.
@@ -2570,7 +2592,8 @@
     # ----------
     # id() and identityhash() support
 
-    def _allocate_shadow(self, obj):
+    @specialize.arg(2)
+    def _allocate_shadow(self, obj, copy=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         size = self.get_size(obj)
         shadowhdr = self._malloc_out_of_nursery(size_gc_header +
@@ -2592,6 +2615,12 @@
         #
         self.header(obj).tid |= GCFLAG_HAS_SHADOW
         self.nursery_objects_shadows.setitem(obj, shadow)
+
+        if copy:
+            self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED
+            totalsize = size_gc_header + self.get_size(obj)
+            llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize)
+
         return shadow
 
     def _find_shadow(self, obj):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -551,6 +551,13 @@
                                               [s_gc, SomeAddress()],
                                               annmodel.s_None)
 
+        self.move_out_of_nursery_ptr = None
+        if hasattr(GCClass, 'move_out_of_nursery'):
+            self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
+                                              [s_gc, SomeAddress()],
+                                              SomeAddress())
+
+
     def create_custom_trace_funcs(self, gc, rtyper):
         custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
         rtyper.custom_trace_funcs = custom_trace_funcs
@@ -1585,6 +1592,17 @@
             hop.genop("direct_call", [self.ignore_finalizer_ptr,
                                       self.c_const_gc, v_adr])
 
+    def gct_gc_move_out_of_nursery(self, hop):
+        if self.move_out_of_nursery_ptr is not None:
+            v_adr = hop.genop("cast_ptr_to_adr", [hop.spaceop.args[0]],
+                              resulttype=llmemory.Address)
+            v_ret = hop.genop("direct_call", [self.move_out_of_nursery_ptr,
+                                      self.c_const_gc, v_adr],
+                                      resulttype=llmemory.Address)
+            hop.genop("cast_adr_to_ptr", [v_ret],
+                      resultvar = hop.spaceop.result)
+
+
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -2,6 +2,8 @@
 Buffer protocol support.
 """
 from rpython.rlib import jit
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+        nonmoving_raw_ptr_for_resizable_list)
 
 
 class Buffer(object):
@@ -84,7 +86,7 @@
 
     def __init__(self, value):
         self.value = value
-        self.readonly = True
+        self.readonly = 1
 
     def getlength(self):
         return len(self.value)
@@ -108,6 +110,9 @@
             return self.value[start:stop]
         return Buffer.getslice(self, start, stop, step, size)
 
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        return rffi.get_raw_address_of_string(self.value)
 
 class SubBuffer(Buffer):
     _attrs_ = ['buffer', 'offset', 'size', 'readonly']
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -535,6 +535,25 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     llop.gc_ignore_finalizer(lltype.Void, obj)
 
+ at jit.dont_look_inside
+def move_out_of_nursery(obj):
+    """ Returns another object which is a copy of obj; but at any point
+        (either now or in the future) the returned object might suddenly
+        become identical to the one returned.
+
+        NOTE: Only use for immutable objects!
+    """
+    pass
+
+class MoveOutOfNurseryEntry(ExtRegistryEntry):
+    _about_ = move_out_of_nursery
+
+    def compute_result_annotation(self, s_obj):
+        return s_obj
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -997,12 +997,12 @@
                 if signal_checker is not None:
                     signal_checker()
 
-    def sendto(self, data, flags, address):
+    def sendto(self, data, length, flags, address):
         """Like send(data, flags) but allows specifying the destination
         address.  (Note that 'flags' is mandatory here.)"""
         self.wait_for_data(True)
         addr = address.lock()
-        res = _c.sendto(self.fd, data, len(data), flags,
+        res = _c.sendto(self.fd, data, length, flags,
                         addr, address.addrlen)
         address.unlock()
         if res < 0:
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -1,4 +1,4 @@
-from rpython.rlib.buffer import *
+from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer
 from rpython.annotator.annrpython import RPythonAnnotator
 from rpython.annotator.model import SomeInteger
 
@@ -64,3 +64,10 @@
     for i in range(9999, 9, -1):
         buf = SubBuffer(buf, 1, i)
     assert buf.getlength() == 10
+
+def test_string_buffer_as_buffer():
+    buf = StringBuffer(b'hello world')
+    addr = buf.get_raw_address()
+    assert addr[0] == b'h'
+    assert addr[4] == b'o'
+    assert addr[6] == b'w'
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -320,7 +320,7 @@
     s2.bind(INETAddress('127.0.0.1', INADDR_ANY))
     addr2 = s2.getsockname()
 
-    s1.sendto('?', 0, addr2)
+    s1.sendto('?', 1, 0, addr2)
     buf = s2.recv(100)
     assert buf == '?'
     s2.connect(addr)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1138,6 +1138,9 @@
         exc_data.exc_value = lltype.typeOf(evalue)._defl()
         return bool(etype)
 
+    def op_gc_move_out_of_nursery(self, obj):
+        raise NotImplementedError("gc_move_out_of_nursery")
+
 
 class Tracer(object):
     Counter = 0
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -496,6 +496,8 @@
     'gc_rawrefcount_to_obj':            LLOp(sideeffects=False),
     'gc_rawrefcount_next_dead':         LLOp(),
 
+    'gc_move_out_of_nursery':           LLOp(),
+
     # ------- JIT & GC interaction, only for some GCs ----------
 
     'gc_adr_of_nursery_free' : LLOp(),
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -739,6 +739,9 @@
 def op_gc_ignore_finalizer(obj):
     pass
 
+def op_gc_move_out_of_nursery(obj):
+    return obj
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -754,6 +754,7 @@
 SIGNED = lltype.Signed
 SIGNEDP = lltype.Ptr(lltype.Array(SIGNED, hints={'nolength': True}))
 
+
 # various type mapping
 
 # conversions between str and char*
@@ -876,6 +877,7 @@
     get_nonmovingbuffer._always_inline_ = 'try' # get rid of the returned tuple
     get_nonmovingbuffer._annenforceargs_ = [strtype]
 
+
     @jit.dont_look_inside
     def get_nonmovingbuffer_final_null(data):
         tup = get_nonmovingbuffer(data)
@@ -1310,3 +1312,52 @@
             releasegil=False,
             calling_conv='c',
         )
+
+
+if not we_are_translated():
+    class RawBytes(object):
+        # literal copy of _cffi_backend/func.py
+        def __init__(self, string):
+            self.ptr = str2charp(string, track_allocation=False)
+        def __del__(self):
+            if free_charp is not None:    # CPython shutdown
+                free_charp(self.ptr, track_allocation=False)
+
+    TEST_RAW_ADDR_KEEP_ALIVE = {}
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(string):
+    """Returns a 'char *' that is valid as long as the rpython string object is alive.
+    Two calls to to this function, given the same string parameter,
+    are guaranteed to return the same pointer.
+
+    The extra parameter key is necessary to create a weak reference.
+    The buffer of the returned pointer (if object is young) lives as long
+    as key is alive. If key goes out of scope, the buffer will eventually
+    be freed. `string` cannot go out of scope until the RawBytes object
+    referencing it goes out of scope.
+    """
+    assert isinstance(string, str)
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rtyper.lltypesystem.rstr import STR
+    from rpython.rtyper.lltypesystem import llmemory
+    from rpython.rlib import rgc
+
+    if we_are_translated():
+        if rgc.can_move(string):
+            string = rgc.move_out_of_nursery(string)
+
+        # string cannot move now! return the address
+        lldata = llstr(string)
+        data_start = (llmemory.cast_ptr_to_adr(lldata) +
+                      offsetof(STR, 'chars') +
+                      llmemory.itemoffsetof(STR.chars, 0))
+        data_start = cast(CCHARP, data_start)
+        data_start[len(string)] = '\x00'   # write the final extra null
+        return data_start
+    else:
+        global TEST_RAW_ADDR_KEEP_ALIVE
+        if string in TEST_RAW_ADDR_KEEP_ALIVE:
+            return TEST_RAW_ADDR_KEEP_ALIVE[string].ptr
+        TEST_RAW_ADDR_KEEP_ALIVE[string] = rb = RawBytes(string)
+        return rb.ptr
diff --git a/rpython/rtyper/lltypesystem/test/test_ztranslated.py b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
@@ -0,0 +1,59 @@
+import gc
+from rpython.translator.c.test.test_genc import compile
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rlib import rgc
+
+def debug_assert(boolresult, msg):
+    if not boolresult:
+        llop.debug_print(lltype.Void, "\n\nassert failed: %s\n\n" % msg)
+        assert boolresult
+
+def use_str():
+    mystr = b'abc'
+    #debug_assert(rgc.can_move(mystr), "short string cannot move... why?")
+    ptr = rffi.get_raw_address_of_string(mystr)
+    ptr2 = rffi.get_raw_address_of_string(mystr)
+    debug_assert(ptr == ptr2, "ptr != ptr2")
+    debug_assert(ptr[0] == b'a', "notnurseryadr[0] == b'a' is is %s" % ptr[0])
+    ptr[0] = b'x' # oh no no, in real programs nobody is allowed to modify that
+    debug_assert(mystr[0] == b'a', "mystr[0] != b'a'")
+    debug_assert(ptr[0] == b'x', "notnurseryadr[0] == b'x'")
+    gc.collect()
+    nptr = rffi.get_raw_address_of_string(mystr)
+    debug_assert(nptr == ptr, "second call to mystr must return the same ptr")
+    debug_assert(ptr[0] == b'x', "failure a")
+    debug_assert(nptr[0] == b'x', "failure b")
+    mystr = None
+
+def long_str(lstr):
+    ptr = rffi.get_raw_address_of_string(lstr)
+    for i,c in enumerate(lstr):
+        debug_assert(ptr[i] == c, "failure c")
+    gc.collect()
+    ptr2 = rffi.get_raw_address_of_string(lstr)
+    debug_assert(ptr == ptr2, "ptr != ptr2!!!")
+    return ptr
+
+def main(argv=[]):
+    use_str()
+    gc.collect()
+    mystr = b"12341234aa"*4096*10
+    #debug_assert(not rgc.can_move(mystr), "long string can move... why?")
+    p1 = long_str(mystr)
+    gc.collect()
+    copystr = mystr[:]
+    copystr += 'a'
+    p2 = long_str(copystr)
+    debug_assert(p1 != p2, "p1 == p2")
+    return 0
+
+# ____________________________________________________________
+
+def target(driver, args):
+    return main
+
+def test_compiled_incminimark():
+    fn = compile(main, [], gcpolicy="incminimark")
+    fn()
diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py
--- a/rpython/rtyper/rlist.py
+++ b/rpython/rtyper/rlist.py
@@ -588,6 +588,7 @@
     l.ll_setitem_fast(length, newitem)
 
 # this one is for the special case of insert(0, x)
+ at jit.look_inside_iff(lambda l,n: jit.isvirtual(l))
 def ll_prepend(l, newitem):
     length = l.ll_length()
     l._ll_resize_ge(length+1)           # see "a note about overflows" above
@@ -597,7 +598,6 @@
         l.ll_setitem_fast(dst, l.ll_getitem_fast(src))
         dst = src
     l.ll_setitem_fast(0, newitem)
-ll_prepend.oopspec = 'list.insert(l, 0, newitem)'
 
 def ll_concat(RESLIST, l1, l2):
     len1 = l1.ll_length()
@@ -612,6 +612,7 @@
     return l
 # no oopspec -- the function is inlined by the JIT
 
+ at jit.look_inside_iff(lambda l,i,n: jit.isvirtual(l) and jit.isconstant(i))
 def ll_insert_nonneg(l, index, newitem):
     length = l.ll_length()
     ll_assert(0 <= index, "negative list insertion index")
@@ -623,7 +624,6 @@
         l.ll_setitem_fast(dst, l.ll_getitem_fast(src))
         dst = src
     l.ll_setitem_fast(index, newitem)
-ll_insert_nonneg.oopspec = 'list.insert(l, index, newitem)'
 
 def ll_pop_nonneg(func, l, index):
     ll_assert(index >= 0, "unexpectedly negative list pop index")
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -33,6 +33,10 @@
             value, len(value), 'strict', final=True,
             errorhandler=self.ll_raise_unicode_exception_decode,
             allow_surrogates=False, result=result)
+        # XXX should it really be 'allow_surrogates=False'?  In RPython,
+        # unicode.decode('utf-8') happily accepts surrogates.  This
+        # makes it hard to test untranslated (it's the cause of a
+        # failure in lib-python's test_warnings on PyPy3, for example)
         return self.ll.llunicode(result.build())
 
     @staticmethod

From pypy.commits at gmail.com  Mon Jan  9 06:54:55 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 03:54:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: spaces instead of tabs
Message-ID: <58737a0f.cb911c0a.87fc9.1bdb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89434:3aca47a57d23
Date: 2017-01-09 11:13 +0100
http://bitbucket.org/pypy/pypy/changeset/3aca47a57d23/

Log:	spaces instead of tabs

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -205,7 +205,7 @@
     The memoryview object then owns the buffer represented by view, which
     means you shouldn't try to call PyBuffer_Release() yourself: it
     will be done on deallocation of the memoryview object."""
-	assert view.c_obj
+    assert view.c_obj
     w_obj = from_ref(space, view.c_obj)
     if isinstance(w_obj, W_MemoryView):
         return w_obj

From pypy.commits at gmail.com  Mon Jan  9 08:32:43 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 05:32:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge issure found
Message-ID: <587390fb.8d071c0a.adc2b.7aa8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89435:fd5118ae627c
Date: 2017-01-09 14:30 +0100
http://bitbucket.org/pypy/pypy/changeset/fd5118ae627c/

Log:	merge issure found

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -525,7 +525,7 @@
     'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor',
     'PyCapsule_SetContext', 'PyCapsule_Import', 'PyCapsule_Type', '_Py_get_capsule_type',
 
-    'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
+    #'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
 
     'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
 
@@ -1091,7 +1091,7 @@
             ptr.value = ctypes.cast(ll2ctypes.lltype2ctypes(value),
                                     ctypes.c_void_p).value
         elif typ in ('PyObject*', 'PyTypeObject*'):
-            if name.startswith('PyPyExc_') or name.startswith('cpyexttestExc_'):
+            if name.startswith('PyExc_'):
                 # we already have the pointer
                 in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, mname)
                 py_obj = ll2ctypes.ctypes2lltype(PyObject, in_dll)

From pypy.commits at gmail.com  Mon Jan  9 08:41:59 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 05:41:59 -0800 (PST)
Subject: [pypy-commit] pypy default: memoryview(b"hello") == 104 != "h", 2to3
Message-ID: <58739327.cb911c0a.87fc9.4b1d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r89436:aebaccb5bc42
Date: 2017-01-09 14:41 +0100
http://bitbucket.org/pypy/pypy/changeset/aebaccb5bc42/

Log:	memoryview(b"hello") == 104 != "h", 2to3

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -14,7 +14,7 @@
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
-        assert space.eq_w(w_char, space.wrap('h'))
+        assert space.eq_w(w_char, space.wrap(ord('h')))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 

From pypy.commits at gmail.com  Mon Jan  9 08:59:48 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 05:59:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: comment PyObject_CheckBuffer check in
 test (was commented before merge as well)
Message-ID: <58739754.46bb1c0a.9036.7f47@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89437:99efa1c2bbd5
Date: 2017-01-09 14:59 +0100
http://bitbucket.org/pypy/pypy/changeset/99efa1c2bbd5/

Log:	comment PyObject_CheckBuffer check in test (was commented before
	merge as well)

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -11,7 +11,7 @@
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
-        assert api.PyObject_CheckBuffer(w_hello)
+        #assert api.PyObject_CheckBuffer(w_hello)
         w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
         assert space.eq_w(w_char, space.wrap('h'))

From pypy.commits at gmail.com  Mon Jan  9 09:09:30 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 06:09:30 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merged py3.5-time changes
Message-ID: <5873999a.d4051c0a.e55e0.7e26@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89438:f58422e7ce44
Date: 2017-01-09 15:07 +0100
http://bitbucket.org/pypy/pypy/changeset/f58422e7ce44/

Log:	merged py3.5-time changes

diff --git a/lib_pypy/_structseq.py b/lib_pypy/_structseq.py
--- a/lib_pypy/_structseq.py
+++ b/lib_pypy/_structseq.py
@@ -41,18 +41,25 @@
                 assert field._index not in fields_by_index
                 fields_by_index[field._index] = field
                 field.__name__ = name
-        dict['n_fields'] = len(fields_by_index)
+        n_fields = len(fields_by_index)
+        dict['n_fields'] = n_fields
 
         extra_fields = sorted(fields_by_index.items())
         n_sequence_fields = 0
-        while extra_fields and extra_fields[0][0] == n_sequence_fields:
-            extra_fields.pop(0)
-            n_sequence_fields += 1
+        invis_fields = []
+        if 'n_sequence_fields' in dict:
+            n_sequence_fields = dict['n_sequence_fields']
+            extra_fields = extra_fields[n_sequence_fields:]
+        else:
+            while extra_fields and extra_fields[0][0] == n_sequence_fields:
+                extra_fields.pop(0)
+                n_sequence_fields += 1
+
         dict['n_sequence_fields'] = n_sequence_fields
         dict['n_unnamed_fields'] = 0     # no fully anonymous fields in PyPy
 
         extra_fields = [field for index, field in extra_fields]
-        for field in extra_fields:
+        for i,field in enumerate(extra_fields):
             field.index = None     # no longer relevant
 
         assert '__new__' not in dict
@@ -70,34 +77,39 @@
 def structseq_new(cls, sequence, dict={}):
     sequence = tuple(sequence)
     dict = builtin_dict(dict)
-    N = cls.n_sequence_fields
-    if len(sequence) < N:
-        if N < cls.n_fields:
+    # visible fields
+    visible_count = cls.n_sequence_fields
+    # total fields (unnamed are not yet supported, extra fields not included)
+    real_count = cls.n_fields
+    length = len(sequence)
+    if length < visible_count:
+        if visible_count < real_count:
             msg = "at least"
         else:
             msg = "exactly"
-        raise TypeError("expected a sequence with %s %d items" % (
-            msg, N))
-    if len(sequence) > N:
-        if len(sequence) > cls.n_fields:
-            if N < cls.n_fields:
+        raise TypeError("expected a sequence with %s %d items. has %d" % (
+            msg, visible_count, length))
+    if length > visible_count:
+        if length > real_count:
+            if visible_count < real_count:
                 msg = "at most"
             else:
                 msg = "exactly"
-            raise TypeError("expected a sequence with %s %d items" % (
-                msg, cls.n_fields))
-        for field, value in zip(cls._extra_fields, sequence[N:]):
+            raise TypeError("expected a sequence with %s %d items. has %d" \
+                            % (msg, real_count, length))
+        for field, value in zip(cls._extra_fields, sequence[visible_count:]):
             name = field.__name__
             if name in dict:
                 raise TypeError("duplicate value for %r" % (name,))
             dict[name] = value
-        sequence = sequence[:N]
+        sequence = sequence[:visible_count]
     result = tuple.__new__(cls, sequence)
     object.__setattr__(result, '__dict__', dict)
     for field in cls._extra_fields:
         name = field.__name__
         if name not in dict:
             dict[name] = field._default(result)
+
     return result
 
 def structseq_reduce(self):
@@ -109,9 +121,11 @@
 
 def structseq_repr(self):
     fields = {}
+    visible_count = self.n_sequence_fields
     for field in type(self).__dict__.values():
-        if isinstance(field, structseqfield):
+        if isinstance(field, structseqfield) and \
+           field._index <= visible_count:
             fields[field._index] = field
     parts = ["%s=%r" % (fields[index].__name__, value)
-             for index, value in enumerate(self)]
+            for index, value in enumerate(self[:visible_count])]
     return "%s(%s)" % (self._name, ", ".join(parts))
diff --git a/pypy/module/time/app_time.py b/pypy/module/time/app_time.py
--- a/pypy/module/time/app_time.py
+++ b/pypy/module/time/app_time.py
@@ -3,19 +3,26 @@
 from _structseq import structseqtype, structseqfield
 from types import SimpleNamespace
 import time
+
 class struct_time(metaclass=structseqtype):
     __module__ = 'time'
     name = 'time.struct_time'
 
-    tm_year   = structseqfield(0)
-    tm_mon    = structseqfield(1)
-    tm_mday   = structseqfield(2)
-    tm_hour   = structseqfield(3)
-    tm_min    = structseqfield(4)
-    tm_sec    = structseqfield(5)
-    tm_wday   = structseqfield(6)
-    tm_yday   = structseqfield(7)
-    tm_isdst  = structseqfield(8)
+    n_sequence_fields = 9
+
+    tm_year   = structseqfield(0, "year, for example, 1993")
+    tm_mon    = structseqfield(1, "month of year, range [1, 12]")
+    tm_mday   = structseqfield(2, "day of month, range [1, 31]")
+    tm_hour   = structseqfield(3, "hours, range [0, 23]")
+    tm_min    = structseqfield(4, "minutes, range [0, 59]")
+    tm_sec    = structseqfield(5, "seconds, range [0, 61])")
+    tm_wday   = structseqfield(6, "day of week, range [0, 6], Monday is 0")
+    tm_yday   = structseqfield(7, "day of year, range [1, 366]")
+    tm_isdst  = structseqfield(8, "1 if summer time is in effect, 0 if not"
+                                  ", and -1 if unknown")
+    tm_zone   = structseqfield(9, "abbreviation of timezone name")
+    tm_gmtoff = structseqfield(10,"offset from UTC in seconds")
+
 
 def strptime(string, format="%a %b %d %H:%M:%S %Y"):
     """strptime(string, format) -> struct_time
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -1,8 +1,10 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror, exception_from_saved_errno
+from pypy.interpreter.error import (OperationError, oefmt,
+        strerror as _strerror, exception_from_saved_errno)
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter import timeutils
+from pypy.interpreter.unicodehelper import decode_utf8, encode_utf8
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat, widen
 from rpython.rlib.rtime import (GETTIMEOFDAY_NO_TZ, TIMEVAL,
@@ -164,6 +166,8 @@
     CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
     has_gettimeofday = platform.Has('gettimeofday')
 
+HAS_TM_ZONE = False
+
 if _POSIX:
     calling_conv = 'c'
     CConfig.timeval = platform.Struct("struct timeval",
@@ -180,6 +184,9 @@
             ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
             ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG),
             ("tm_zone", rffi.CCHARP)])
+
+        HAS_TM_ZONE = True
+
 elif _WIN:
     calling_conv = 'win'
     CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
@@ -195,6 +202,8 @@
 
 # XXX: optionally support the 2 additional tz fields
 _STRUCT_TM_ITEMS = 9
+if HAS_TM_ZONE:
+    _STRUCT_TM_ITEMS = 11
 
 class cConfig:
     pass
@@ -537,9 +546,18 @@
         space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
         space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
 
+    if HAS_TM_ZONE:
+        # CPython calls PyUnicode_DecodeLocale here should we do the same?
+        tm_zone = decode_utf8(space, rffi.charp2str(t.c_tm_zone),
+                              allow_surrogates=True)
+        extra = [space.newunicode(tm_zone),
+                 space.wrap(rffi.getintfield(t, 'c_tm_gmtoff'))]
+        w_time_tuple = space.newtuple(time_tuple + extra)
+    else:
+        w_time_tuple = space.newtuple(time_tuple)
     w_struct_time = _get_module_object(space, 'struct_time')
-    w_time_tuple = space.newtuple(time_tuple)
-    return space.call_function(w_struct_time, w_time_tuple)
+    w_obj = space.call_function(w_struct_time, w_time_tuple)
+    return w_obj
 
 def _gettmarg(space, w_tup, allowNone=True):
     if space.is_none(w_tup):
@@ -559,9 +577,9 @@
         return pbuf
 
     tup_w = space.fixedview(w_tup)
-    if len(tup_w) != 9:
+    if len(tup_w) < 9:
         raise oefmt(space.w_TypeError,
-                    "argument must be sequence of length 9, not %d",
+                    "argument must be sequence of at least length 9, not %d",
                     len(tup_w))
 
     y = space.c_int_w(tup_w[0])
@@ -582,13 +600,23 @@
     rffi.setintfield(glob_buf, 'c_tm_wday', space.c_int_w(tup_w[6]))
     rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
     rffi.setintfield(glob_buf, 'c_tm_isdst', space.c_int_w(tup_w[8]))
-    if _POSIX:
-        if _CYGWIN:
-            pass
-        else:
-            # actually never happens, but makes annotator happy
-            glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
-            rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
+    #
+    old_tm_zone = glob_buf.c_tm_zone
+    glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
+    rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
+    if HAS_TM_ZONE :
+        if len(tup_w) >= 10:
+            # NOTE this is not cleanly solved!
+            # it saves the string that is later deleted when this
+            # function is called again. A refactoring of this module
+            # could remove this
+            tm_zone = encode_utf8(space, space.unicode_w(tup_w[9]), allow_surrogates=True)
+            malloced_str = rffi.str2charp(tm_zone, track_allocation=False)
+            if old_tm_zone != lltype.nullptr(rffi.CCHARP.TO):
+                rffi.free_charp(old_tm_zone, track_allocation=False)
+            glob_buf.c_tm_zone = malloced_str
+        if len(tup_w) >= 11:
+            rffi.setintfield(glob_buf, 'c_tm_gmtoff', space.c_int_w(tup_w[10]))
 
     # tm_wday does not need checking of its upper-bound since taking "%
     #  7" in _gettmarg() automatically restricts the range.
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -254,6 +254,22 @@
                 del os.environ['TZ']
             time.tzset()
 
+    def test_localtime_timezone(self):
+        import os, time
+        org_TZ = os.environ.get('TZ', None)
+        try:
+            os.environ['TZ'] = 'Europe/Kiev'
+            time.tzset()
+            localtm = time.localtime(0)
+            assert localtm.tm_zone == "MSK"
+            assert localtm.tm_gmtoff == 10800
+        finally:
+            if org_TZ is not None:
+                os.environ['TZ'] = org_TZ
+            elif 'TZ' in os.environ:
+                del os.environ['TZ']
+            time.tzset()
+
     def test_strftime(self):
         import time
         import os, sys

From pypy.commits at gmail.com  Mon Jan  9 09:09:35 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 06:09:35 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add two branches to thasnew pypy3
Message-ID: <5873999f.4dd41c0a.82ec4.8619@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89440:ec81607f0874
Date: 2017-01-09 15:08 +0100
http://bitbucket.org/pypy/pypy/changeset/ec81607f0874/

Log:	add two branches to thasnew pypy3

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -33,3 +33,6 @@
 
 .. branch: py3k-update
 
+.. branch: py3.5-time
+
+.. branch: py3.5-ssl

From pypy.commits at gmail.com  Mon Jan  9 09:09:33 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 06:09:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-time: close branch
Message-ID: <5873999d.4406c30a.6667b.ed36@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-time
Changeset: r89439:fd3ba0d1944d
Date: 2017-01-09 15:08 +0100
http://bitbucket.org/pypy/pypy/changeset/fd3ba0d1944d/

Log:	close branch


From pypy.commits at gmail.com  Mon Jan  9 09:34:07 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 06:34:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: do not use BaseImportTest as baseclass
 for zipimport tests (this class was removed)
Message-ID: <58739f5f.c9f8c20a.4c3dc.42ac@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89441:21a8a2cfa452
Date: 2017-01-09 15:33 +0100
http://bitbucket.org/pypy/pypy/changeset/21a8a2cfa452/

Log:	do not use BaseImportTest as baseclass for zipimport tests (this
	class was removed)

diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -4,11 +4,10 @@
 import time
 from zipfile import ZIP_STORED
 
-from pypy.module.imp.test.support import BaseImportTest
 from rpython.tool.udir import udir
 
 
-class AppTestZipimport(BaseImportTest):
+class AppTestZipimport:
     """ A bit structurized tests stolen and adapted from
     cpy's regression tests
     """
@@ -20,7 +19,6 @@
 
     @classmethod
     def make_class(cls):
-        BaseImportTest.setup_class.im_func(cls)
         space = cls.space
         w = space.wrap
 

From pypy.commits at gmail.com  Mon Jan  9 09:53:18 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 06:53:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: use BaseFSEncodeTest instead (support
 file was moved), this seems like a merge conflict introduced long ago
Message-ID: <5873a3de.4f831c0a.ea763.9825@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89442:01b498518160
Date: 2017-01-09 15:52 +0100
http://bitbucket.org/pypy/pypy/changeset/01b498518160/

Log:	use BaseFSEncodeTest instead (support file was moved), this seems
	like a merge conflict introduced long ago

diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -4,10 +4,11 @@
 import time
 from zipfile import ZIP_STORED
 
+from pypy.interpreter.test.test_fsencode import BaseFSEncodeTest
 from rpython.tool.udir import udir
 
 
-class AppTestZipimport:
+class AppTestZipimport(BaseFSEncodeTest):
     """ A bit structurized tests stolen and adapted from
     cpy's regression tests
     """
@@ -19,6 +20,7 @@
 
     @classmethod
     def make_class(cls):
+        BaseFSEncodeTest.setup_class.im_func(cls)
         space = cls.space
         w = space.wrap
 

From pypy.commits at gmail.com  Mon Jan  9 09:55:56 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 06:55:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: readd PyComplex_AsCComplex &
 PyComplex_FromCComplex to C_SYMBOLS
Message-ID: <5873a47c.aa1ac20a.c4a2.947f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89443:3e07ac8b799c
Date: 2017-01-09 15:55 +0100
http://bitbucket.org/pypy/pypy/changeset/3e07ac8b799c/

Log:	readd PyComplex_AsCComplex & PyComplex_FromCComplex to C_SYMBOLS

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -525,7 +525,7 @@
     'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor',
     'PyCapsule_SetContext', 'PyCapsule_Import', 'PyCapsule_Type', '_Py_get_capsule_type',
 
-    #'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
+    'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
 
     'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
 

From pypy.commits at gmail.com  Mon Jan  9 10:08:41 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 07:08:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: cpyext compile sources in complexobject.c
Message-ID: <5873a779.ca10c20a.b0762.5403@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89444:c32c88be4363
Date: 2017-01-09 16:07 +0100
http://bitbucket.org/pypy/pypy/changeset/c32c88be4363/

Log:	cpyext compile sources in complexobject.c

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1253,6 +1253,7 @@
                          source_dir / "missing.c",
                          source_dir / "pymem.c",
                          source_dir / "bytesobject.c",
+                         source_dir / "complexobject.c",
                          ]
 
 def build_eci(code, use_micronumpy=False, translating=False):

From pypy.commits at gmail.com  Mon Jan  9 11:02:29 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 08:02:29 -0800 (PST)
Subject: [pypy-commit] pypy default: add b prefix to string to pass bytes
 for this test
Message-ID: <5873b415.54b31c0a.57346.b28b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r89445:245faec2ea07
Date: 2017-01-09 17:01 +0100
http://bitbucket.org/pypy/pypy/changeset/245faec2ea07/

Log:	add b prefix to string to pass bytes for this test

diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -56,7 +56,7 @@
         assert u"abc" != memoryview("abc")
 
     def test_pypy_raw_address_base(self):
-        a = memoryview("foobar")._pypy_raw_address()
+        a = memoryview(b"foobar")._pypy_raw_address()
         assert a != 0
-        b = memoryview(bytearray("foobar"))._pypy_raw_address()
+        b = memoryview(bytearray(b"foobar"))._pypy_raw_address()
         assert b != 0

From pypy.commits at gmail.com  Mon Jan  9 11:02:31 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 08:02:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge default
Message-ID: <5873b417.a351c20a.48d77.79c6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89446:596ff2168cf5
Date: 2017-01-09 17:01 +0100
http://bitbucket.org/pypy/pypy/changeset/596ff2168cf5/

Log:	merge default

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -14,7 +14,7 @@
         #assert api.PyObject_CheckBuffer(w_hello)
         w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
-        assert space.eq_w(w_char, space.wrap('h'))
+        assert space.eq_w(w_char, space.wrap(ord('h')))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -185,9 +185,9 @@
         assert m[2] == 1
 
     def test_pypy_raw_address_base(self):
-        a = memoryview("foobar")._pypy_raw_address()
+        a = memoryview(b"foobar")._pypy_raw_address()
         assert a != 0
-        b = memoryview(bytearray("foobar"))._pypy_raw_address()
+        b = memoryview(bytearray(b"foobar"))._pypy_raw_address()
         assert b != 0
 
     def test_hex(self):

From pypy.commits at gmail.com  Mon Jan  9 12:25:22 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 09 Jan 2017 09:25:22 -0800 (PST)
Subject: [pypy-commit] pypy default: revert aebaccb5bc42,
 checked in on wrong branch
Message-ID: <5873c782.2738c20a.6d2db.9f0d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r89447:43b0e50c2403
Date: 2017-01-09 18:24 +0100
http://bitbucket.org/pypy/pypy/changeset/43b0e50c2403/

Log:	revert aebaccb5bc42, checked in on wrong branch

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -14,7 +14,7 @@
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
-        assert space.eq_w(w_char, space.wrap(ord('h')))
+        assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 

From pypy.commits at gmail.com  Mon Jan  9 12:37:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 09 Jan 2017 09:37:58 -0800 (PST)
Subject: [pypy-commit] pypy default: Regenerate unicodedb versions 6.1.0 and
 6.2.0 with a fixed cjk_interval
Message-ID: <5873ca76.ce181c0a.14d74.ec21@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89448:dc1ce991bcaf
Date: 2017-01-09 18:37 +0100
http://bitbucket.org/pypy/pypy/changeset/dc1ce991bcaf/

Log:	Regenerate unicodedb versions 6.1.0 and 6.2.0 with a fixed
	cjk_interval

diff --git a/rpython/rlib/unicodedata/generate_unicodedb.py b/rpython/rlib/unicodedata/generate_unicodedb.py
--- a/rpython/rlib/unicodedata/generate_unicodedb.py
+++ b/rpython/rlib/unicodedata/generate_unicodedb.py
@@ -569,9 +569,15 @@
                         " 0x4E00 <= code <= 0x9FCB or"
                         " 0x20000 <= code <= 0x2A6D6 or"
                         " 0x2A700 <= code <= 0x2B734)")
+    elif version < "6.1":
+        cjk_interval = ("(0x3400 <= code <= 0x4DB5 or"
+                        " 0x4E00 <= code <= 0x9FCB or"
+                        " 0x20000 <= code <= 0x2A6D6 or"
+                        " 0x2A700 <= code <= 0x2B734 or"
+                        " 0x2B740 <= code <= 0x2B81D)")
     else:
         cjk_interval = ("(0x3400 <= code <= 0x4DB5 or"
-                        " 0x4E00 <= code <= 0x9FCB or"
+                        " 0x4E00 <= code <= 0x9FCC or"
                         " 0x20000 <= code <= 0x2A6D6 or"
                         " 0x2A700 <= code <= 0x2B734 or"
                         " 0x2B740 <= code <= 0x2B81D)")
diff --git a/rpython/rlib/unicodedata/unicodedb_6_1_0.py b/rpython/rlib/unicodedata/unicodedb_6_1_0.py
--- a/rpython/rlib/unicodedata/unicodedb_6_1_0.py
+++ b/rpython/rlib/unicodedata/unicodedb_6_1_0.py
@@ -6882,7 +6882,7 @@
         if not ('0' <= c <= '9' or 'A' <= c <= 'F'):
             raise KeyError
     code = int(cjk_code, 16)
-    if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCB or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D):
+    if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCC or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D):
         return code
     raise KeyError
 
@@ -6907,7 +6907,7 @@
     return code
 
 def name(code):
-    if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCB or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D):
+    if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCC or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D):
         return "CJK UNIFIED IDEOGRAPH-" + hex(code)[2:].upper()
     if 0xAC00 <= code <= 0xD7A3:
         # vl_code, t_code = divmod(code - 0xAC00, len(_hangul_T))
diff --git a/rpython/rlib/unicodedata/unicodedb_6_2_0.py b/rpython/rlib/unicodedata/unicodedb_6_2_0.py
--- a/rpython/rlib/unicodedata/unicodedb_6_2_0.py
+++ b/rpython/rlib/unicodedata/unicodedb_6_2_0.py
@@ -6886,7 +6886,7 @@
         if not ('0' <= c <= '9' or 'A' <= c <= 'F'):
             raise KeyError
     code = int(cjk_code, 16)
-    if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCB or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D):
+    if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCC or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D):
         return code
     raise KeyError
 
@@ -6911,7 +6911,7 @@
     return code
 
 def name(code):
-    if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCB or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D):
+    if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCC or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D):
         return "CJK UNIFIED IDEOGRAPH-" + hex(code)[2:].upper()
     if 0xAC00 <= code <= 0xD7A3:
         # vl_code, t_code = divmod(code - 0xAC00, len(_hangul_T))

From pypy.commits at gmail.com  Mon Jan  9 12:55:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 09 Jan 2017 09:55:58 -0800 (PST)
Subject: [pypy-commit] pypy default: Add version 8.0.0 of the unicode
 database (for 3.5)
Message-ID: <5873ceae.4c9d1c0a.2a4b4.f25e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89449:17b03637ebc2
Date: 2017-01-09 18:54 +0100
http://bitbucket.org/pypy/pypy/changeset/17b03637ebc2/

Log:	Add version 8.0.0 of the unicode database (for 3.5)

	Note that CPython uses DerivedNormalizationProps-x.x.x.txt at least
	since 3.2, which is missing here.

diff too long, truncating to 2000 out of 70274 lines

diff --git a/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
@@ -0,0 +1,1414 @@
+# CaseFolding-8.0.0.txt
+# Date: 2015-01-13, 18:16:36 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#223;e" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding 
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+# Property: Case_Folding
+
+#  All code points not explicitly listed for Case_Folding
+#  have the value C for the status field, and the code point itself for the mapping field.
+
+# =================================================================
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE
+023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE
+023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR
+023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
+0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP
+0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE
+0244; C; 0289; # LATIN CAPITAL LETTER U BAR
+0245; C; 028C; # LATIN CAPITAL LETTER TURNED V
+0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE
+0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE
+024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
+024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE
+024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0370; C; 0371; # GREEK CAPITAL LETTER HETA
+0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI
+0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
+037F; C; 03F3; # GREEK CAPITAL LETTER YOT
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+03F7; C; 03F8; # GREEK CAPITAL LETTER SHO
+03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL
+03FA; C; 03FB; # GREEK CAPITAL LETTER SAN
+03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
+03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
+03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
+04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK
+04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE
+0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK
+0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA
+0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA
+0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE
+051A; C; 051B; # CYRILLIC CAPITAL LETTER QA
+051C; C; 051D; # CYRILLIC CAPITAL LETTER WE
+051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA
+0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
+0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
+0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER
+0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER
+0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK
+052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE
+052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE
+052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN
+10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN
+10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN
+10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON
+10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN
+10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN
+10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN
+10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN
+10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN
+10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN
+10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS
+10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN
+10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR
+10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON
+10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR
+10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR
+10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE
+10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN
+10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR
+10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN
+10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR
+10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR
+10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN
+10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR
+10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN
+10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN
+10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN
+10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL
+10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL
+10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR
+10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN
+10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN
+10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE
+10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE
+10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE
+10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE
+10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR
+10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE
+10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN
+10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN
+13F8; C; 13F0; # CHEROKEE SMALL LETTER YE
+13F9; C; 13F1; # CHEROKEE SMALL LETTER YI
+13FA; C; 13F2; # CHEROKEE SMALL LETTER YO
+13FB; C; 13F3; # CHEROKEE SMALL LETTER YU
+13FC; C; 13F4; # CHEROKEE SMALL LETTER YV
+13FD; C; 13F5; # CHEROKEE SMALL LETTER MV
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S
+1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL
+1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V
+1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2132; C; 214E; # TURNED CAPITAL F
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N
+24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O
+24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P
+24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q
+24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R
+24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S
+24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T
+24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U
+24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V
+24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W
+24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X
+24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y
+24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z
+2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU
+2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY
+2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE
+2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI
+2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO
+2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU
+2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE
+2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO
+2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA
+2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE
+2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE
+2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I
+2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI
+2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO
+2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE
+2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE
+2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI
+2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU
+2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI
+2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI
+2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO
+2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO
+2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU
+2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU
+2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU
+2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU
+2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE
+2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA
+2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI
+2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI
+2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA
+2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU
+2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI
+2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI
+2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA
+2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU
+2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS
+2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL
+2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO
+2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS
+2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS
+2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS
+2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA
+2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA
+2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC
+2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A
+2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE
+2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR
+2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE
+2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE
+2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL
+2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER
+2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER
+2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER
+2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA
+2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK
+2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A
+2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA
+2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK
+2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H
+2C7E; C; 023F; # LATIN CAPITAL LETTER S WITH SWASH TAIL
+2C7F; C; 0240; # LATIN CAPITAL LETTER Z WITH SWASH TAIL
+2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA
+2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA
+2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA
+2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA
+2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE
+2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU
+2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA
+2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE
+2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE
+2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA
+2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA
+2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA
+2C98; C; 2C99; # COPTIC CAPITAL LETTER MI
+2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI
+2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI
+2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O
+2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI
+2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO
+2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA
+2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU
+2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA
+2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI
+2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI
+2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI
+2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU
+2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF
+2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN
+2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE
+2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA
+2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI
+2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI
+2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU
+2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI
+2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI
+2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI
+2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH
+2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI
+2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI
+2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI
+2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA
+2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA
+2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI
+2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT
+2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA
+2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA
+2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA
+2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA
+2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI
+2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI
+2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU
+2CEB; C; 2CEC; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI
+2CED; C; 2CEE; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA
+2CF2; C; 2CF3; # COPTIC CAPITAL LETTER BOHAIRIC KHEI
+A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA
+A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO
+A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE
+A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA
+A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV
+A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK
+A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA
+A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER
+A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER
+A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT
+A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU
+A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A
+A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS
+A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS
+A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS
+A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN
+A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE
+A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE
+A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL
+A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM
+A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O
+A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O
+A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O
+A680; C; A681; # CYRILLIC CAPITAL LETTER DWE
+A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE
+A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE
+A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE
+A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE
+A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK
+A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE
+A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE
+A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE
+A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE
+A694; C; A695; # CYRILLIC CAPITAL LETTER HWE
+A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE
+A698; C; A699; # CYRILLIC CAPITAL LETTER DOUBLE O
+A69A; C; A69B; # CYRILLIC CAPITAL LETTER CROSSED O
+A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF
+A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN
+A726; C; A727; # LATIN CAPITAL LETTER HENG
+A728; C; A729; # LATIN CAPITAL LETTER TZ
+A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO
+A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO
+A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA
+A732; C; A733; # LATIN CAPITAL LETTER AA
+A734; C; A735; # LATIN CAPITAL LETTER AO
+A736; C; A737; # LATIN CAPITAL LETTER AU
+A738; C; A739; # LATIN CAPITAL LETTER AV
+A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR
+A73C; C; A73D; # LATIN CAPITAL LETTER AY
+A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT
+A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE
+A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE
+A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE
+A746; C; A747; # LATIN CAPITAL LETTER BROKEN L
+A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE
+A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY
+A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP
+A74E; C; A74F; # LATIN CAPITAL LETTER OO
+A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER
+A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH
+A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL
+A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER
+A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE
+A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA
+A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA
+A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE
+A760; C; A761; # LATIN CAPITAL LETTER VY
+A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z
+A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE
+A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER
+A768; C; A769; # LATIN CAPITAL LETTER VEND
+A76A; C; A76B; # LATIN CAPITAL LETTER ET
+A76C; C; A76D; # LATIN CAPITAL LETTER IS
+A76E; C; A76F; # LATIN CAPITAL LETTER CON
+A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D
+A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F
+A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G
+A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G
+A780; C; A781; # LATIN CAPITAL LETTER TURNED L
+A782; C; A783; # LATIN CAPITAL LETTER INSULAR R
+A784; C; A785; # LATIN CAPITAL LETTER INSULAR S
+A786; C; A787; # LATIN CAPITAL LETTER INSULAR T
+A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO
+A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H
+A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER
+A792; C; A793; # LATIN CAPITAL LETTER C WITH BAR
+A796; C; A797; # LATIN CAPITAL LETTER B WITH FLOURISH
+A798; C; A799; # LATIN CAPITAL LETTER F WITH STROKE
+A79A; C; A79B; # LATIN CAPITAL LETTER VOLAPUK AE
+A79C; C; A79D; # LATIN CAPITAL LETTER VOLAPUK OE
+A79E; C; A79F; # LATIN CAPITAL LETTER VOLAPUK UE
+A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE
+A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE
+A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE
+A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE
+A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE
+A7AA; C; 0266; # LATIN CAPITAL LETTER H WITH HOOK
+A7AB; C; 025C; # LATIN CAPITAL LETTER REVERSED OPEN E
+A7AC; C; 0261; # LATIN CAPITAL LETTER SCRIPT G
+A7AD; C; 026C; # LATIN CAPITAL LETTER L WITH BELT
+A7B0; C; 029E; # LATIN CAPITAL LETTER TURNED K
+A7B1; C; 0287; # LATIN CAPITAL LETTER TURNED T
+A7B2; C; 029D; # LATIN CAPITAL LETTER J WITH CROSSED-TAIL
+A7B3; C; AB53; # LATIN CAPITAL LETTER CHI
+A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA
+A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA
+AB70; C; 13A0; # CHEROKEE SMALL LETTER A
+AB71; C; 13A1; # CHEROKEE SMALL LETTER E
+AB72; C; 13A2; # CHEROKEE SMALL LETTER I
+AB73; C; 13A3; # CHEROKEE SMALL LETTER O
+AB74; C; 13A4; # CHEROKEE SMALL LETTER U
+AB75; C; 13A5; # CHEROKEE SMALL LETTER V
+AB76; C; 13A6; # CHEROKEE SMALL LETTER GA
+AB77; C; 13A7; # CHEROKEE SMALL LETTER KA
+AB78; C; 13A8; # CHEROKEE SMALL LETTER GE
+AB79; C; 13A9; # CHEROKEE SMALL LETTER GI
+AB7A; C; 13AA; # CHEROKEE SMALL LETTER GO
+AB7B; C; 13AB; # CHEROKEE SMALL LETTER GU
+AB7C; C; 13AC; # CHEROKEE SMALL LETTER GV
+AB7D; C; 13AD; # CHEROKEE SMALL LETTER HA
+AB7E; C; 13AE; # CHEROKEE SMALL LETTER HE
+AB7F; C; 13AF; # CHEROKEE SMALL LETTER HI
+AB80; C; 13B0; # CHEROKEE SMALL LETTER HO
+AB81; C; 13B1; # CHEROKEE SMALL LETTER HU
+AB82; C; 13B2; # CHEROKEE SMALL LETTER HV
+AB83; C; 13B3; # CHEROKEE SMALL LETTER LA
+AB84; C; 13B4; # CHEROKEE SMALL LETTER LE
+AB85; C; 13B5; # CHEROKEE SMALL LETTER LI
+AB86; C; 13B6; # CHEROKEE SMALL LETTER LO
+AB87; C; 13B7; # CHEROKEE SMALL LETTER LU
+AB88; C; 13B8; # CHEROKEE SMALL LETTER LV
+AB89; C; 13B9; # CHEROKEE SMALL LETTER MA
+AB8A; C; 13BA; # CHEROKEE SMALL LETTER ME
+AB8B; C; 13BB; # CHEROKEE SMALL LETTER MI
+AB8C; C; 13BC; # CHEROKEE SMALL LETTER MO
+AB8D; C; 13BD; # CHEROKEE SMALL LETTER MU
+AB8E; C; 13BE; # CHEROKEE SMALL LETTER NA
+AB8F; C; 13BF; # CHEROKEE SMALL LETTER HNA
+AB90; C; 13C0; # CHEROKEE SMALL LETTER NAH
+AB91; C; 13C1; # CHEROKEE SMALL LETTER NE
+AB92; C; 13C2; # CHEROKEE SMALL LETTER NI
+AB93; C; 13C3; # CHEROKEE SMALL LETTER NO
+AB94; C; 13C4; # CHEROKEE SMALL LETTER NU
+AB95; C; 13C5; # CHEROKEE SMALL LETTER NV
+AB96; C; 13C6; # CHEROKEE SMALL LETTER QUA
+AB97; C; 13C7; # CHEROKEE SMALL LETTER QUE
+AB98; C; 13C8; # CHEROKEE SMALL LETTER QUI
+AB99; C; 13C9; # CHEROKEE SMALL LETTER QUO
+AB9A; C; 13CA; # CHEROKEE SMALL LETTER QUU
+AB9B; C; 13CB; # CHEROKEE SMALL LETTER QUV
+AB9C; C; 13CC; # CHEROKEE SMALL LETTER SA
+AB9D; C; 13CD; # CHEROKEE SMALL LETTER S
+AB9E; C; 13CE; # CHEROKEE SMALL LETTER SE
+AB9F; C; 13CF; # CHEROKEE SMALL LETTER SI
+ABA0; C; 13D0; # CHEROKEE SMALL LETTER SO
+ABA1; C; 13D1; # CHEROKEE SMALL LETTER SU
+ABA2; C; 13D2; # CHEROKEE SMALL LETTER SV
+ABA3; C; 13D3; # CHEROKEE SMALL LETTER DA
+ABA4; C; 13D4; # CHEROKEE SMALL LETTER TA
+ABA5; C; 13D5; # CHEROKEE SMALL LETTER DE
+ABA6; C; 13D6; # CHEROKEE SMALL LETTER TE
+ABA7; C; 13D7; # CHEROKEE SMALL LETTER DI
+ABA8; C; 13D8; # CHEROKEE SMALL LETTER TI
+ABA9; C; 13D9; # CHEROKEE SMALL LETTER DO
+ABAA; C; 13DA; # CHEROKEE SMALL LETTER DU
+ABAB; C; 13DB; # CHEROKEE SMALL LETTER DV
+ABAC; C; 13DC; # CHEROKEE SMALL LETTER DLA
+ABAD; C; 13DD; # CHEROKEE SMALL LETTER TLA
+ABAE; C; 13DE; # CHEROKEE SMALL LETTER TLE
+ABAF; C; 13DF; # CHEROKEE SMALL LETTER TLI
+ABB0; C; 13E0; # CHEROKEE SMALL LETTER TLO
+ABB1; C; 13E1; # CHEROKEE SMALL LETTER TLU
+ABB2; C; 13E2; # CHEROKEE SMALL LETTER TLV
+ABB3; C; 13E3; # CHEROKEE SMALL LETTER TSA
+ABB4; C; 13E4; # CHEROKEE SMALL LETTER TSE
+ABB5; C; 13E5; # CHEROKEE SMALL LETTER TSI
+ABB6; C; 13E6; # CHEROKEE SMALL LETTER TSO
+ABB7; C; 13E7; # CHEROKEE SMALL LETTER TSU
+ABB8; C; 13E8; # CHEROKEE SMALL LETTER TSV
+ABB9; C; 13E9; # CHEROKEE SMALL LETTER WA
+ABBA; C; 13EA; # CHEROKEE SMALL LETTER WE
+ABBB; C; 13EB; # CHEROKEE SMALL LETTER WI
+ABBC; C; 13EC; # CHEROKEE SMALL LETTER WO
+ABBD; C; 13ED; # CHEROKEE SMALL LETTER WU
+ABBE; C; 13EE; # CHEROKEE SMALL LETTER WV
+ABBF; C; 13EF; # CHEROKEE SMALL LETTER YA
+FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF
+FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI
+FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
+FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI
+FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL
+FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T
+FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST
+FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW
+FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH
+FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI
+FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW
+FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH
+FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A
+FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B
+FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C
+FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D
+FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E
+FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F
+FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G
+FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H
+FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I
+FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J
+FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K
+FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L
+FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M
+FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N
+FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O
+FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P
+FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q
+FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R
+FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S
+FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T
+FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U
+FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V
+FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W
+FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X
+FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y
+FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
+10400; C; 10428; # DESERET CAPITAL LETTER LONG I
+10401; C; 10429; # DESERET CAPITAL LETTER LONG E
+10402; C; 1042A; # DESERET CAPITAL LETTER LONG A
+10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH
+10404; C; 1042C; # DESERET CAPITAL LETTER LONG O
+10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO
+10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I
+10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E
+10408; C; 10430; # DESERET CAPITAL LETTER SHORT A
+10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH
+1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O
+1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO
+1040C; C; 10434; # DESERET CAPITAL LETTER AY
+1040D; C; 10435; # DESERET CAPITAL LETTER OW
+1040E; C; 10436; # DESERET CAPITAL LETTER WU
+1040F; C; 10437; # DESERET CAPITAL LETTER YEE
+10410; C; 10438; # DESERET CAPITAL LETTER H
+10411; C; 10439; # DESERET CAPITAL LETTER PEE
+10412; C; 1043A; # DESERET CAPITAL LETTER BEE
+10413; C; 1043B; # DESERET CAPITAL LETTER TEE
+10414; C; 1043C; # DESERET CAPITAL LETTER DEE
+10415; C; 1043D; # DESERET CAPITAL LETTER CHEE
+10416; C; 1043E; # DESERET CAPITAL LETTER JEE
+10417; C; 1043F; # DESERET CAPITAL LETTER KAY
+10418; C; 10440; # DESERET CAPITAL LETTER GAY
+10419; C; 10441; # DESERET CAPITAL LETTER EF
+1041A; C; 10442; # DESERET CAPITAL LETTER VEE
+1041B; C; 10443; # DESERET CAPITAL LETTER ETH
+1041C; C; 10444; # DESERET CAPITAL LETTER THEE
+1041D; C; 10445; # DESERET CAPITAL LETTER ES
+1041E; C; 10446; # DESERET CAPITAL LETTER ZEE
+1041F; C; 10447; # DESERET CAPITAL LETTER ESH
+10420; C; 10448; # DESERET CAPITAL LETTER ZHEE
+10421; C; 10449; # DESERET CAPITAL LETTER ER
+10422; C; 1044A; # DESERET CAPITAL LETTER EL
+10423; C; 1044B; # DESERET CAPITAL LETTER EM
+10424; C; 1044C; # DESERET CAPITAL LETTER EN
+10425; C; 1044D; # DESERET CAPITAL LETTER ENG
+10426; C; 1044E; # DESERET CAPITAL LETTER OI
+10427; C; 1044F; # DESERET CAPITAL LETTER EW
+10C80; C; 10CC0; # OLD HUNGARIAN CAPITAL LETTER A
+10C81; C; 10CC1; # OLD HUNGARIAN CAPITAL LETTER AA
+10C82; C; 10CC2; # OLD HUNGARIAN CAPITAL LETTER EB
+10C83; C; 10CC3; # OLD HUNGARIAN CAPITAL LETTER AMB
+10C84; C; 10CC4; # OLD HUNGARIAN CAPITAL LETTER EC
+10C85; C; 10CC5; # OLD HUNGARIAN CAPITAL LETTER ENC
+10C86; C; 10CC6; # OLD HUNGARIAN CAPITAL LETTER ECS
+10C87; C; 10CC7; # OLD HUNGARIAN CAPITAL LETTER ED
+10C88; C; 10CC8; # OLD HUNGARIAN CAPITAL LETTER AND
+10C89; C; 10CC9; # OLD HUNGARIAN CAPITAL LETTER E
+10C8A; C; 10CCA; # OLD HUNGARIAN CAPITAL LETTER CLOSE E
+10C8B; C; 10CCB; # OLD HUNGARIAN CAPITAL LETTER EE
+10C8C; C; 10CCC; # OLD HUNGARIAN CAPITAL LETTER EF
+10C8D; C; 10CCD; # OLD HUNGARIAN CAPITAL LETTER EG
+10C8E; C; 10CCE; # OLD HUNGARIAN CAPITAL LETTER EGY
+10C8F; C; 10CCF; # OLD HUNGARIAN CAPITAL LETTER EH
+10C90; C; 10CD0; # OLD HUNGARIAN CAPITAL LETTER I
+10C91; C; 10CD1; # OLD HUNGARIAN CAPITAL LETTER II
+10C92; C; 10CD2; # OLD HUNGARIAN CAPITAL LETTER EJ
+10C93; C; 10CD3; # OLD HUNGARIAN CAPITAL LETTER EK
+10C94; C; 10CD4; # OLD HUNGARIAN CAPITAL LETTER AK
+10C95; C; 10CD5; # OLD HUNGARIAN CAPITAL LETTER UNK
+10C96; C; 10CD6; # OLD HUNGARIAN CAPITAL LETTER EL
+10C97; C; 10CD7; # OLD HUNGARIAN CAPITAL LETTER ELY
+10C98; C; 10CD8; # OLD HUNGARIAN CAPITAL LETTER EM
+10C99; C; 10CD9; # OLD HUNGARIAN CAPITAL LETTER EN
+10C9A; C; 10CDA; # OLD HUNGARIAN CAPITAL LETTER ENY
+10C9B; C; 10CDB; # OLD HUNGARIAN CAPITAL LETTER O
+10C9C; C; 10CDC; # OLD HUNGARIAN CAPITAL LETTER OO
+10C9D; C; 10CDD; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE
+10C9E; C; 10CDE; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE
+10C9F; C; 10CDF; # OLD HUNGARIAN CAPITAL LETTER OEE
+10CA0; C; 10CE0; # OLD HUNGARIAN CAPITAL LETTER EP
+10CA1; C; 10CE1; # OLD HUNGARIAN CAPITAL LETTER EMP
+10CA2; C; 10CE2; # OLD HUNGARIAN CAPITAL LETTER ER
+10CA3; C; 10CE3; # OLD HUNGARIAN CAPITAL LETTER SHORT ER
+10CA4; C; 10CE4; # OLD HUNGARIAN CAPITAL LETTER ES
+10CA5; C; 10CE5; # OLD HUNGARIAN CAPITAL LETTER ESZ
+10CA6; C; 10CE6; # OLD HUNGARIAN CAPITAL LETTER ET
+10CA7; C; 10CE7; # OLD HUNGARIAN CAPITAL LETTER ENT
+10CA8; C; 10CE8; # OLD HUNGARIAN CAPITAL LETTER ETY
+10CA9; C; 10CE9; # OLD HUNGARIAN CAPITAL LETTER ECH
+10CAA; C; 10CEA; # OLD HUNGARIAN CAPITAL LETTER U
+10CAB; C; 10CEB; # OLD HUNGARIAN CAPITAL LETTER UU
+10CAC; C; 10CEC; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE
+10CAD; C; 10CED; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE
+10CAE; C; 10CEE; # OLD HUNGARIAN CAPITAL LETTER EV
+10CAF; C; 10CEF; # OLD HUNGARIAN CAPITAL LETTER EZ
+10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS
+10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN
+10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US
+118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA
+118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A
+118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI
+118A3; C; 118C3; # WARANG CITI CAPITAL LETTER YU
+118A4; C; 118C4; # WARANG CITI CAPITAL LETTER YA
+118A5; C; 118C5; # WARANG CITI CAPITAL LETTER YO
+118A6; C; 118C6; # WARANG CITI CAPITAL LETTER II
+118A7; C; 118C7; # WARANG CITI CAPITAL LETTER UU
+118A8; C; 118C8; # WARANG CITI CAPITAL LETTER E
+118A9; C; 118C9; # WARANG CITI CAPITAL LETTER O
+118AA; C; 118CA; # WARANG CITI CAPITAL LETTER ANG
+118AB; C; 118CB; # WARANG CITI CAPITAL LETTER GA
+118AC; C; 118CC; # WARANG CITI CAPITAL LETTER KO
+118AD; C; 118CD; # WARANG CITI CAPITAL LETTER ENY
+118AE; C; 118CE; # WARANG CITI CAPITAL LETTER YUJ
+118AF; C; 118CF; # WARANG CITI CAPITAL LETTER UC
+118B0; C; 118D0; # WARANG CITI CAPITAL LETTER ENN
+118B1; C; 118D1; # WARANG CITI CAPITAL LETTER ODD
+118B2; C; 118D2; # WARANG CITI CAPITAL LETTER TTE
+118B3; C; 118D3; # WARANG CITI CAPITAL LETTER NUNG
+118B4; C; 118D4; # WARANG CITI CAPITAL LETTER DA
+118B5; C; 118D5; # WARANG CITI CAPITAL LETTER AT
+118B6; C; 118D6; # WARANG CITI CAPITAL LETTER AM
+118B7; C; 118D7; # WARANG CITI CAPITAL LETTER BU
+118B8; C; 118D8; # WARANG CITI CAPITAL LETTER PU
+118B9; C; 118D9; # WARANG CITI CAPITAL LETTER HIYO
+118BA; C; 118DA; # WARANG CITI CAPITAL LETTER HOLO
+118BB; C; 118DB; # WARANG CITI CAPITAL LETTER HORR
+118BC; C; 118DC; # WARANG CITI CAPITAL LETTER HAR
+118BD; C; 118DD; # WARANG CITI CAPITAL LETTER SSUU
+118BE; C; 118DE; # WARANG CITI CAPITAL LETTER SII
+118BF; C; 118DF; # WARANG CITI CAPITAL LETTER VIYO
+#
+# EOF
diff --git a/rpython/rlib/unicodedata/CompositionExclusions-8.0.0.txt b/rpython/rlib/unicodedata/CompositionExclusions-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CompositionExclusions-8.0.0.txt
@@ -0,0 +1,206 @@
+# CompositionExclusions-8.0.0.txt
+# Date: 2015-02-19, 00:30:00 GMT [KW, LI]
+#
+# This file lists the characters for the Composition Exclusion Table
+# defined in UAX #15, Unicode Normalization Forms.
+#
+# This file is a normative contributory data file in the
+# Unicode Character Database.
+#
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+#
+# For more information, see
+# http://www.unicode.org/unicode/reports/tr15/#Primary_Exclusion_List_Table
+#
+# For a full derivation of composition exclusions, see the derived property
+# Full_Composition_Exclusion in DerivedNormalizationProps.txt
+#
+
+# ================================================
+# (1) Script Specifics
+#
+# This list of characters cannot be derived from the UnicodeData.txt file.
+# ================================================
+
+0958    #  DEVANAGARI LETTER QA
+0959    #  DEVANAGARI LETTER KHHA
+095A    #  DEVANAGARI LETTER GHHA
+095B    #  DEVANAGARI LETTER ZA
+095C    #  DEVANAGARI LETTER DDDHA
+095D    #  DEVANAGARI LETTER RHA
+095E    #  DEVANAGARI LETTER FA
+095F    #  DEVANAGARI LETTER YYA
+09DC    #  BENGALI LETTER RRA
+09DD    #  BENGALI LETTER RHA
+09DF    #  BENGALI LETTER YYA
+0A33    #  GURMUKHI LETTER LLA
+0A36    #  GURMUKHI LETTER SHA
+0A59    #  GURMUKHI LETTER KHHA
+0A5A    #  GURMUKHI LETTER GHHA
+0A5B    #  GURMUKHI LETTER ZA
+0A5E    #  GURMUKHI LETTER FA
+0B5C    #  ORIYA LETTER RRA
+0B5D    #  ORIYA LETTER RHA
+0F43    #  TIBETAN LETTER GHA
+0F4D    #  TIBETAN LETTER DDHA
+0F52    #  TIBETAN LETTER DHA
+0F57    #  TIBETAN LETTER BHA
+0F5C    #  TIBETAN LETTER DZHA
+0F69    #  TIBETAN LETTER KSSA
+0F76    #  TIBETAN VOWEL SIGN VOCALIC R
+0F78    #  TIBETAN VOWEL SIGN VOCALIC L
+0F93    #  TIBETAN SUBJOINED LETTER GHA
+0F9D    #  TIBETAN SUBJOINED LETTER DDHA
+0FA2    #  TIBETAN SUBJOINED LETTER DHA
+0FA7    #  TIBETAN SUBJOINED LETTER BHA
+0FAC    #  TIBETAN SUBJOINED LETTER DZHA
+0FB9    #  TIBETAN SUBJOINED LETTER KSSA
+FB1D    #  HEBREW LETTER YOD WITH HIRIQ
+FB1F    #  HEBREW LIGATURE YIDDISH YOD YOD PATAH
+FB2A    #  HEBREW LETTER SHIN WITH SHIN DOT
+FB2B    #  HEBREW LETTER SHIN WITH SIN DOT
+FB2C    #  HEBREW LETTER SHIN WITH DAGESH AND SHIN DOT
+FB2D    #  HEBREW LETTER SHIN WITH DAGESH AND SIN DOT
+FB2E    #  HEBREW LETTER ALEF WITH PATAH
+FB2F    #  HEBREW LETTER ALEF WITH QAMATS
+FB30    #  HEBREW LETTER ALEF WITH MAPIQ
+FB31    #  HEBREW LETTER BET WITH DAGESH
+FB32    #  HEBREW LETTER GIMEL WITH DAGESH
+FB33    #  HEBREW LETTER DALET WITH DAGESH
+FB34    #  HEBREW LETTER HE WITH MAPIQ
+FB35    #  HEBREW LETTER VAV WITH DAGESH
+FB36    #  HEBREW LETTER ZAYIN WITH DAGESH
+FB38    #  HEBREW LETTER TET WITH DAGESH
+FB39    #  HEBREW LETTER YOD WITH DAGESH
+FB3A    #  HEBREW LETTER FINAL KAF WITH DAGESH
+FB3B    #  HEBREW LETTER KAF WITH DAGESH
+FB3C    #  HEBREW LETTER LAMED WITH DAGESH
+FB3E    #  HEBREW LETTER MEM WITH DAGESH
+FB40    #  HEBREW LETTER NUN WITH DAGESH
+FB41    #  HEBREW LETTER SAMEKH WITH DAGESH
+FB43    #  HEBREW LETTER FINAL PE WITH DAGESH
+FB44    #  HEBREW LETTER PE WITH DAGESH
+FB46    #  HEBREW LETTER TSADI WITH DAGESH
+FB47    #  HEBREW LETTER QOF WITH DAGESH
+FB48    #  HEBREW LETTER RESH WITH DAGESH
+FB49    #  HEBREW LETTER SHIN WITH DAGESH
+FB4A    #  HEBREW LETTER TAV WITH DAGESH
+FB4B    #  HEBREW LETTER VAV WITH HOLAM
+FB4C    #  HEBREW LETTER BET WITH RAFE
+FB4D    #  HEBREW LETTER KAF WITH RAFE
+FB4E    #  HEBREW LETTER PE WITH RAFE
+
+# Total code points: 67
+
+# ================================================
+# (2) Post Composition Version precomposed characters
+#
+# These characters cannot be derived solely from the UnicodeData.txt file
+# in this version of Unicode.
+#
+# Note that characters added to the standard after the
+# Composition Version and which have canonical decomposition mappings
+# are not automatically added to this list of Post Composition
+# Version precomposed characters.
+# ================================================
+
+2ADC    #  FORKING
+1D15E   #  MUSICAL SYMBOL HALF NOTE
+1D15F   #  MUSICAL SYMBOL QUARTER NOTE
+1D160   #  MUSICAL SYMBOL EIGHTH NOTE
+1D161   #  MUSICAL SYMBOL SIXTEENTH NOTE
+1D162   #  MUSICAL SYMBOL THIRTY-SECOND NOTE
+1D163   #  MUSICAL SYMBOL SIXTY-FOURTH NOTE
+1D164   #  MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE
+1D1BB   #  MUSICAL SYMBOL MINIMA
+1D1BC   #  MUSICAL SYMBOL MINIMA BLACK
+1D1BD   #  MUSICAL SYMBOL SEMIMINIMA WHITE
+1D1BE   #  MUSICAL SYMBOL SEMIMINIMA BLACK
+1D1BF   #  MUSICAL SYMBOL FUSA WHITE
+1D1C0   #  MUSICAL SYMBOL FUSA BLACK
+
+# Total code points: 14
+
+# ================================================
+# (3) Singleton Decompositions
+#
+# These characters can be derived from the UnicodeData.txt file
+# by including all canonically decomposable characters whose
+# canonical decomposition consists of a single character.
+#
+# These characters are simply quoted here for reference.
+# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt
+# ================================================
+
+# 0340..0341       [2] COMBINING GRAVE TONE MARK..COMBINING ACUTE TONE MARK
+# 0343                 COMBINING GREEK KORONIS
+# 0374                 GREEK NUMERAL SIGN
+# 037E                 GREEK QUESTION MARK
+# 0387                 GREEK ANO TELEIA
+# 1F71                 GREEK SMALL LETTER ALPHA WITH OXIA
+# 1F73                 GREEK SMALL LETTER EPSILON WITH OXIA
+# 1F75                 GREEK SMALL LETTER ETA WITH OXIA
+# 1F77                 GREEK SMALL LETTER IOTA WITH OXIA
+# 1F79                 GREEK SMALL LETTER OMICRON WITH OXIA
+# 1F7B                 GREEK SMALL LETTER UPSILON WITH OXIA
+# 1F7D                 GREEK SMALL LETTER OMEGA WITH OXIA
+# 1FBB                 GREEK CAPITAL LETTER ALPHA WITH OXIA
+# 1FBE                 GREEK PROSGEGRAMMENI
+# 1FC9                 GREEK CAPITAL LETTER EPSILON WITH OXIA
+# 1FCB                 GREEK CAPITAL LETTER ETA WITH OXIA
+# 1FD3                 GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+# 1FDB                 GREEK CAPITAL LETTER IOTA WITH OXIA
+# 1FE3                 GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+# 1FEB                 GREEK CAPITAL LETTER UPSILON WITH OXIA
+# 1FEE..1FEF       [2] GREEK DIALYTIKA AND OXIA..GREEK VARIA
+# 1FF9                 GREEK CAPITAL LETTER OMICRON WITH OXIA
+# 1FFB                 GREEK CAPITAL LETTER OMEGA WITH OXIA
+# 1FFD                 GREEK OXIA
+# 2000..2001       [2] EN QUAD..EM QUAD
+# 2126                 OHM SIGN
+# 212A..212B       [2] KELVIN SIGN..ANGSTROM SIGN
+# 2329                 LEFT-POINTING ANGLE BRACKET
+# 232A                 RIGHT-POINTING ANGLE BRACKET
+# F900..FA0D     [270] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA0D
+# FA10                 CJK COMPATIBILITY IDEOGRAPH-FA10
+# FA12                 CJK COMPATIBILITY IDEOGRAPH-FA12
+# FA15..FA1E      [10] CJK COMPATIBILITY IDEOGRAPH-FA15..CJK COMPATIBILITY IDEOGRAPH-FA1E
+# FA20                 CJK COMPATIBILITY IDEOGRAPH-FA20
+# FA22                 CJK COMPATIBILITY IDEOGRAPH-FA22
+# FA25..FA26       [2] CJK COMPATIBILITY IDEOGRAPH-FA25..CJK COMPATIBILITY IDEOGRAPH-FA26
+# FA2A..FA6D      [68] CJK COMPATIBILITY IDEOGRAPH-FA2A..CJK COMPATIBILITY IDEOGRAPH-FA6D
+# FA70..FAD9     [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9
+# 2F800..2FA1D   [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
+
+# Total code points: 1035
+
+# ================================================
+# (4) Non-Starter Decompositions
+#
+# These characters can be derived from the UnicodeData.txt file
+# by including each expanding canonical decomposition
+# (i.e., those which canonically decompose to a sequence
+# of characters instead of a single character), such that:
+#
+# A. The character is not a Starter.
+#
+# OR (inclusive)
+#
+# B. The character's canonical decomposition begins
+# with a character that is not a Starter.
+#
+# Note that a "Starter" is any character with a zero combining class.
+#
+# These characters are simply quoted here for reference.
+# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt
+# ================================================
+
+# 0344                 COMBINING GREEK DIALYTIKA TONOS
+# 0F73                 TIBETAN VOWEL SIGN II
+# 0F75                 TIBETAN VOWEL SIGN UU
+# 0F81                 TIBETAN VOWEL SIGN REVERSED II
+
+# Total code points: 4
+
+# EOF
diff --git a/rpython/rlib/unicodedata/DerivedCoreProperties-8.0.0.txt b/rpython/rlib/unicodedata/DerivedCoreProperties-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/DerivedCoreProperties-8.0.0.txt
@@ -0,0 +1,11029 @@
+# DerivedCoreProperties-8.0.0.txt
+# Date: 2015-03-11, 22:29:21 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+
+# ================================================
+
+# Derived Property: Math
+#  Generated from: Sm + Other_Math
+
+002B          ; Math # Sm       PLUS SIGN
+003C..003E    ; Math # Sm   [3] LESS-THAN SIGN..GREATER-THAN SIGN
+005E          ; Math # Sk       CIRCUMFLEX ACCENT
+007C          ; Math # Sm       VERTICAL LINE
+007E          ; Math # Sm       TILDE
+00AC          ; Math # Sm       NOT SIGN
+00B1          ; Math # Sm       PLUS-MINUS SIGN
+00D7          ; Math # Sm       MULTIPLICATION SIGN
+00F7          ; Math # Sm       DIVISION SIGN
+03D0..03D2    ; Math # L&   [3] GREEK BETA SYMBOL..GREEK UPSILON WITH HOOK SYMBOL
+03D5          ; Math # L&       GREEK PHI SYMBOL
+03F0..03F1    ; Math # L&   [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL
+03F4..03F5    ; Math # L&   [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL
+03F6          ; Math # Sm       GREEK REVERSED LUNATE EPSILON SYMBOL
+0606..0608    ; Math # Sm   [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY
+2016          ; Math # Po       DOUBLE VERTICAL LINE
+2032..2034    ; Math # Po   [3] PRIME..TRIPLE PRIME
+2040          ; Math # Pc       CHARACTER TIE
+2044          ; Math # Sm       FRACTION SLASH
+2052          ; Math # Sm       COMMERCIAL MINUS SIGN
+2061..2064    ; Math # Cf   [4] FUNCTION APPLICATION..INVISIBLE PLUS
+207A..207C    ; Math # Sm   [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN
+207D          ; Math # Ps       SUPERSCRIPT LEFT PARENTHESIS
+207E          ; Math # Pe       SUPERSCRIPT RIGHT PARENTHESIS
+208A..208C    ; Math # Sm   [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN
+208D          ; Math # Ps       SUBSCRIPT LEFT PARENTHESIS
+208E          ; Math # Pe       SUBSCRIPT RIGHT PARENTHESIS
+20D0..20DC    ; Math # Mn  [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE
+20E1          ; Math # Mn       COMBINING LEFT RIGHT ARROW ABOVE
+20E5..20E6    ; Math # Mn   [2] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING DOUBLE VERTICAL STROKE OVERLAY
+20EB..20EF    ; Math # Mn   [5] COMBINING LONG DOUBLE SOLIDUS OVERLAY..COMBINING RIGHT ARROW BELOW
+2102          ; Math # L&       DOUBLE-STRUCK CAPITAL C
+2107          ; Math # L&       EULER CONSTANT
+210A..2113    ; Math # L&  [10] SCRIPT SMALL G..SCRIPT SMALL L
+2115          ; Math # L&       DOUBLE-STRUCK CAPITAL N
+2118          ; Math # Sm       SCRIPT CAPITAL P
+2119..211D    ; Math # L&   [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R
+2124          ; Math # L&       DOUBLE-STRUCK CAPITAL Z
+2128          ; Math # L&       BLACK-LETTER CAPITAL Z
+2129          ; Math # So       TURNED GREEK SMALL LETTER IOTA
+212C..212D    ; Math # L&   [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C
+212F..2131    ; Math # L&   [3] SCRIPT SMALL E..SCRIPT CAPITAL F
+2133..2134    ; Math # L&   [2] SCRIPT CAPITAL M..SCRIPT SMALL O
+2135..2138    ; Math # Lo   [4] ALEF SYMBOL..DALET SYMBOL
+213C..213F    ; Math # L&   [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI
+2140..2144    ; Math # Sm   [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y
+2145..2149    ; Math # L&   [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J
+214B          ; Math # Sm       TURNED AMPERSAND
+2190..2194    ; Math # Sm   [5] LEFTWARDS ARROW..LEFT RIGHT ARROW
+2195..2199    ; Math # So   [5] UP DOWN ARROW..SOUTH WEST ARROW
+219A..219B    ; Math # Sm   [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE
+219C..219F    ; Math # So   [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW
+21A0          ; Math # Sm       RIGHTWARDS TWO HEADED ARROW
+21A1..21A2    ; Math # So   [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL
+21A3          ; Math # Sm       RIGHTWARDS ARROW WITH TAIL
+21A4..21A5    ; Math # So   [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR
+21A6          ; Math # Sm       RIGHTWARDS ARROW FROM BAR
+21A7          ; Math # So       DOWNWARDS ARROW FROM BAR
+21A9..21AD    ; Math # So   [5] LEFTWARDS ARROW WITH HOOK..LEFT RIGHT WAVE ARROW
+21AE          ; Math # Sm       LEFT RIGHT ARROW WITH STROKE
+21B0..21B1    ; Math # So   [2] UPWARDS ARROW WITH TIP LEFTWARDS..UPWARDS ARROW WITH TIP RIGHTWARDS
+21B6..21B7    ; Math # So   [2] ANTICLOCKWISE TOP SEMICIRCLE ARROW..CLOCKWISE TOP SEMICIRCLE ARROW
+21BC..21CD    ; Math # So  [18] LEFTWARDS HARPOON WITH BARB UPWARDS..LEFTWARDS DOUBLE ARROW WITH STROKE
+21CE..21CF    ; Math # Sm   [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE
+21D0..21D1    ; Math # So   [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW
+21D2          ; Math # Sm       RIGHTWARDS DOUBLE ARROW
+21D3          ; Math # So       DOWNWARDS DOUBLE ARROW
+21D4          ; Math # Sm       LEFT RIGHT DOUBLE ARROW
+21D5..21DB    ; Math # So   [7] UP DOWN DOUBLE ARROW..RIGHTWARDS TRIPLE ARROW
+21DD          ; Math # So       RIGHTWARDS SQUIGGLE ARROW
+21E4..21E5    ; Math # So   [2] LEFTWARDS ARROW TO BAR..RIGHTWARDS ARROW TO BAR
+21F4..22FF    ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP
+2308          ; Math # Ps       LEFT CEILING
+2309          ; Math # Pe       RIGHT CEILING
+230A          ; Math # Ps       LEFT FLOOR
+230B          ; Math # Pe       RIGHT FLOOR
+2320..2321    ; Math # Sm   [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL
+237C          ; Math # Sm       RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW
+239B..23B3    ; Math # Sm  [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM
+23B4..23B5    ; Math # So   [2] TOP SQUARE BRACKET..BOTTOM SQUARE BRACKET
+23B7          ; Math # So       RADICAL SYMBOL BOTTOM
+23D0          ; Math # So       VERTICAL LINE EXTENSION
+23DC..23E1    ; Math # Sm   [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET
+23E2          ; Math # So       WHITE TRAPEZIUM
+25A0..25A1    ; Math # So   [2] BLACK SQUARE..WHITE SQUARE
+25AE..25B6    ; Math # So   [9] BLACK VERTICAL RECTANGLE..BLACK RIGHT-POINTING TRIANGLE
+25B7          ; Math # Sm       WHITE RIGHT-POINTING TRIANGLE
+25BC..25C0    ; Math # So   [5] BLACK DOWN-POINTING TRIANGLE..BLACK LEFT-POINTING TRIANGLE
+25C1          ; Math # Sm       WHITE LEFT-POINTING TRIANGLE
+25C6..25C7    ; Math # So   [2] BLACK DIAMOND..WHITE DIAMOND
+25CA..25CB    ; Math # So   [2] LOZENGE..WHITE CIRCLE
+25CF..25D3    ; Math # So   [5] BLACK CIRCLE..CIRCLE WITH UPPER HALF BLACK
+25E2          ; Math # So       BLACK LOWER RIGHT TRIANGLE
+25E4          ; Math # So       BLACK UPPER LEFT TRIANGLE
+25E7..25EC    ; Math # So   [6] SQUARE WITH LEFT HALF BLACK..WHITE UP-POINTING TRIANGLE WITH DOT
+25F8..25FF    ; Math # Sm   [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE
+2605..2606    ; Math # So   [2] BLACK STAR..WHITE STAR
+2640          ; Math # So       FEMALE SIGN
+2642          ; Math # So       MALE SIGN
+2660..2663    ; Math # So   [4] BLACK SPADE SUIT..BLACK CLUB SUIT
+266D..266E    ; Math # So   [2] MUSIC FLAT SIGN..MUSIC NATURAL SIGN
+266F          ; Math # Sm       MUSIC SHARP SIGN
+27C0..27C4    ; Math # Sm   [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET
+27C5          ; Math # Ps       LEFT S-SHAPED BAG DELIMITER
+27C6          ; Math # Pe       RIGHT S-SHAPED BAG DELIMITER
+27C7..27E5    ; Math # Sm  [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK
+27E6          ; Math # Ps       MATHEMATICAL LEFT WHITE SQUARE BRACKET
+27E7          ; Math # Pe       MATHEMATICAL RIGHT WHITE SQUARE BRACKET
+27E8          ; Math # Ps       MATHEMATICAL LEFT ANGLE BRACKET
+27E9          ; Math # Pe       MATHEMATICAL RIGHT ANGLE BRACKET
+27EA          ; Math # Ps       MATHEMATICAL LEFT DOUBLE ANGLE BRACKET
+27EB          ; Math # Pe       MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET
+27EC          ; Math # Ps       MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET
+27ED          ; Math # Pe       MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET
+27EE          ; Math # Ps       MATHEMATICAL LEFT FLATTENED PARENTHESIS
+27EF          ; Math # Pe       MATHEMATICAL RIGHT FLATTENED PARENTHESIS
+27F0..27FF    ; Math # Sm  [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW
+2900..2982    ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON
+2983          ; Math # Ps       LEFT WHITE CURLY BRACKET
+2984          ; Math # Pe       RIGHT WHITE CURLY BRACKET
+2985          ; Math # Ps       LEFT WHITE PARENTHESIS
+2986          ; Math # Pe       RIGHT WHITE PARENTHESIS
+2987          ; Math # Ps       Z NOTATION LEFT IMAGE BRACKET
+2988          ; Math # Pe       Z NOTATION RIGHT IMAGE BRACKET
+2989          ; Math # Ps       Z NOTATION LEFT BINDING BRACKET
+298A          ; Math # Pe       Z NOTATION RIGHT BINDING BRACKET
+298B          ; Math # Ps       LEFT SQUARE BRACKET WITH UNDERBAR
+298C          ; Math # Pe       RIGHT SQUARE BRACKET WITH UNDERBAR
+298D          ; Math # Ps       LEFT SQUARE BRACKET WITH TICK IN TOP CORNER
+298E          ; Math # Pe       RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER
+298F          ; Math # Ps       LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER
+2990          ; Math # Pe       RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER
+2991          ; Math # Ps       LEFT ANGLE BRACKET WITH DOT
+2992          ; Math # Pe       RIGHT ANGLE BRACKET WITH DOT
+2993          ; Math # Ps       LEFT ARC LESS-THAN BRACKET
+2994          ; Math # Pe       RIGHT ARC GREATER-THAN BRACKET
+2995          ; Math # Ps       DOUBLE LEFT ARC GREATER-THAN BRACKET
+2996          ; Math # Pe       DOUBLE RIGHT ARC LESS-THAN BRACKET
+2997          ; Math # Ps       LEFT BLACK TORTOISE SHELL BRACKET
+2998          ; Math # Pe       RIGHT BLACK TORTOISE SHELL BRACKET
+2999..29D7    ; Math # Sm  [63] DOTTED FENCE..BLACK HOURGLASS
+29D8          ; Math # Ps       LEFT WIGGLY FENCE
+29D9          ; Math # Pe       RIGHT WIGGLY FENCE
+29DA          ; Math # Ps       LEFT DOUBLE WIGGLY FENCE
+29DB          ; Math # Pe       RIGHT DOUBLE WIGGLY FENCE
+29DC..29FB    ; Math # Sm  [32] INCOMPLETE INFINITY..TRIPLE PLUS
+29FC          ; Math # Ps       LEFT-POINTING CURVED ANGLE BRACKET
+29FD          ; Math # Pe       RIGHT-POINTING CURVED ANGLE BRACKET
+29FE..2AFF    ; Math # Sm [258] TINY..N-ARY WHITE VERTICAL BAR
+2B30..2B44    ; Math # Sm  [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET
+2B47..2B4C    ; Math # Sm   [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR
+FB29          ; Math # Sm       HEBREW LETTER ALTERNATIVE PLUS SIGN
+FE61          ; Math # Po       SMALL ASTERISK
+FE62          ; Math # Sm       SMALL PLUS SIGN
+FE63          ; Math # Pd       SMALL HYPHEN-MINUS
+FE64..FE66    ; Math # Sm   [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN
+FE68          ; Math # Po       SMALL REVERSE SOLIDUS
+FF0B          ; Math # Sm       FULLWIDTH PLUS SIGN
+FF1C..FF1E    ; Math # Sm   [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN
+FF3C          ; Math # Po       FULLWIDTH REVERSE SOLIDUS
+FF3E          ; Math # Sk       FULLWIDTH CIRCUMFLEX ACCENT
+FF5C          ; Math # Sm       FULLWIDTH VERTICAL LINE
+FF5E          ; Math # Sm       FULLWIDTH TILDE
+FFE2          ; Math # Sm       FULLWIDTH NOT SIGN
+FFE9..FFEC    ; Math # Sm   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW
+1D400..1D454  ; Math # L&  [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G
+1D456..1D49C  ; Math # L&  [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A
+1D49E..1D49F  ; Math # L&   [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D
+1D4A2         ; Math # L&       MATHEMATICAL SCRIPT CAPITAL G
+1D4A5..1D4A6  ; Math # L&   [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K
+1D4A9..1D4AC  ; Math # L&   [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q
+1D4AE..1D4B9  ; Math # L&  [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D
+1D4BB         ; Math # L&       MATHEMATICAL SCRIPT SMALL F
+1D4BD..1D4C3  ; Math # L&   [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N
+1D4C5..1D505  ; Math # L&  [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B
+1D507..1D50A  ; Math # L&   [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G
+1D50D..1D514  ; Math # L&   [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q
+1D516..1D51C  ; Math # L&   [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y
+1D51E..1D539  ; Math # L&  [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B
+1D53B..1D53E  ; Math # L&   [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G
+1D540..1D544  ; Math # L&   [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M
+1D546         ; Math # L&       MATHEMATICAL DOUBLE-STRUCK CAPITAL O
+1D54A..1D550  ; Math # L&   [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y
+1D552..1D6A5  ; Math # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J
+1D6A8..1D6C0  ; Math # L&  [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA
+1D6C1         ; Math # Sm       MATHEMATICAL BOLD NABLA
+1D6C2..1D6DA  ; Math # L&  [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA
+1D6DB         ; Math # Sm       MATHEMATICAL BOLD PARTIAL DIFFERENTIAL
+1D6DC..1D6FA  ; Math # L&  [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA
+1D6FB         ; Math # Sm       MATHEMATICAL ITALIC NABLA
+1D6FC..1D714  ; Math # L&  [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA
+1D715         ; Math # Sm       MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL
+1D716..1D734  ; Math # L&  [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA
+1D735         ; Math # Sm       MATHEMATICAL BOLD ITALIC NABLA
+1D736..1D74E  ; Math # L&  [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA
+1D74F         ; Math # Sm       MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL
+1D750..1D76E  ; Math # L&  [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA
+1D76F         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD NABLA
+1D770..1D788  ; Math # L&  [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA
+1D789         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL
+1D78A..1D7A8  ; Math # L&  [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA
+1D7A9         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA
+1D7AA..1D7C2  ; Math # L&  [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA
+1D7C3         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL
+1D7C4..1D7CB  ; Math # L&   [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA
+1D7CE..1D7FF  ; Math # Nd  [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE
+1EE00..1EE03  ; Math # Lo   [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL
+1EE05..1EE1F  ; Math # Lo  [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF
+1EE21..1EE22  ; Math # Lo   [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM
+1EE24         ; Math # Lo       ARABIC MATHEMATICAL INITIAL HEH
+1EE27         ; Math # Lo       ARABIC MATHEMATICAL INITIAL HAH
+1EE29..1EE32  ; Math # Lo  [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF
+1EE34..1EE37  ; Math # Lo   [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH
+1EE39         ; Math # Lo       ARABIC MATHEMATICAL INITIAL DAD
+1EE3B         ; Math # Lo       ARABIC MATHEMATICAL INITIAL GHAIN
+1EE42         ; Math # Lo       ARABIC MATHEMATICAL TAILED JEEM
+1EE47         ; Math # Lo       ARABIC MATHEMATICAL TAILED HAH
+1EE49         ; Math # Lo       ARABIC MATHEMATICAL TAILED YEH
+1EE4B         ; Math # Lo       ARABIC MATHEMATICAL TAILED LAM
+1EE4D..1EE4F  ; Math # Lo   [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN
+1EE51..1EE52  ; Math # Lo   [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF
+1EE54         ; Math # Lo       ARABIC MATHEMATICAL TAILED SHEEN
+1EE57         ; Math # Lo       ARABIC MATHEMATICAL TAILED KHAH
+1EE59         ; Math # Lo       ARABIC MATHEMATICAL TAILED DAD
+1EE5B         ; Math # Lo       ARABIC MATHEMATICAL TAILED GHAIN
+1EE5D         ; Math # Lo       ARABIC MATHEMATICAL TAILED DOTLESS NOON
+1EE5F         ; Math # Lo       ARABIC MATHEMATICAL TAILED DOTLESS QAF
+1EE61..1EE62  ; Math # Lo   [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM
+1EE64         ; Math # Lo       ARABIC MATHEMATICAL STRETCHED HEH
+1EE67..1EE6A  ; Math # Lo   [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF
+1EE6C..1EE72  ; Math # Lo   [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF
+1EE74..1EE77  ; Math # Lo   [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH
+1EE79..1EE7C  ; Math # Lo   [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH
+1EE7E         ; Math # Lo       ARABIC MATHEMATICAL STRETCHED DOTLESS FEH
+1EE80..1EE89  ; Math # Lo  [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH
+1EE8B..1EE9B  ; Math # Lo  [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN
+1EEA1..1EEA3  ; Math # Lo   [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL
+1EEA5..1EEA9  ; Math # Lo   [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH
+1EEAB..1EEBB  ; Math # Lo  [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN
+1EEF0..1EEF1  ; Math # Sm   [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL
+
+# Total code points: 2310
+
+# ================================================
+
+# Derived Property: Alphabetic
+#  Generated from: Uppercase + Lowercase + Lt + Lm + Lo + Nl + Other_Alphabetic
+
+0041..005A    ; Alphabetic # L&  [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z
+0061..007A    ; Alphabetic # L&  [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z
+00AA          ; Alphabetic # Lo       FEMININE ORDINAL INDICATOR
+00B5          ; Alphabetic # L&       MICRO SIGN
+00BA          ; Alphabetic # Lo       MASCULINE ORDINAL INDICATOR
+00C0..00D6    ; Alphabetic # L&  [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8..00F6    ; Alphabetic # L&  [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS
+00F8..01BA    ; Alphabetic # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL
+01BB          ; Alphabetic # Lo       LATIN LETTER TWO WITH STROKE
+01BC..01BF    ; Alphabetic # L&   [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN
+01C0..01C3    ; Alphabetic # Lo   [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK
+01C4..0293    ; Alphabetic # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL
+0294          ; Alphabetic # Lo       LATIN LETTER GLOTTAL STOP
+0295..02AF    ; Alphabetic # L&  [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL
+02B0..02C1    ; Alphabetic # Lm  [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP
+02C6..02D1    ; Alphabetic # Lm  [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON
+02E0..02E4    ; Alphabetic # Lm   [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP
+02EC          ; Alphabetic # Lm       MODIFIER LETTER VOICING
+02EE          ; Alphabetic # Lm       MODIFIER LETTER DOUBLE APOSTROPHE
+0345          ; Alphabetic # Mn       COMBINING GREEK YPOGEGRAMMENI
+0370..0373    ; Alphabetic # L&   [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI
+0374          ; Alphabetic # Lm       GREEK NUMERAL SIGN
+0376..0377    ; Alphabetic # L&   [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA
+037A          ; Alphabetic # Lm       GREEK YPOGEGRAMMENI
+037B..037D    ; Alphabetic # L&   [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL
+037F          ; Alphabetic # L&       GREEK CAPITAL LETTER YOT
+0386          ; Alphabetic # L&       GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388..038A    ; Alphabetic # L&   [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS
+038C          ; Alphabetic # L&       GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E..03A1    ; Alphabetic # L&  [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO
+03A3..03F5    ; Alphabetic # L&  [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL
+03F7..0481    ; Alphabetic # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA
+048A..052F    ; Alphabetic # L& [166] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EL WITH DESCENDER
+0531..0556    ; Alphabetic # L&  [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH
+0559          ; Alphabetic # Lm       ARMENIAN MODIFIER LETTER LEFT HALF RING
+0561..0587    ; Alphabetic # L&  [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN
+05B0..05BD    ; Alphabetic # Mn  [14] HEBREW POINT SHEVA..HEBREW POINT METEG
+05BF          ; Alphabetic # Mn       HEBREW POINT RAFE
+05C1..05C2    ; Alphabetic # Mn   [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT
+05C4..05C5    ; Alphabetic # Mn   [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT
+05C7          ; Alphabetic # Mn       HEBREW POINT QAMATS QATAN
+05D0..05EA    ; Alphabetic # Lo  [27] HEBREW LETTER ALEF..HEBREW LETTER TAV
+05F0..05F2    ; Alphabetic # Lo   [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD
+0610..061A    ; Alphabetic # Mn  [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA
+0620..063F    ; Alphabetic # Lo  [32] ARABIC LETTER KASHMIRI YEH..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE
+0640          ; Alphabetic # Lm       ARABIC TATWEEL
+0641..064A    ; Alphabetic # Lo  [10] ARABIC LETTER FEH..ARABIC LETTER YEH
+064B..0657    ; Alphabetic # Mn  [13] ARABIC FATHATAN..ARABIC INVERTED DAMMA
+0659..065F    ; Alphabetic # Mn   [7] ARABIC ZWARAKAY..ARABIC WAVY HAMZA BELOW
+066E..066F    ; Alphabetic # Lo   [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF
+0670          ; Alphabetic # Mn       ARABIC LETTER SUPERSCRIPT ALEF
+0671..06D3    ; Alphabetic # Lo  [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE
+06D5          ; Alphabetic # Lo       ARABIC LETTER AE
+06D6..06DC    ; Alphabetic # Mn   [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN
+06E1..06E4    ; Alphabetic # Mn   [4] ARABIC SMALL HIGH DOTLESS HEAD OF KHAH..ARABIC SMALL HIGH MADDA
+06E5..06E6    ; Alphabetic # Lm   [2] ARABIC SMALL WAW..ARABIC SMALL YEH
+06E7..06E8    ; Alphabetic # Mn   [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON
+06ED          ; Alphabetic # Mn       ARABIC SMALL LOW MEEM
+06EE..06EF    ; Alphabetic # Lo   [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V
+06FA..06FC    ; Alphabetic # Lo   [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW
+06FF          ; Alphabetic # Lo       ARABIC LETTER HEH WITH INVERTED V
+0710          ; Alphabetic # Lo       SYRIAC LETTER ALAPH
+0711          ; Alphabetic # Mn       SYRIAC LETTER SUPERSCRIPT ALAPH
+0712..072F    ; Alphabetic # Lo  [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH
+0730..073F    ; Alphabetic # Mn  [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA
+074D..07A5    ; Alphabetic # Lo  [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU
+07A6..07B0    ; Alphabetic # Mn  [11] THAANA ABAFILI..THAANA SUKUN
+07B1          ; Alphabetic # Lo       THAANA LETTER NAA
+07CA..07EA    ; Alphabetic # Lo  [33] NKO LETTER A..NKO LETTER JONA RA
+07F4..07F5    ; Alphabetic # Lm   [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE
+07FA          ; Alphabetic # Lm       NKO LAJANYALAN
+0800..0815    ; Alphabetic # Lo  [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF
+0816..0817    ; Alphabetic # Mn   [2] SAMARITAN MARK IN..SAMARITAN MARK IN-ALAF
+081A          ; Alphabetic # Lm       SAMARITAN MODIFIER LETTER EPENTHETIC YUT
+081B..0823    ; Alphabetic # Mn   [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A
+0824          ; Alphabetic # Lm       SAMARITAN MODIFIER LETTER SHORT A
+0825..0827    ; Alphabetic # Mn   [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U
+0828          ; Alphabetic # Lm       SAMARITAN MODIFIER LETTER I
+0829..082C    ; Alphabetic # Mn   [4] SAMARITAN VOWEL SIGN LONG I..SAMARITAN VOWEL SIGN SUKUN
+0840..0858    ; Alphabetic # Lo  [25] MANDAIC LETTER HALQA..MANDAIC LETTER AIN
+08A0..08B4    ; Alphabetic # Lo  [21] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER KAF WITH DOT BELOW
+08E3..08E9    ; Alphabetic # Mn   [7] ARABIC TURNED DAMMA BELOW..ARABIC CURLY KASRATAN
+08F0..0902    ; Alphabetic # Mn  [19] ARABIC OPEN FATHATAN..DEVANAGARI SIGN ANUSVARA
+0903          ; Alphabetic # Mc       DEVANAGARI SIGN VISARGA
+0904..0939    ; Alphabetic # Lo  [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA
+093A          ; Alphabetic # Mn       DEVANAGARI VOWEL SIGN OE
+093B          ; Alphabetic # Mc       DEVANAGARI VOWEL SIGN OOE
+093D          ; Alphabetic # Lo       DEVANAGARI SIGN AVAGRAHA
+093E..0940    ; Alphabetic # Mc   [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II
+0941..0948    ; Alphabetic # Mn   [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI
+0949..094C    ; Alphabetic # Mc   [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU
+094E..094F    ; Alphabetic # Mc   [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW
+0950          ; Alphabetic # Lo       DEVANAGARI OM
+0955..0957    ; Alphabetic # Mn   [3] DEVANAGARI VOWEL SIGN CANDRA LONG E..DEVANAGARI VOWEL SIGN UUE
+0958..0961    ; Alphabetic # Lo  [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL
+0962..0963    ; Alphabetic # Mn   [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL
+0971          ; Alphabetic # Lm       DEVANAGARI SIGN HIGH SPACING DOT
+0972..0980    ; Alphabetic # Lo  [15] DEVANAGARI LETTER CANDRA A..BENGALI ANJI
+0981          ; Alphabetic # Mn       BENGALI SIGN CANDRABINDU
+0982..0983    ; Alphabetic # Mc   [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA
+0985..098C    ; Alphabetic # Lo   [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L
+098F..0990    ; Alphabetic # Lo   [2] BENGALI LETTER E..BENGALI LETTER AI
+0993..09A8    ; Alphabetic # Lo  [22] BENGALI LETTER O..BENGALI LETTER NA
+09AA..09B0    ; Alphabetic # Lo   [7] BENGALI LETTER PA..BENGALI LETTER RA

From pypy.commits at gmail.com  Mon Jan  9 12:57:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 09 Jan 2017 09:57:38 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Remove old file
Message-ID: <5873cf12.6a5cc20a.d0b08.8c96@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89450:c1e95bbad178
Date: 2017-01-09 18:57 +0100
http://bitbucket.org/pypy/pypy/changeset/c1e95bbad178/

Log:	Remove old file

diff too long, truncating to 2000 out of 9248 lines

diff --git a/pypy/module/unicodedata/DerivedCoreProperties-5.2.0.txt b/pypy/module/unicodedata/DerivedCoreProperties-5.2.0.txt
deleted file mode 100644
--- a/pypy/module/unicodedata/DerivedCoreProperties-5.2.0.txt
+++ /dev/null
@@ -1,9243 +0,0 @@
-# DerivedCoreProperties-5.2.0.txt
-# Date: 2009-08-26, 00:45:22 GMT [MD]
-#
-# Unicode Character Database
-# Copyright (c) 1991-2009 Unicode, Inc.
-# For terms of use, see http://www.unicode.org/terms_of_use.html
-# For documentation, see http://www.unicode.org/reports/tr44/
-
-# ================================================
-
-# Derived Property: Math
-#  Generated from: Sm + Other_Math
-
-002B          ; Math # Sm       PLUS SIGN
-003C..003E    ; Math # Sm   [3] LESS-THAN SIGN..GREATER-THAN SIGN
-005E          ; Math # Sk       CIRCUMFLEX ACCENT
-007C          ; Math # Sm       VERTICAL LINE
-007E          ; Math # Sm       TILDE
-00AC          ; Math # Sm       NOT SIGN
-00B1          ; Math # Sm       PLUS-MINUS SIGN
-00D7          ; Math # Sm       MULTIPLICATION SIGN
-00F7          ; Math # Sm       DIVISION SIGN
-03D0..03D2    ; Math # L&   [3] GREEK BETA SYMBOL..GREEK UPSILON WITH HOOK SYMBOL
-03D5          ; Math # L&       GREEK PHI SYMBOL
-03F0..03F1    ; Math # L&   [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL
-03F4..03F5    ; Math # L&   [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL
-03F6          ; Math # Sm       GREEK REVERSED LUNATE EPSILON SYMBOL
-0606..0608    ; Math # Sm   [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY
-2016          ; Math # Po       DOUBLE VERTICAL LINE
-2032..2034    ; Math # Po   [3] PRIME..TRIPLE PRIME
-2040          ; Math # Pc       CHARACTER TIE
-2044          ; Math # Sm       FRACTION SLASH
-2052          ; Math # Sm       COMMERCIAL MINUS SIGN
-2061..2064    ; Math # Cf   [4] FUNCTION APPLICATION..INVISIBLE PLUS
-207A..207C    ; Math # Sm   [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN
-207D          ; Math # Ps       SUPERSCRIPT LEFT PARENTHESIS
-207E          ; Math # Pe       SUPERSCRIPT RIGHT PARENTHESIS
-208A..208C    ; Math # Sm   [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN
-208D          ; Math # Ps       SUBSCRIPT LEFT PARENTHESIS
-208E          ; Math # Pe       SUBSCRIPT RIGHT PARENTHESIS
-20D0..20DC    ; Math # Mn  [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE
-20E1          ; Math # Mn       COMBINING LEFT RIGHT ARROW ABOVE
-20E5..20E6    ; Math # Mn   [2] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING DOUBLE VERTICAL STROKE OVERLAY
-20EB..20EF    ; Math # Mn   [5] COMBINING LONG DOUBLE SOLIDUS OVERLAY..COMBINING RIGHT ARROW BELOW
-2102          ; Math # L&       DOUBLE-STRUCK CAPITAL C
-210A..2113    ; Math # L&  [10] SCRIPT SMALL G..SCRIPT SMALL L
-2115          ; Math # L&       DOUBLE-STRUCK CAPITAL N
-2119..211D    ; Math # L&   [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R
-2124          ; Math # L&       DOUBLE-STRUCK CAPITAL Z
-2128          ; Math # L&       BLACK-LETTER CAPITAL Z
-2129          ; Math # So       TURNED GREEK SMALL LETTER IOTA
-212C..212D    ; Math # L&   [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C
-212F..2131    ; Math # L&   [3] SCRIPT SMALL E..SCRIPT CAPITAL F
-2133..2134    ; Math # L&   [2] SCRIPT CAPITAL M..SCRIPT SMALL O
-2135..2138    ; Math # Lo   [4] ALEF SYMBOL..DALET SYMBOL
-213C..213F    ; Math # L&   [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI
-2140..2144    ; Math # Sm   [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y
-2145..2149    ; Math # L&   [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J
-214B          ; Math # Sm       TURNED AMPERSAND
-2190..2194    ; Math # Sm   [5] LEFTWARDS ARROW..LEFT RIGHT ARROW
-2195..2199    ; Math # So   [5] UP DOWN ARROW..SOUTH WEST ARROW
-219A..219B    ; Math # Sm   [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE
-219C..219F    ; Math # So   [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW
-21A0          ; Math # Sm       RIGHTWARDS TWO HEADED ARROW
-21A1..21A2    ; Math # So   [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL
-21A3          ; Math # Sm       RIGHTWARDS ARROW WITH TAIL
-21A4..21A5    ; Math # So   [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR
-21A6          ; Math # Sm       RIGHTWARDS ARROW FROM BAR
-21A7          ; Math # So       DOWNWARDS ARROW FROM BAR
-21A9..21AD    ; Math # So   [5] LEFTWARDS ARROW WITH HOOK..LEFT RIGHT WAVE ARROW
-21AE          ; Math # Sm       LEFT RIGHT ARROW WITH STROKE
-21B0..21B1    ; Math # So   [2] UPWARDS ARROW WITH TIP LEFTWARDS..UPWARDS ARROW WITH TIP RIGHTWARDS
-21B6..21B7    ; Math # So   [2] ANTICLOCKWISE TOP SEMICIRCLE ARROW..CLOCKWISE TOP SEMICIRCLE ARROW
-21BC..21CD    ; Math # So  [18] LEFTWARDS HARPOON WITH BARB UPWARDS..LEFTWARDS DOUBLE ARROW WITH STROKE
-21CE..21CF    ; Math # Sm   [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE
-21D0..21D1    ; Math # So   [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW
-21D2          ; Math # Sm       RIGHTWARDS DOUBLE ARROW
-21D3          ; Math # So       DOWNWARDS DOUBLE ARROW
-21D4          ; Math # Sm       LEFT RIGHT DOUBLE ARROW
-21D5..21DB    ; Math # So   [7] UP DOWN DOUBLE ARROW..RIGHTWARDS TRIPLE ARROW
-21DD          ; Math # So       RIGHTWARDS SQUIGGLE ARROW
-21E4..21E5    ; Math # So   [2] LEFTWARDS ARROW TO BAR..RIGHTWARDS ARROW TO BAR
-21F4..22FF    ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP
-2308..230B    ; Math # Sm   [4] LEFT CEILING..RIGHT FLOOR
-2320..2321    ; Math # Sm   [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL
-237C          ; Math # Sm       RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW
-239B..23B3    ; Math # Sm  [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM
-23B4..23B5    ; Math # So   [2] TOP SQUARE BRACKET..BOTTOM SQUARE BRACKET
-23B7          ; Math # So       RADICAL SYMBOL BOTTOM
-23D0          ; Math # So       VERTICAL LINE EXTENSION
-23DC..23E1    ; Math # Sm   [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET
-23E2          ; Math # So       WHITE TRAPEZIUM
-25A0..25A1    ; Math # So   [2] BLACK SQUARE..WHITE SQUARE
-25AE..25B6    ; Math # So   [9] BLACK VERTICAL RECTANGLE..BLACK RIGHT-POINTING TRIANGLE
-25B7          ; Math # Sm       WHITE RIGHT-POINTING TRIANGLE
-25BC..25C0    ; Math # So   [5] BLACK DOWN-POINTING TRIANGLE..BLACK LEFT-POINTING TRIANGLE
-25C1          ; Math # Sm       WHITE LEFT-POINTING TRIANGLE
-25C6..25C7    ; Math # So   [2] BLACK DIAMOND..WHITE DIAMOND
-25CA..25CB    ; Math # So   [2] LOZENGE..WHITE CIRCLE
-25CF..25D3    ; Math # So   [5] BLACK CIRCLE..CIRCLE WITH UPPER HALF BLACK
-25E2          ; Math # So       BLACK LOWER RIGHT TRIANGLE
-25E4          ; Math # So       BLACK UPPER LEFT TRIANGLE
-25E7..25EC    ; Math # So   [6] SQUARE WITH LEFT HALF BLACK..WHITE UP-POINTING TRIANGLE WITH DOT
-25F8..25FF    ; Math # Sm   [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE
-2605..2606    ; Math # So   [2] BLACK STAR..WHITE STAR
-2640          ; Math # So       FEMALE SIGN
-2642          ; Math # So       MALE SIGN
-2660..2663    ; Math # So   [4] BLACK SPADE SUIT..BLACK CLUB SUIT
-266D..266E    ; Math # So   [2] MUSIC FLAT SIGN..MUSIC NATURAL SIGN
-266F          ; Math # Sm       MUSIC SHARP SIGN
-27C0..27C4    ; Math # Sm   [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET
-27C5          ; Math # Ps       LEFT S-SHAPED BAG DELIMITER
-27C6          ; Math # Pe       RIGHT S-SHAPED BAG DELIMITER
-27C7..27CA    ; Math # Sm   [4] OR WITH DOT INSIDE..VERTICAL BAR WITH HORIZONTAL STROKE
-27CC          ; Math # Sm       LONG DIVISION
-27D0..27E5    ; Math # Sm  [22] WHITE DIAMOND WITH CENTRED DOT..WHITE SQUARE WITH RIGHTWARDS TICK
-27E6          ; Math # Ps       MATHEMATICAL LEFT WHITE SQUARE BRACKET
-27E7          ; Math # Pe       MATHEMATICAL RIGHT WHITE SQUARE BRACKET
-27E8          ; Math # Ps       MATHEMATICAL LEFT ANGLE BRACKET
-27E9          ; Math # Pe       MATHEMATICAL RIGHT ANGLE BRACKET
-27EA          ; Math # Ps       MATHEMATICAL LEFT DOUBLE ANGLE BRACKET
-27EB          ; Math # Pe       MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET
-27EC          ; Math # Ps       MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET
-27ED          ; Math # Pe       MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET
-27EE          ; Math # Ps       MATHEMATICAL LEFT FLATTENED PARENTHESIS
-27EF          ; Math # Pe       MATHEMATICAL RIGHT FLATTENED PARENTHESIS
-27F0..27FF    ; Math # Sm  [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW
-2900..2982    ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON
-2983          ; Math # Ps       LEFT WHITE CURLY BRACKET
-2984          ; Math # Pe       RIGHT WHITE CURLY BRACKET
-2985          ; Math # Ps       LEFT WHITE PARENTHESIS
-2986          ; Math # Pe       RIGHT WHITE PARENTHESIS
-2987          ; Math # Ps       Z NOTATION LEFT IMAGE BRACKET
-2988          ; Math # Pe       Z NOTATION RIGHT IMAGE BRACKET
-2989          ; Math # Ps       Z NOTATION LEFT BINDING BRACKET
-298A          ; Math # Pe       Z NOTATION RIGHT BINDING BRACKET
-298B          ; Math # Ps       LEFT SQUARE BRACKET WITH UNDERBAR
-298C          ; Math # Pe       RIGHT SQUARE BRACKET WITH UNDERBAR
-298D          ; Math # Ps       LEFT SQUARE BRACKET WITH TICK IN TOP CORNER
-298E          ; Math # Pe       RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER
-298F          ; Math # Ps       LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER
-2990          ; Math # Pe       RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER
-2991          ; Math # Ps       LEFT ANGLE BRACKET WITH DOT
-2992          ; Math # Pe       RIGHT ANGLE BRACKET WITH DOT
-2993          ; Math # Ps       LEFT ARC LESS-THAN BRACKET
-2994          ; Math # Pe       RIGHT ARC GREATER-THAN BRACKET
-2995          ; Math # Ps       DOUBLE LEFT ARC GREATER-THAN BRACKET
-2996          ; Math # Pe       DOUBLE RIGHT ARC LESS-THAN BRACKET
-2997          ; Math # Ps       LEFT BLACK TORTOISE SHELL BRACKET
-2998          ; Math # Pe       RIGHT BLACK TORTOISE SHELL BRACKET
-2999..29D7    ; Math # Sm  [63] DOTTED FENCE..BLACK HOURGLASS
-29D8          ; Math # Ps       LEFT WIGGLY FENCE
-29D9          ; Math # Pe       RIGHT WIGGLY FENCE
-29DA          ; Math # Ps       LEFT DOUBLE WIGGLY FENCE
-29DB          ; Math # Pe       RIGHT DOUBLE WIGGLY FENCE
-29DC..29FB    ; Math # Sm  [32] INCOMPLETE INFINITY..TRIPLE PLUS
-29FC          ; Math # Ps       LEFT-POINTING CURVED ANGLE BRACKET
-29FD          ; Math # Pe       RIGHT-POINTING CURVED ANGLE BRACKET
-29FE..2AFF    ; Math # Sm [258] TINY..N-ARY WHITE VERTICAL BAR
-2B30..2B44    ; Math # Sm  [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET
-2B47..2B4C    ; Math # Sm   [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR
-FB29          ; Math # Sm       HEBREW LETTER ALTERNATIVE PLUS SIGN
-FE61          ; Math # Po       SMALL ASTERISK
-FE62          ; Math # Sm       SMALL PLUS SIGN
-FE63          ; Math # Pd       SMALL HYPHEN-MINUS
-FE64..FE66    ; Math # Sm   [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN
-FE68          ; Math # Po       SMALL REVERSE SOLIDUS
-FF0B          ; Math # Sm       FULLWIDTH PLUS SIGN
-FF1C..FF1E    ; Math # Sm   [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN
-FF3C          ; Math # Po       FULLWIDTH REVERSE SOLIDUS
-FF3E          ; Math # Sk       FULLWIDTH CIRCUMFLEX ACCENT
-FF5C          ; Math # Sm       FULLWIDTH VERTICAL LINE
-FF5E          ; Math # Sm       FULLWIDTH TILDE
-FFE2          ; Math # Sm       FULLWIDTH NOT SIGN
-FFE9..FFEC    ; Math # Sm   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW
-1D400..1D454  ; Math # L&  [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G
-1D456..1D49C  ; Math # L&  [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A
-1D49E..1D49F  ; Math # L&   [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D
-1D4A2         ; Math # L&       MATHEMATICAL SCRIPT CAPITAL G
-1D4A5..1D4A6  ; Math # L&   [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K
-1D4A9..1D4AC  ; Math # L&   [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q
-1D4AE..1D4B9  ; Math # L&  [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D
-1D4BB         ; Math # L&       MATHEMATICAL SCRIPT SMALL F
-1D4BD..1D4C3  ; Math # L&   [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N
-1D4C5..1D505  ; Math # L&  [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B
-1D507..1D50A  ; Math # L&   [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G
-1D50D..1D514  ; Math # L&   [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q
-1D516..1D51C  ; Math # L&   [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y
-1D51E..1D539  ; Math # L&  [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B
-1D53B..1D53E  ; Math # L&   [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G
-1D540..1D544  ; Math # L&   [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M
-1D546         ; Math # L&       MATHEMATICAL DOUBLE-STRUCK CAPITAL O
-1D54A..1D550  ; Math # L&   [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y
-1D552..1D6A5  ; Math # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J
-1D6A8..1D6C0  ; Math # L&  [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA
-1D6C1         ; Math # Sm       MATHEMATICAL BOLD NABLA
-1D6C2..1D6DA  ; Math # L&  [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA
-1D6DB         ; Math # Sm       MATHEMATICAL BOLD PARTIAL DIFFERENTIAL
-1D6DC..1D6FA  ; Math # L&  [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA
-1D6FB         ; Math # Sm       MATHEMATICAL ITALIC NABLA
-1D6FC..1D714  ; Math # L&  [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA
-1D715         ; Math # Sm       MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL
-1D716..1D734  ; Math # L&  [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA
-1D735         ; Math # Sm       MATHEMATICAL BOLD ITALIC NABLA
-1D736..1D74E  ; Math # L&  [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA
-1D74F         ; Math # Sm       MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL
-1D750..1D76E  ; Math # L&  [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA
-1D76F         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD NABLA
-1D770..1D788  ; Math # L&  [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA
-1D789         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL
-1D78A..1D7A8  ; Math # L&  [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA
-1D7A9         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA
-1D7AA..1D7C2  ; Math # L&  [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA
-1D7C3         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL
-1D7C4..1D7CB  ; Math # L&   [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA
-1D7CE..1D7FF  ; Math # Nd  [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE
-
-# Total code points: 2161
-
-# ================================================
-
-# Derived Property: Alphabetic
-#  Generated from: Lu+Ll+Lt+Lm+Lo+Nl + Other_Alphabetic
-
-0041..005A    ; Alphabetic # L&  [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z
-0061..007A    ; Alphabetic # L&  [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z
-00AA          ; Alphabetic # L&       FEMININE ORDINAL INDICATOR
-00B5          ; Alphabetic # L&       MICRO SIGN
-00BA          ; Alphabetic # L&       MASCULINE ORDINAL INDICATOR
-00C0..00D6    ; Alphabetic # L&  [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS
-00D8..00F6    ; Alphabetic # L&  [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS
-00F8..01BA    ; Alphabetic # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL
-01BB          ; Alphabetic # Lo       LATIN LETTER TWO WITH STROKE
-01BC..01BF    ; Alphabetic # L&   [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN
-01C0..01C3    ; Alphabetic # Lo   [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK
-01C4..0293    ; Alphabetic # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL
-0294          ; Alphabetic # Lo       LATIN LETTER GLOTTAL STOP
-0295..02AF    ; Alphabetic # L&  [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL
-02B0..02C1    ; Alphabetic # Lm  [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP
-02C6..02D1    ; Alphabetic # Lm  [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON
-02E0..02E4    ; Alphabetic # Lm   [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP
-02EC          ; Alphabetic # Lm       MODIFIER LETTER VOICING
-02EE          ; Alphabetic # Lm       MODIFIER LETTER DOUBLE APOSTROPHE
-0345          ; Alphabetic # Mn       COMBINING GREEK YPOGEGRAMMENI
-0370..0373    ; Alphabetic # L&   [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI
-0374          ; Alphabetic # Lm       GREEK NUMERAL SIGN
-0376..0377    ; Alphabetic # L&   [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA
-037A          ; Alphabetic # Lm       GREEK YPOGEGRAMMENI
-037B..037D    ; Alphabetic # L&   [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL
-0386          ; Alphabetic # L&       GREEK CAPITAL LETTER ALPHA WITH TONOS
-0388..038A    ; Alphabetic # L&   [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS
-038C          ; Alphabetic # L&       GREEK CAPITAL LETTER OMICRON WITH TONOS
-038E..03A1    ; Alphabetic # L&  [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO
-03A3..03F5    ; Alphabetic # L&  [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL
-03F7..0481    ; Alphabetic # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA
-048A..0525    ; Alphabetic # L& [156] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER PE WITH DESCENDER
-0531..0556    ; Alphabetic # L&  [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH
-0559          ; Alphabetic # Lm       ARMENIAN MODIFIER LETTER LEFT HALF RING
-0561..0587    ; Alphabetic # L&  [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN
-05B0..05BD    ; Alphabetic # Mn  [14] HEBREW POINT SHEVA..HEBREW POINT METEG
-05BF          ; Alphabetic # Mn       HEBREW POINT RAFE
-05C1..05C2    ; Alphabetic # Mn   [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT
-05C4..05C5    ; Alphabetic # Mn   [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT
-05C7          ; Alphabetic # Mn       HEBREW POINT QAMATS QATAN
-05D0..05EA    ; Alphabetic # Lo  [27] HEBREW LETTER ALEF..HEBREW LETTER TAV
-05F0..05F2    ; Alphabetic # Lo   [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD
-0610..061A    ; Alphabetic # Mn  [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA
-0621..063F    ; Alphabetic # Lo  [31] ARABIC LETTER HAMZA..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE
-0640          ; Alphabetic # Lm       ARABIC TATWEEL
-0641..064A    ; Alphabetic # Lo  [10] ARABIC LETTER FEH..ARABIC LETTER YEH
-064B..0657    ; Alphabetic # Mn  [13] ARABIC FATHATAN..ARABIC INVERTED DAMMA
-0659..065E    ; Alphabetic # Mn   [6] ARABIC ZWARAKAY..ARABIC FATHA WITH TWO DOTS
-066E..066F    ; Alphabetic # Lo   [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF
-0670          ; Alphabetic # Mn       ARABIC LETTER SUPERSCRIPT ALEF
-0671..06D3    ; Alphabetic # Lo  [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE
-06D5          ; Alphabetic # Lo       ARABIC LETTER AE
-06D6..06DC    ; Alphabetic # Mn   [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN
-06E1..06E4    ; Alphabetic # Mn   [4] ARABIC SMALL HIGH DOTLESS HEAD OF KHAH..ARABIC SMALL HIGH MADDA
-06E5..06E6    ; Alphabetic # Lm   [2] ARABIC SMALL WAW..ARABIC SMALL YEH
-06E7..06E8    ; Alphabetic # Mn   [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON
-06ED          ; Alphabetic # Mn       ARABIC SMALL LOW MEEM
-06EE..06EF    ; Alphabetic # Lo   [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V
-06FA..06FC    ; Alphabetic # Lo   [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW
-06FF          ; Alphabetic # Lo       ARABIC LETTER HEH WITH INVERTED V
-0710          ; Alphabetic # Lo       SYRIAC LETTER ALAPH
-0711          ; Alphabetic # Mn       SYRIAC LETTER SUPERSCRIPT ALAPH
-0712..072F    ; Alphabetic # Lo  [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH
-0730..073F    ; Alphabetic # Mn  [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA
-074D..07A5    ; Alphabetic # Lo  [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU
-07A6..07B0    ; Alphabetic # Mn  [11] THAANA ABAFILI..THAANA SUKUN
-07B1          ; Alphabetic # Lo       THAANA LETTER NAA
-07CA..07EA    ; Alphabetic # Lo  [33] NKO LETTER A..NKO LETTER JONA RA
-07F4..07F5    ; Alphabetic # Lm   [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE
-07FA          ; Alphabetic # Lm       NKO LAJANYALAN
-0800..0815    ; Alphabetic # Lo  [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF
-0816..0817    ; Alphabetic # Mn   [2] SAMARITAN MARK IN..SAMARITAN MARK IN-ALAF
-081A          ; Alphabetic # Lm       SAMARITAN MODIFIER LETTER EPENTHETIC YUT
-081B..0823    ; Alphabetic # Mn   [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A
-0824          ; Alphabetic # Lm       SAMARITAN MODIFIER LETTER SHORT A
-0825..0827    ; Alphabetic # Mn   [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U
-0828          ; Alphabetic # Lm       SAMARITAN MODIFIER LETTER I
-0829..082C    ; Alphabetic # Mn   [4] SAMARITAN VOWEL SIGN LONG I..SAMARITAN VOWEL SIGN SUKUN
-0900..0902    ; Alphabetic # Mn   [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA
-0903          ; Alphabetic # Mc       DEVANAGARI SIGN VISARGA
-0904..0939    ; Alphabetic # Lo  [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA
-093D          ; Alphabetic # Lo       DEVANAGARI SIGN AVAGRAHA
-093E..0940    ; Alphabetic # Mc   [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II
-0941..0948    ; Alphabetic # Mn   [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI
-0949..094C    ; Alphabetic # Mc   [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU
-094E          ; Alphabetic # Mc       DEVANAGARI VOWEL SIGN PRISHTHAMATRA E
-0950          ; Alphabetic # Lo       DEVANAGARI OM
-0955          ; Alphabetic # Mn       DEVANAGARI VOWEL SIGN CANDRA LONG E
-0958..0961    ; Alphabetic # Lo  [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL
-0962..0963    ; Alphabetic # Mn   [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL
-0971          ; Alphabetic # Lm       DEVANAGARI SIGN HIGH SPACING DOT
-0972          ; Alphabetic # Lo       DEVANAGARI LETTER CANDRA A
-0979..097F    ; Alphabetic # Lo   [7] DEVANAGARI LETTER ZHA..DEVANAGARI LETTER BBA
-0981          ; Alphabetic # Mn       BENGALI SIGN CANDRABINDU
-0982..0983    ; Alphabetic # Mc   [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA
-0985..098C    ; Alphabetic # Lo   [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L
-098F..0990    ; Alphabetic # Lo   [2] BENGALI LETTER E..BENGALI LETTER AI
-0993..09A8    ; Alphabetic # Lo  [22] BENGALI LETTER O..BENGALI LETTER NA
-09AA..09B0    ; Alphabetic # Lo   [7] BENGALI LETTER PA..BENGALI LETTER RA
-09B2          ; Alphabetic # Lo       BENGALI LETTER LA
-09B6..09B9    ; Alphabetic # Lo   [4] BENGALI LETTER SHA..BENGALI LETTER HA
-09BD          ; Alphabetic # Lo       BENGALI SIGN AVAGRAHA
-09BE..09C0    ; Alphabetic # Mc   [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II
-09C1..09C4    ; Alphabetic # Mn   [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR
-09C7..09C8    ; Alphabetic # Mc   [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI
-09CB..09CC    ; Alphabetic # Mc   [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU
-09CE          ; Alphabetic # Lo       BENGALI LETTER KHANDA TA
-09D7          ; Alphabetic # Mc       BENGALI AU LENGTH MARK
-09DC..09DD    ; Alphabetic # Lo   [2] BENGALI LETTER RRA..BENGALI LETTER RHA
-09DF..09E1    ; Alphabetic # Lo   [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL
-09E2..09E3    ; Alphabetic # Mn   [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL
-09F0..09F1    ; Alphabetic # Lo   [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL
-0A01..0A02    ; Alphabetic # Mn   [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI
-0A03          ; Alphabetic # Mc       GURMUKHI SIGN VISARGA
-0A05..0A0A    ; Alphabetic # Lo   [6] GURMUKHI LETTER A..GURMUKHI LETTER UU
-0A0F..0A10    ; Alphabetic # Lo   [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI
-0A13..0A28    ; Alphabetic # Lo  [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA
-0A2A..0A30    ; Alphabetic # Lo   [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA
-0A32..0A33    ; Alphabetic # Lo   [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA
-0A35..0A36    ; Alphabetic # Lo   [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA
-0A38..0A39    ; Alphabetic # Lo   [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA
-0A3E..0A40    ; Alphabetic # Mc   [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II
-0A41..0A42    ; Alphabetic # Mn   [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU
-0A47..0A48    ; Alphabetic # Mn   [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI
-0A4B..0A4C    ; Alphabetic # Mn   [2] GURMUKHI VOWEL SIGN OO..GURMUKHI VOWEL SIGN AU
-0A51          ; Alphabetic # Mn       GURMUKHI SIGN UDAAT
-0A59..0A5C    ; Alphabetic # Lo   [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA
-0A5E          ; Alphabetic # Lo       GURMUKHI LETTER FA
-0A70..0A71    ; Alphabetic # Mn   [2] GURMUKHI TIPPI..GURMUKHI ADDAK
-0A72..0A74    ; Alphabetic # Lo   [3] GURMUKHI IRI..GURMUKHI EK ONKAR
-0A75          ; Alphabetic # Mn       GURMUKHI SIGN YAKASH
-0A81..0A82    ; Alphabetic # Mn   [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA
-0A83          ; Alphabetic # Mc       GUJARATI SIGN VISARGA
-0A85..0A8D    ; Alphabetic # Lo   [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E
-0A8F..0A91    ; Alphabetic # Lo   [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O
-0A93..0AA8    ; Alphabetic # Lo  [22] GUJARATI LETTER O..GUJARATI LETTER NA
-0AAA..0AB0    ; Alphabetic # Lo   [7] GUJARATI LETTER PA..GUJARATI LETTER RA
-0AB2..0AB3    ; Alphabetic # Lo   [2] GUJARATI LETTER LA..GUJARATI LETTER LLA
-0AB5..0AB9    ; Alphabetic # Lo   [5] GUJARATI LETTER VA..GUJARATI LETTER HA
-0ABD          ; Alphabetic # Lo       GUJARATI SIGN AVAGRAHA
-0ABE..0AC0    ; Alphabetic # Mc   [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II
-0AC1..0AC5    ; Alphabetic # Mn   [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E
-0AC7..0AC8    ; Alphabetic # Mn   [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI
-0AC9          ; Alphabetic # Mc       GUJARATI VOWEL SIGN CANDRA O
-0ACB..0ACC    ; Alphabetic # Mc   [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU
-0AD0          ; Alphabetic # Lo       GUJARATI OM
-0AE0..0AE1    ; Alphabetic # Lo   [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL
-0AE2..0AE3    ; Alphabetic # Mn   [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL
-0B01          ; Alphabetic # Mn       ORIYA SIGN CANDRABINDU
-0B02..0B03    ; Alphabetic # Mc   [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA
-0B05..0B0C    ; Alphabetic # Lo   [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L
-0B0F..0B10    ; Alphabetic # Lo   [2] ORIYA LETTER E..ORIYA LETTER AI
-0B13..0B28    ; Alphabetic # Lo  [22] ORIYA LETTER O..ORIYA LETTER NA
-0B2A..0B30    ; Alphabetic # Lo   [7] ORIYA LETTER PA..ORIYA LETTER RA
-0B32..0B33    ; Alphabetic # Lo   [2] ORIYA LETTER LA..ORIYA LETTER LLA
-0B35..0B39    ; Alphabetic # Lo   [5] ORIYA LETTER VA..ORIYA LETTER HA
-0B3D          ; Alphabetic # Lo       ORIYA SIGN AVAGRAHA
-0B3E          ; Alphabetic # Mc       ORIYA VOWEL SIGN AA
-0B3F          ; Alphabetic # Mn       ORIYA VOWEL SIGN I
-0B40          ; Alphabetic # Mc       ORIYA VOWEL SIGN II
-0B41..0B44    ; Alphabetic # Mn   [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR
-0B47..0B48    ; Alphabetic # Mc   [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI
-0B4B..0B4C    ; Alphabetic # Mc   [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU
-0B56          ; Alphabetic # Mn       ORIYA AI LENGTH MARK
-0B57          ; Alphabetic # Mc       ORIYA AU LENGTH MARK
-0B5C..0B5D    ; Alphabetic # Lo   [2] ORIYA LETTER RRA..ORIYA LETTER RHA
-0B5F..0B61    ; Alphabetic # Lo   [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL
-0B62..0B63    ; Alphabetic # Mn   [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL
-0B71          ; Alphabetic # Lo       ORIYA LETTER WA
-0B82          ; Alphabetic # Mn       TAMIL SIGN ANUSVARA
-0B83          ; Alphabetic # Lo       TAMIL SIGN VISARGA
-0B85..0B8A    ; Alphabetic # Lo   [6] TAMIL LETTER A..TAMIL LETTER UU
-0B8E..0B90    ; Alphabetic # Lo   [3] TAMIL LETTER E..TAMIL LETTER AI
-0B92..0B95    ; Alphabetic # Lo   [4] TAMIL LETTER O..TAMIL LETTER KA
-0B99..0B9A    ; Alphabetic # Lo   [2] TAMIL LETTER NGA..TAMIL LETTER CA
-0B9C          ; Alphabetic # Lo       TAMIL LETTER JA
-0B9E..0B9F    ; Alphabetic # Lo   [2] TAMIL LETTER NYA..TAMIL LETTER TTA
-0BA3..0BA4    ; Alphabetic # Lo   [2] TAMIL LETTER NNA..TAMIL LETTER TA
-0BA8..0BAA    ; Alphabetic # Lo   [3] TAMIL LETTER NA..TAMIL LETTER PA
-0BAE..0BB9    ; Alphabetic # Lo  [12] TAMIL LETTER MA..TAMIL LETTER HA
-0BBE..0BBF    ; Alphabetic # Mc   [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I
-0BC0          ; Alphabetic # Mn       TAMIL VOWEL SIGN II
-0BC1..0BC2    ; Alphabetic # Mc   [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU
-0BC6..0BC8    ; Alphabetic # Mc   [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI
-0BCA..0BCC    ; Alphabetic # Mc   [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU
-0BD0          ; Alphabetic # Lo       TAMIL OM
-0BD7          ; Alphabetic # Mc       TAMIL AU LENGTH MARK
-0C01..0C03    ; Alphabetic # Mc   [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA
-0C05..0C0C    ; Alphabetic # Lo   [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L
-0C0E..0C10    ; Alphabetic # Lo   [3] TELUGU LETTER E..TELUGU LETTER AI
-0C12..0C28    ; Alphabetic # Lo  [23] TELUGU LETTER O..TELUGU LETTER NA
-0C2A..0C33    ; Alphabetic # Lo  [10] TELUGU LETTER PA..TELUGU LETTER LLA
-0C35..0C39    ; Alphabetic # Lo   [5] TELUGU LETTER VA..TELUGU LETTER HA
-0C3D          ; Alphabetic # Lo       TELUGU SIGN AVAGRAHA
-0C3E..0C40    ; Alphabetic # Mn   [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II
-0C41..0C44    ; Alphabetic # Mc   [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR
-0C46..0C48    ; Alphabetic # Mn   [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI
-0C4A..0C4C    ; Alphabetic # Mn   [3] TELUGU VOWEL SIGN O..TELUGU VOWEL SIGN AU
-0C55..0C56    ; Alphabetic # Mn   [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK
-0C58..0C59    ; Alphabetic # Lo   [2] TELUGU LETTER TSA..TELUGU LETTER DZA
-0C60..0C61    ; Alphabetic # Lo   [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL
-0C62..0C63    ; Alphabetic # Mn   [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL
-0C82..0C83    ; Alphabetic # Mc   [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA
-0C85..0C8C    ; Alphabetic # Lo   [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L
-0C8E..0C90    ; Alphabetic # Lo   [3] KANNADA LETTER E..KANNADA LETTER AI
-0C92..0CA8    ; Alphabetic # Lo  [23] KANNADA LETTER O..KANNADA LETTER NA
-0CAA..0CB3    ; Alphabetic # Lo  [10] KANNADA LETTER PA..KANNADA LETTER LLA
-0CB5..0CB9    ; Alphabetic # Lo   [5] KANNADA LETTER VA..KANNADA LETTER HA
-0CBD          ; Alphabetic # Lo       KANNADA SIGN AVAGRAHA
-0CBE          ; Alphabetic # Mc       KANNADA VOWEL SIGN AA
-0CBF          ; Alphabetic # Mn       KANNADA VOWEL SIGN I
-0CC0..0CC4    ; Alphabetic # Mc   [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR
-0CC6          ; Alphabetic # Mn       KANNADA VOWEL SIGN E
-0CC7..0CC8    ; Alphabetic # Mc   [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI
-0CCA..0CCB    ; Alphabetic # Mc   [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO
-0CCC          ; Alphabetic # Mn       KANNADA VOWEL SIGN AU
-0CD5..0CD6    ; Alphabetic # Mc   [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK
-0CDE          ; Alphabetic # Lo       KANNADA LETTER FA
-0CE0..0CE1    ; Alphabetic # Lo   [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL
-0CE2..0CE3    ; Alphabetic # Mn   [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL
-0D02..0D03    ; Alphabetic # Mc   [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA
-0D05..0D0C    ; Alphabetic # Lo   [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L
-0D0E..0D10    ; Alphabetic # Lo   [3] MALAYALAM LETTER E..MALAYALAM LETTER AI
-0D12..0D28    ; Alphabetic # Lo  [23] MALAYALAM LETTER O..MALAYALAM LETTER NA
-0D2A..0D39    ; Alphabetic # Lo  [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA
-0D3D          ; Alphabetic # Lo       MALAYALAM SIGN AVAGRAHA
-0D3E..0D40    ; Alphabetic # Mc   [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II
-0D41..0D44    ; Alphabetic # Mn   [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR
-0D46..0D48    ; Alphabetic # Mc   [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI
-0D4A..0D4C    ; Alphabetic # Mc   [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU
-0D57          ; Alphabetic # Mc       MALAYALAM AU LENGTH MARK
-0D60..0D61    ; Alphabetic # Lo   [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL
-0D62..0D63    ; Alphabetic # Mn   [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL
-0D7A..0D7F    ; Alphabetic # Lo   [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K
-0D82..0D83    ; Alphabetic # Mc   [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA
-0D85..0D96    ; Alphabetic # Lo  [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA
-0D9A..0DB1    ; Alphabetic # Lo  [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA
-0DB3..0DBB    ; Alphabetic # Lo   [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA
-0DBD          ; Alphabetic # Lo       SINHALA LETTER DANTAJA LAYANNA
-0DC0..0DC6    ; Alphabetic # Lo   [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA
-0DCF..0DD1    ; Alphabetic # Mc   [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA
-0DD2..0DD4    ; Alphabetic # Mn   [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA
-0DD6          ; Alphabetic # Mn       SINHALA VOWEL SIGN DIGA PAA-PILLA
-0DD8..0DDF    ; Alphabetic # Mc   [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA
-0DF2..0DF3    ; Alphabetic # Mc   [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA
-0E01..0E30    ; Alphabetic # Lo  [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A
-0E31          ; Alphabetic # Mn       THAI CHARACTER MAI HAN-AKAT
-0E32..0E33    ; Alphabetic # Lo   [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM
-0E34..0E3A    ; Alphabetic # Mn   [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU
-0E40..0E45    ; Alphabetic # Lo   [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO
-0E46          ; Alphabetic # Lm       THAI CHARACTER MAIYAMOK
-0E4D          ; Alphabetic # Mn       THAI CHARACTER NIKHAHIT
-0E81..0E82    ; Alphabetic # Lo   [2] LAO LETTER KO..LAO LETTER KHO SUNG
-0E84          ; Alphabetic # Lo       LAO LETTER KHO TAM
-0E87..0E88    ; Alphabetic # Lo   [2] LAO LETTER NGO..LAO LETTER CO
-0E8A          ; Alphabetic # Lo       LAO LETTER SO TAM
-0E8D          ; Alphabetic # Lo       LAO LETTER NYO
-0E94..0E97    ; Alphabetic # Lo   [4] LAO LETTER DO..LAO LETTER THO TAM
-0E99..0E9F    ; Alphabetic # Lo   [7] LAO LETTER NO..LAO LETTER FO SUNG
-0EA1..0EA3    ; Alphabetic # Lo   [3] LAO LETTER MO..LAO LETTER LO LING
-0EA5          ; Alphabetic # Lo       LAO LETTER LO LOOT
-0EA7          ; Alphabetic # Lo       LAO LETTER WO
-0EAA..0EAB    ; Alphabetic # Lo   [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG
-0EAD..0EB0    ; Alphabetic # Lo   [4] LAO LETTER O..LAO VOWEL SIGN A
-0EB1          ; Alphabetic # Mn       LAO VOWEL SIGN MAI KAN
-0EB2..0EB3    ; Alphabetic # Lo   [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM
-0EB4..0EB9    ; Alphabetic # Mn   [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU
-0EBB..0EBC    ; Alphabetic # Mn   [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO
-0EBD          ; Alphabetic # Lo       LAO SEMIVOWEL SIGN NYO
-0EC0..0EC4    ; Alphabetic # Lo   [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI
-0EC6          ; Alphabetic # Lm       LAO KO LA
-0ECD          ; Alphabetic # Mn       LAO NIGGAHITA
-0EDC..0EDD    ; Alphabetic # Lo   [2] LAO HO NO..LAO HO MO
-0F00          ; Alphabetic # Lo       TIBETAN SYLLABLE OM
-0F40..0F47    ; Alphabetic # Lo   [8] TIBETAN LETTER KA..TIBETAN LETTER JA
-0F49..0F6C    ; Alphabetic # Lo  [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA
-0F71..0F7E    ; Alphabetic # Mn  [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO
-0F7F          ; Alphabetic # Mc       TIBETAN SIGN RNAM BCAD
-0F80..0F81    ; Alphabetic # Mn   [2] TIBETAN VOWEL SIGN REVERSED I..TIBETAN VOWEL SIGN REVERSED II
-0F88..0F8B    ; Alphabetic # Lo   [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS
-0F90..0F97    ; Alphabetic # Mn   [8] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER JA
-0F99..0FBC    ; Alphabetic # Mn  [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA
-1000..102A    ; Alphabetic # Lo  [43] MYANMAR LETTER KA..MYANMAR LETTER AU
-102B..102C    ; Alphabetic # Mc   [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA
-102D..1030    ; Alphabetic # Mn   [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU
-1031          ; Alphabetic # Mc       MYANMAR VOWEL SIGN E
-1032..1036    ; Alphabetic # Mn   [5] MYANMAR VOWEL SIGN AI..MYANMAR SIGN ANUSVARA
-1038          ; Alphabetic # Mc       MYANMAR SIGN VISARGA
-103B..103C    ; Alphabetic # Mc   [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA
-103D..103E    ; Alphabetic # Mn   [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA
-103F          ; Alphabetic # Lo       MYANMAR LETTER GREAT SA
-1050..1055    ; Alphabetic # Lo   [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL
-1056..1057    ; Alphabetic # Mc   [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR
-1058..1059    ; Alphabetic # Mn   [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL
-105A..105D    ; Alphabetic # Lo   [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE
-105E..1060    ; Alphabetic # Mn   [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA
-1061          ; Alphabetic # Lo       MYANMAR LETTER SGAW KAREN SHA
-1062          ; Alphabetic # Mc       MYANMAR VOWEL SIGN SGAW KAREN EU
-1065..1066    ; Alphabetic # Lo   [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA
-1067..1068    ; Alphabetic # Mc   [2] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR VOWEL SIGN WESTERN PWO KAREN UE
-106E..1070    ; Alphabetic # Lo   [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA
-1071..1074    ; Alphabetic # Mn   [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE
-1075..1081    ; Alphabetic # Lo  [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA
-1082          ; Alphabetic # Mn       MYANMAR CONSONANT SIGN SHAN MEDIAL WA
-1083..1084    ; Alphabetic # Mc   [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E
-1085..1086    ; Alphabetic # Mn   [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y
-108E          ; Alphabetic # Lo       MYANMAR LETTER RUMAI PALAUNG FA
-109C          ; Alphabetic # Mc       MYANMAR VOWEL SIGN AITON A
-109D          ; Alphabetic # Mn       MYANMAR VOWEL SIGN AITON AI
-10A0..10C5    ; Alphabetic # L&  [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE
-10D0..10FA    ; Alphabetic # Lo  [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN
-10FC          ; Alphabetic # Lm       MODIFIER LETTER GEORGIAN NAR
-1100..1248    ; Alphabetic # Lo [329] HANGUL CHOSEONG KIYEOK..ETHIOPIC SYLLABLE QWA
-124A..124D    ; Alphabetic # Lo   [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE
-1250..1256    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO
-1258          ; Alphabetic # Lo       ETHIOPIC SYLLABLE QHWA
-125A..125D    ; Alphabetic # Lo   [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE
-1260..1288    ; Alphabetic # Lo  [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA
-128A..128D    ; Alphabetic # Lo   [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE
-1290..12B0    ; Alphabetic # Lo  [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA
-12B2..12B5    ; Alphabetic # Lo   [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE
-12B8..12BE    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO
-12C0          ; Alphabetic # Lo       ETHIOPIC SYLLABLE KXWA
-12C2..12C5    ; Alphabetic # Lo   [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE
-12C8..12D6    ; Alphabetic # Lo  [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O
-12D8..1310    ; Alphabetic # Lo  [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA
-1312..1315    ; Alphabetic # Lo   [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE
-1318..135A    ; Alphabetic # Lo  [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA
-135F          ; Alphabetic # Mn       ETHIOPIC COMBINING GEMINATION MARK
-1380..138F    ; Alphabetic # Lo  [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE
-13A0..13F4    ; Alphabetic # Lo  [85] CHEROKEE LETTER A..CHEROKEE LETTER YV
-1401..166C    ; Alphabetic # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA
-166F..167F    ; Alphabetic # Lo  [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W
-1681..169A    ; Alphabetic # Lo  [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH
-16A0..16EA    ; Alphabetic # Lo  [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X
-16EE..16F0    ; Alphabetic # Nl   [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL
-1700..170C    ; Alphabetic # Lo  [13] TAGALOG LETTER A..TAGALOG LETTER YA
-170E..1711    ; Alphabetic # Lo   [4] TAGALOG LETTER LA..TAGALOG LETTER HA
-1712..1713    ; Alphabetic # Mn   [2] TAGALOG VOWEL SIGN I..TAGALOG VOWEL SIGN U
-1720..1731    ; Alphabetic # Lo  [18] HANUNOO LETTER A..HANUNOO LETTER HA
-1732..1733    ; Alphabetic # Mn   [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U
-1740..1751    ; Alphabetic # Lo  [18] BUHID LETTER A..BUHID LETTER HA
-1752..1753    ; Alphabetic # Mn   [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U
-1760..176C    ; Alphabetic # Lo  [13] TAGBANWA LETTER A..TAGBANWA LETTER YA
-176E..1770    ; Alphabetic # Lo   [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA
-1772..1773    ; Alphabetic # Mn   [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U
-1780..17B3    ; Alphabetic # Lo  [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU
-17B6          ; Alphabetic # Mc       KHMER VOWEL SIGN AA
-17B7..17BD    ; Alphabetic # Mn   [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA
-17BE..17C5    ; Alphabetic # Mc   [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU
-17C6          ; Alphabetic # Mn       KHMER SIGN NIKAHIT
-17C7..17C8    ; Alphabetic # Mc   [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU
-17D7          ; Alphabetic # Lm       KHMER SIGN LEK TOO
-17DC          ; Alphabetic # Lo       KHMER SIGN AVAKRAHASANYA
-1820..1842    ; Alphabetic # Lo  [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI
-1843          ; Alphabetic # Lm       MONGOLIAN LETTER TODO LONG VOWEL SIGN
-1844..1877    ; Alphabetic # Lo  [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA
-1880..18A8    ; Alphabetic # Lo  [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA
-18A9          ; Alphabetic # Mn       MONGOLIAN LETTER ALI GALI DAGALGA
-18AA          ; Alphabetic # Lo       MONGOLIAN LETTER MANCHU ALI GALI LHA
-18B0..18F5    ; Alphabetic # Lo  [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S
-1900..191C    ; Alphabetic # Lo  [29] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER HA
-1920..1922    ; Alphabetic # Mn   [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U
-1923..1926    ; Alphabetic # Mc   [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU
-1927..1928    ; Alphabetic # Mn   [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O
-1929..192B    ; Alphabetic # Mc   [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA
-1930..1931    ; Alphabetic # Mc   [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA
-1932          ; Alphabetic # Mn       LIMBU SMALL LETTER ANUSVARA
-1933..1938    ; Alphabetic # Mc   [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA
-1950..196D    ; Alphabetic # Lo  [30] TAI LE LETTER KA..TAI LE LETTER AI
-1970..1974    ; Alphabetic # Lo   [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6
-1980..19AB    ; Alphabetic # Lo  [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA
-19B0..19C0    ; Alphabetic # Mc  [17] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE VOWEL SIGN IY
-19C1..19C7    ; Alphabetic # Lo   [7] NEW TAI LUE LETTER FINAL V..NEW TAI LUE LETTER FINAL B
-19C8..19C9    ; Alphabetic # Mc   [2] NEW TAI LUE TONE MARK-1..NEW TAI LUE TONE MARK-2
-1A00..1A16    ; Alphabetic # Lo  [23] BUGINESE LETTER KA..BUGINESE LETTER HA
-1A17..1A18    ; Alphabetic # Mn   [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U
-1A19..1A1B    ; Alphabetic # Mc   [3] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN AE
-1A20..1A54    ; Alphabetic # Lo  [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA
-1A55          ; Alphabetic # Mc       TAI THAM CONSONANT SIGN MEDIAL RA
-1A56          ; Alphabetic # Mn       TAI THAM CONSONANT SIGN MEDIAL LA
-1A57          ; Alphabetic # Mc       TAI THAM CONSONANT SIGN LA TANG LAI
-1A58..1A5E    ; Alphabetic # Mn   [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA
-1A61          ; Alphabetic # Mc       TAI THAM VOWEL SIGN A
-1A62          ; Alphabetic # Mn       TAI THAM VOWEL SIGN MAI SAT
-1A63..1A64    ; Alphabetic # Mc   [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA
-1A65..1A6C    ; Alphabetic # Mn   [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW
-1A6D..1A72    ; Alphabetic # Mc   [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI
-1A73..1A74    ; Alphabetic # Mn   [2] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN MAI KANG
-1AA7          ; Alphabetic # Lm       TAI THAM SIGN MAI YAMOK
-1B00..1B03    ; Alphabetic # Mn   [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG
-1B04          ; Alphabetic # Mc       BALINESE SIGN BISAH
-1B05..1B33    ; Alphabetic # Lo  [47] BALINESE LETTER AKARA..BALINESE LETTER HA
-1B35          ; Alphabetic # Mc       BALINESE VOWEL SIGN TEDUNG
-1B36..1B3A    ; Alphabetic # Mn   [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA
-1B3B          ; Alphabetic # Mc       BALINESE VOWEL SIGN RA REPA TEDUNG
-1B3C          ; Alphabetic # Mn       BALINESE VOWEL SIGN LA LENGA
-1B3D..1B41    ; Alphabetic # Mc   [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG
-1B42          ; Alphabetic # Mn       BALINESE VOWEL SIGN PEPET
-1B43          ; Alphabetic # Mc       BALINESE VOWEL SIGN PEPET TEDUNG
-1B45..1B4B    ; Alphabetic # Lo   [7] BALINESE LETTER KAF SASAK..BALINESE LETTER ASYURA SASAK
-1B80..1B81    ; Alphabetic # Mn   [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR
-1B82          ; Alphabetic # Mc       SUNDANESE SIGN PANGWISAD
-1B83..1BA0    ; Alphabetic # Lo  [30] SUNDANESE LETTER A..SUNDANESE LETTER HA
-1BA1          ; Alphabetic # Mc       SUNDANESE CONSONANT SIGN PAMINGKAL
-1BA2..1BA5    ; Alphabetic # Mn   [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU
-1BA6..1BA7    ; Alphabetic # Mc   [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG
-1BA8..1BA9    ; Alphabetic # Mn   [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG
-1BAE..1BAF    ; Alphabetic # Lo   [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA
-1C00..1C23    ; Alphabetic # Lo  [36] LEPCHA LETTER KA..LEPCHA LETTER A
-1C24..1C2B    ; Alphabetic # Mc   [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU
-1C2C..1C33    ; Alphabetic # Mn   [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T
-1C34..1C35    ; Alphabetic # Mc   [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG
-1C4D..1C4F    ; Alphabetic # Lo   [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA
-1C5A..1C77    ; Alphabetic # Lo  [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH
-1C78..1C7D    ; Alphabetic # Lm   [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD
-1CE9..1CEC    ; Alphabetic # Lo   [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL
-1CEE..1CF1    ; Alphabetic # Lo   [4] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ANUSVARA UBHAYATO MUKHA
-1CF2          ; Alphabetic # Mc       VEDIC SIGN ARDHAVISARGA
-1D00..1D2B    ; Alphabetic # L&  [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL
-1D2C..1D61    ; Alphabetic # Lm  [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI
-1D62..1D77    ; Alphabetic # L&  [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G
-1D78          ; Alphabetic # Lm       MODIFIER LETTER CYRILLIC EN
-1D79..1D9A    ; Alphabetic # L&  [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK
-1D9B..1DBF    ; Alphabetic # Lm  [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA
-1E00..1F15    ; Alphabetic # L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA
-1F18..1F1D    ; Alphabetic # L&   [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
-1F20..1F45    ; Alphabetic # L&  [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA
-1F48..1F4D    ; Alphabetic # L&   [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
-1F50..1F57    ; Alphabetic # L&   [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI
-1F59          ; Alphabetic # L&       GREEK CAPITAL LETTER UPSILON WITH DASIA
-1F5B          ; Alphabetic # L&       GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
-1F5D          ; Alphabetic # L&       GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
-1F5F..1F7D    ; Alphabetic # L&  [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA
-1F80..1FB4    ; Alphabetic # L&  [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
-1FB6..1FBC    ; Alphabetic # L&   [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
-1FBE          ; Alphabetic # L&       GREEK PROSGEGRAMMENI
-1FC2..1FC4    ; Alphabetic # L&   [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
-1FC6..1FCC    ; Alphabetic # L&   [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
-1FD0..1FD3    ; Alphabetic # L&   [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
-1FD6..1FDB    ; Alphabetic # L&   [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA
-1FE0..1FEC    ; Alphabetic # L&  [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA
-1FF2..1FF4    ; Alphabetic # L&   [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
-1FF6..1FFC    ; Alphabetic # L&   [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
-2071          ; Alphabetic # Lm       SUPERSCRIPT LATIN SMALL LETTER I
-207F          ; Alphabetic # Lm       SUPERSCRIPT LATIN SMALL LETTER N
-2090..2094    ; Alphabetic # Lm   [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA
-2102          ; Alphabetic # L&       DOUBLE-STRUCK CAPITAL C
-2107          ; Alphabetic # L&       EULER CONSTANT
-210A..2113    ; Alphabetic # L&  [10] SCRIPT SMALL G..SCRIPT SMALL L
-2115          ; Alphabetic # L&       DOUBLE-STRUCK CAPITAL N
-2119..211D    ; Alphabetic # L&   [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R
-2124          ; Alphabetic # L&       DOUBLE-STRUCK CAPITAL Z
-2126          ; Alphabetic # L&       OHM SIGN
-2128          ; Alphabetic # L&       BLACK-LETTER CAPITAL Z
-212A..212D    ; Alphabetic # L&   [4] KELVIN SIGN..BLACK-LETTER CAPITAL C
-212F..2134    ; Alphabetic # L&   [6] SCRIPT SMALL E..SCRIPT SMALL O
-2135..2138    ; Alphabetic # Lo   [4] ALEF SYMBOL..DALET SYMBOL
-2139          ; Alphabetic # L&       INFORMATION SOURCE
-213C..213F    ; Alphabetic # L&   [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI
-2145..2149    ; Alphabetic # L&   [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J
-214E          ; Alphabetic # L&       TURNED SMALL F
-2160..2182    ; Alphabetic # Nl  [35] ROMAN NUMERAL ONE..ROMAN NUMERAL TEN THOUSAND
-2183..2184    ; Alphabetic # L&   [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C
-2185..2188    ; Alphabetic # Nl   [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND
-24B6..24E9    ; Alphabetic # So  [52] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN SMALL LETTER Z
-2C00..2C2E    ; Alphabetic # L&  [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE
-2C30..2C5E    ; Alphabetic # L&  [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE
-2C60..2C7C    ; Alphabetic # L&  [29] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN SUBSCRIPT SMALL LETTER J
-2C7D          ; Alphabetic # Lm       MODIFIER LETTER CAPITAL V
-2C7E..2CE4    ; Alphabetic # L& [103] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SYMBOL KAI
-2CEB..2CEE    ; Alphabetic # L&   [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA
-2D00..2D25    ; Alphabetic # L&  [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE
-2D30..2D65    ; Alphabetic # Lo  [54] TIFINAGH LETTER YA..TIFINAGH LETTER YAZZ
-2D6F          ; Alphabetic # Lm       TIFINAGH MODIFIER LETTER LABIALIZATION MARK
-2D80..2D96    ; Alphabetic # Lo  [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE
-2DA0..2DA6    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO
-2DA8..2DAE    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO
-2DB0..2DB6    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO
-2DB8..2DBE    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO
-2DC0..2DC6    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO
-2DC8..2DCE    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO
-2DD0..2DD6    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO
-2DD8..2DDE    ; Alphabetic # Lo   [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO
-2DE0..2DFF    ; Alphabetic # Mn  [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS
-2E2F          ; Alphabetic # Lm       VERTICAL TILDE
-3005          ; Alphabetic # Lm       IDEOGRAPHIC ITERATION MARK
-3006          ; Alphabetic # Lo       IDEOGRAPHIC CLOSING MARK
-3007          ; Alphabetic # Nl       IDEOGRAPHIC NUMBER ZERO
-3021..3029    ; Alphabetic # Nl   [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE
-3031..3035    ; Alphabetic # Lm   [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF
-3038..303A    ; Alphabetic # Nl   [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY
-303B          ; Alphabetic # Lm       VERTICAL IDEOGRAPHIC ITERATION MARK
-303C          ; Alphabetic # Lo       MASU MARK
-3041..3096    ; Alphabetic # Lo  [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE
-309D..309E    ; Alphabetic # Lm   [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK
-309F          ; Alphabetic # Lo       HIRAGANA DIGRAPH YORI
-30A1..30FA    ; Alphabetic # Lo  [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO
-30FC..30FE    ; Alphabetic # Lm   [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK
-30FF          ; Alphabetic # Lo       KATAKANA DIGRAPH KOTO
-3105..312D    ; Alphabetic # Lo  [41] BOPOMOFO LETTER B..BOPOMOFO LETTER IH
-3131..318E    ; Alphabetic # Lo  [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE
-31A0..31B7    ; Alphabetic # Lo  [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H
-31F0..31FF    ; Alphabetic # Lo  [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO
-3400..4DB5    ; Alphabetic # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5
-4E00..9FCB    ; Alphabetic # Lo [20940] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FCB
-A000..A014    ; Alphabetic # Lo  [21] YI SYLLABLE IT..YI SYLLABLE E
-A015          ; Alphabetic # Lm       YI SYLLABLE WU
-A016..A48C    ; Alphabetic # Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR
-A4D0..A4F7    ; Alphabetic # Lo  [40] LISU LETTER BA..LISU LETTER OE
-A4F8..A4FD    ; Alphabetic # Lm   [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU
-A500..A60B    ; Alphabetic # Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG
-A60C          ; Alphabetic # Lm       VAI SYLLABLE LENGTHENER
-A610..A61F    ; Alphabetic # Lo  [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG
-A62A..A62B    ; Alphabetic # Lo   [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO
-A640..A65F    ; Alphabetic # L&  [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN
-A662..A66D    ; Alphabetic # L&  [12] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O
-A66E          ; Alphabetic # Lo       CYRILLIC LETTER MULTIOCULAR O
-A67F          ; Alphabetic # Lm       CYRILLIC PAYEROK
-A680..A697    ; Alphabetic # L&  [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE
-A6A0..A6E5    ; Alphabetic # Lo  [70] BAMUM LETTER A..BAMUM LETTER KI
-A6E6..A6EF    ; Alphabetic # Nl  [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM
-A717..A71F    ; Alphabetic # Lm   [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK
-A722..A76F    ; Alphabetic # L&  [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON
-A770          ; Alphabetic # Lm       MODIFIER LETTER US
-A771..A787    ; Alphabetic # L&  [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T
-A788          ; Alphabetic # Lm       MODIFIER LETTER LOW CIRCUMFLEX ACCENT
-A78B..A78C    ; Alphabetic # L&   [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO
-A7FB..A801    ; Alphabetic # Lo   [7] LATIN EPIGRAPHIC LETTER REVERSED F..SYLOTI NAGRI LETTER I
-A803..A805    ; Alphabetic # Lo   [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O
-A807..A80A    ; Alphabetic # Lo   [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO
-A80C..A822    ; Alphabetic # Lo  [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO
-A823..A824    ; Alphabetic # Mc   [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I
-A825..A826    ; Alphabetic # Mn   [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E
-A827          ; Alphabetic # Mc       SYLOTI NAGRI VOWEL SIGN OO
-A840..A873    ; Alphabetic # Lo  [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU
-A880..A881    ; Alphabetic # Mc   [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA
-A882..A8B3    ; Alphabetic # Lo  [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA
-A8B4..A8C3    ; Alphabetic # Mc  [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU
-A8F2..A8F7    ; Alphabetic # Lo   [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA
-A8FB          ; Alphabetic # Lo       DEVANAGARI HEADSTROKE
-A90A..A925    ; Alphabetic # Lo  [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO
-A926..A92A    ; Alphabetic # Mn   [5] KAYAH LI VOWEL UE..KAYAH LI VOWEL O
-A930..A946    ; Alphabetic # Lo  [23] REJANG LETTER KA..REJANG LETTER A
-A947..A951    ; Alphabetic # Mn  [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R
-A952          ; Alphabetic # Mc       REJANG CONSONANT SIGN H
-A960..A97C    ; Alphabetic # Lo  [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH
-A980..A982    ; Alphabetic # Mn   [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR
-A983          ; Alphabetic # Mc       JAVANESE SIGN WIGNYAN
-A984..A9B2    ; Alphabetic # Lo  [47] JAVANESE LETTER A..JAVANESE LETTER HA
-A9B3          ; Alphabetic # Mn       JAVANESE SIGN CECAK TELU
-A9B4..A9B5    ; Alphabetic # Mc   [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG
-A9B6..A9B9    ; Alphabetic # Mn   [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT
-A9BA..A9BB    ; Alphabetic # Mc   [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE
-A9BC          ; Alphabetic # Mn       JAVANESE VOWEL SIGN PEPET
-A9BD..A9BF    ; Alphabetic # Mc   [3] JAVANESE CONSONANT SIGN KERET..JAVANESE CONSONANT SIGN CAKRA
-A9CF          ; Alphabetic # Lm       JAVANESE PANGRANGKEP
-AA00..AA28    ; Alphabetic # Lo  [41] CHAM LETTER A..CHAM LETTER HA
-AA29..AA2E    ; Alphabetic # Mn   [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE
-AA2F..AA30    ; Alphabetic # Mc   [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI
-AA31..AA32    ; Alphabetic # Mn   [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE
-AA33..AA34    ; Alphabetic # Mc   [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA
-AA35..AA36    ; Alphabetic # Mn   [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA
-AA40..AA42    ; Alphabetic # Lo   [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG
-AA43          ; Alphabetic # Mn       CHAM CONSONANT SIGN FINAL NG
-AA44..AA4B    ; Alphabetic # Lo   [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS
-AA4C          ; Alphabetic # Mn       CHAM CONSONANT SIGN FINAL M
-AA4D          ; Alphabetic # Mc       CHAM CONSONANT SIGN FINAL H
-AA60..AA6F    ; Alphabetic # Lo  [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA
-AA70          ; Alphabetic # Lm       MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION
-AA71..AA76    ; Alphabetic # Lo   [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM
-AA7A          ; Alphabetic # Lo       MYANMAR LETTER AITON RA
-AA80..AAAF    ; Alphabetic # Lo  [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O
-AAB0          ; Alphabetic # Mn       TAI VIET MAI KANG
-AAB1          ; Alphabetic # Lo       TAI VIET VOWEL AA
-AAB2..AAB4    ; Alphabetic # Mn   [3] TAI VIET VOWEL I..TAI VIET VOWEL U
-AAB5..AAB6    ; Alphabetic # Lo   [2] TAI VIET VOWEL E..TAI VIET VOWEL O
-AAB7..AAB8    ; Alphabetic # Mn   [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA
-AAB9..AABD    ; Alphabetic # Lo   [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN
-AABE          ; Alphabetic # Mn       TAI VIET VOWEL AM
-AAC0          ; Alphabetic # Lo       TAI VIET TONE MAI NUENG
-AAC2          ; Alphabetic # Lo       TAI VIET TONE MAI SONG
-AADB..AADC    ; Alphabetic # Lo   [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG
-AADD          ; Alphabetic # Lm       TAI VIET SYMBOL SAM
-ABC0..ABE2    ; Alphabetic # Lo  [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM
-ABE3..ABE4    ; Alphabetic # Mc   [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP
-ABE5          ; Alphabetic # Mn       MEETEI MAYEK VOWEL SIGN ANAP
-ABE6..ABE7    ; Alphabetic # Mc   [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP
-ABE8          ; Alphabetic # Mn       MEETEI MAYEK VOWEL SIGN UNAP
-ABE9..ABEA    ; Alphabetic # Mc   [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG
-AC00..D7A3    ; Alphabetic # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH
-D7B0..D7C6    ; Alphabetic # Lo  [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E
-D7CB..D7FB    ; Alphabetic # Lo  [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH
-F900..FA2D    ; Alphabetic # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D
-FA30..FA6D    ; Alphabetic # Lo  [62] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6D
-FA70..FAD9    ; Alphabetic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9
-FB00..FB06    ; Alphabetic # L&   [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST
-FB13..FB17    ; Alphabetic # L&   [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH
-FB1D          ; Alphabetic # Lo       HEBREW LETTER YOD WITH HIRIQ
-FB1E          ; Alphabetic # Mn       HEBREW POINT JUDEO-SPANISH VARIKA
-FB1F..FB28    ; Alphabetic # Lo  [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV
-FB2A..FB36    ; Alphabetic # Lo  [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH
-FB38..FB3C    ; Alphabetic # Lo   [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH
-FB3E          ; Alphabetic # Lo       HEBREW LETTER MEM WITH DAGESH
-FB40..FB41    ; Alphabetic # Lo   [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH
-FB43..FB44    ; Alphabetic # Lo   [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH
-FB46..FBB1    ; Alphabetic # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM
-FBD3..FD3D    ; Alphabetic # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM
-FD50..FD8F    ; Alphabetic # Lo  [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM
-FD92..FDC7    ; Alphabetic # Lo  [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM
-FDF0..FDFB    ; Alphabetic # Lo  [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU
-FE70..FE74    ; Alphabetic # Lo   [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM
-FE76..FEFC    ; Alphabetic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM
-FF21..FF3A    ; Alphabetic # L&  [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z
-FF41..FF5A    ; Alphabetic # L&  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z
-FF66..FF6F    ; Alphabetic # Lo  [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU
-FF70          ; Alphabetic # Lm       HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK
-FF71..FF9D    ; Alphabetic # Lo  [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N
-FF9E..FF9F    ; Alphabetic # Lm   [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK
-FFA0..FFBE    ; Alphabetic # Lo  [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH
-FFC2..FFC7    ; Alphabetic # Lo   [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E
-FFCA..FFCF    ; Alphabetic # Lo   [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE
-FFD2..FFD7    ; Alphabetic # Lo   [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU
-FFDA..FFDC    ; Alphabetic # Lo   [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
-10000..1000B  ; Alphabetic # Lo  [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE
-1000D..10026  ; Alphabetic # Lo  [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO
-10028..1003A  ; Alphabetic # Lo  [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO
-1003C..1003D  ; Alphabetic # Lo   [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE
-1003F..1004D  ; Alphabetic # Lo  [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO
-10050..1005D  ; Alphabetic # Lo  [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089
-10080..100FA  ; Alphabetic # Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305
-10140..10174  ; Alphabetic # Nl  [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS
-10280..1029C  ; Alphabetic # Lo  [29] LYCIAN LETTER A..LYCIAN LETTER X
-102A0..102D0  ; Alphabetic # Lo  [49] CARIAN LETTER A..CARIAN LETTER UUU3
-10300..1031E  ; Alphabetic # Lo  [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU
-10330..10340  ; Alphabetic # Lo  [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA
-10341         ; Alphabetic # Nl       GOTHIC LETTER NINETY
-10342..10349  ; Alphabetic # Lo   [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL
-1034A         ; Alphabetic # Nl       GOTHIC LETTER NINE HUNDRED
-10380..1039D  ; Alphabetic # Lo  [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU
-103A0..103C3  ; Alphabetic # Lo  [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA
-103C8..103CF  ; Alphabetic # Lo   [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH
-103D1..103D5  ; Alphabetic # Nl   [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED
-10400..1044F  ; Alphabetic # L&  [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW
-10450..1049D  ; Alphabetic # Lo  [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO
-10800..10805  ; Alphabetic # Lo   [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA
-10808         ; Alphabetic # Lo       CYPRIOT SYLLABLE JO
-1080A..10835  ; Alphabetic # Lo  [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO
-10837..10838  ; Alphabetic # Lo   [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE
-1083C         ; Alphabetic # Lo       CYPRIOT SYLLABLE ZA
-1083F..10855  ; Alphabetic # Lo  [23] CYPRIOT SYLLABLE ZO..IMPERIAL ARAMAIC LETTER TAW
-10900..10915  ; Alphabetic # Lo  [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU
-10920..10939  ; Alphabetic # Lo  [26] LYDIAN LETTER A..LYDIAN LETTER C
-10A00         ; Alphabetic # Lo       KHAROSHTHI LETTER A
-10A01..10A03  ; Alphabetic # Mn   [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R
-10A05..10A06  ; Alphabetic # Mn   [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O
-10A0C..10A0F  ; Alphabetic # Mn   [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA
-10A10..10A13  ; Alphabetic # Lo   [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA
-10A15..10A17  ; Alphabetic # Lo   [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA
-10A19..10A33  ; Alphabetic # Lo  [27] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER TTTHA
-10A60..10A7C  ; Alphabetic # Lo  [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH
-10B00..10B35  ; Alphabetic # Lo  [54] AVESTAN LETTER A..AVESTAN LETTER HE
-10B40..10B55  ; Alphabetic # Lo  [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW
-10B60..10B72  ; Alphabetic # Lo  [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW
-10C00..10C48  ; Alphabetic # Lo  [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH
-11082         ; Alphabetic # Mc       KAITHI SIGN VISARGA
-11083..110AF  ; Alphabetic # Lo  [45] KAITHI LETTER A..KAITHI LETTER HA
-110B0..110B2  ; Alphabetic # Mc   [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II
-110B3..110B6  ; Alphabetic # Mn   [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI
-110B7..110B8  ; Alphabetic # Mc   [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU
-12000..1236E  ; Alphabetic # Lo [879] CUNEIFORM SIGN A..CUNEIFORM SIGN ZUM
-12400..12462  ; Alphabetic # Nl  [99] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER
-13000..1342E  ; Alphabetic # Lo [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032
-1D400..1D454  ; Alphabetic # L&  [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G
-1D456..1D49C  ; Alphabetic # L&  [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A
-1D49E..1D49F  ; Alphabetic # L&   [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D
-1D4A2         ; Alphabetic # L&       MATHEMATICAL SCRIPT CAPITAL G
-1D4A5..1D4A6  ; Alphabetic # L&   [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K
-1D4A9..1D4AC  ; Alphabetic # L&   [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q
-1D4AE..1D4B9  ; Alphabetic # L&  [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D
-1D4BB         ; Alphabetic # L&       MATHEMATICAL SCRIPT SMALL F
-1D4BD..1D4C3  ; Alphabetic # L&   [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N
-1D4C5..1D505  ; Alphabetic # L&  [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B
-1D507..1D50A  ; Alphabetic # L&   [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G
-1D50D..1D514  ; Alphabetic # L&   [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q
-1D516..1D51C  ; Alphabetic # L&   [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y
-1D51E..1D539  ; Alphabetic # L&  [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B
-1D53B..1D53E  ; Alphabetic # L&   [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G
-1D540..1D544  ; Alphabetic # L&   [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M
-1D546         ; Alphabetic # L&       MATHEMATICAL DOUBLE-STRUCK CAPITAL O
-1D54A..1D550  ; Alphabetic # L&   [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y
-1D552..1D6A5  ; Alphabetic # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J
-1D6A8..1D6C0  ; Alphabetic # L&  [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA
-1D6C2..1D6DA  ; Alphabetic # L&  [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA
-1D6DC..1D6FA  ; Alphabetic # L&  [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA
-1D6FC..1D714  ; Alphabetic # L&  [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA
-1D716..1D734  ; Alphabetic # L&  [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA
-1D736..1D74E  ; Alphabetic # L&  [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA
-1D750..1D76E  ; Alphabetic # L&  [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA
-1D770..1D788  ; Alphabetic # L&  [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA
-1D78A..1D7A8  ; Alphabetic # L&  [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA
-1D7AA..1D7C2  ; Alphabetic # L&  [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA
-1D7C4..1D7CB  ; Alphabetic # L&   [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA
-20000..2A6D6  ; Alphabetic # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6
-2A700..2B734  ; Alphabetic # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734
-2F800..2FA1D  ; Alphabetic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
-
-# Total code points: 100520
-
-# ================================================
-
-# Derived Property: Lowercase
-#  Generated from: Ll + Other_Lowercase
-
-0061..007A    ; Lowercase # L&  [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z
-00AA          ; Lowercase # L&       FEMININE ORDINAL INDICATOR
-00B5          ; Lowercase # L&       MICRO SIGN
-00BA          ; Lowercase # L&       MASCULINE ORDINAL INDICATOR
-00DF..00F6    ; Lowercase # L&  [24] LATIN SMALL LETTER SHARP S..LATIN SMALL LETTER O WITH DIAERESIS
-00F8..00FF    ; Lowercase # L&   [8] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER Y WITH DIAERESIS
-0101          ; Lowercase # L&       LATIN SMALL LETTER A WITH MACRON
-0103          ; Lowercase # L&       LATIN SMALL LETTER A WITH BREVE
-0105          ; Lowercase # L&       LATIN SMALL LETTER A WITH OGONEK
-0107          ; Lowercase # L&       LATIN SMALL LETTER C WITH ACUTE
-0109          ; Lowercase # L&       LATIN SMALL LETTER C WITH CIRCUMFLEX
-010B          ; Lowercase # L&       LATIN SMALL LETTER C WITH DOT ABOVE
-010D          ; Lowercase # L&       LATIN SMALL LETTER C WITH CARON
-010F          ; Lowercase # L&       LATIN SMALL LETTER D WITH CARON
-0111          ; Lowercase # L&       LATIN SMALL LETTER D WITH STROKE
-0113          ; Lowercase # L&       LATIN SMALL LETTER E WITH MACRON
-0115          ; Lowercase # L&       LATIN SMALL LETTER E WITH BREVE
-0117          ; Lowercase # L&       LATIN SMALL LETTER E WITH DOT ABOVE
-0119          ; Lowercase # L&       LATIN SMALL LETTER E WITH OGONEK
-011B          ; Lowercase # L&       LATIN SMALL LETTER E WITH CARON
-011D          ; Lowercase # L&       LATIN SMALL LETTER G WITH CIRCUMFLEX
-011F          ; Lowercase # L&       LATIN SMALL LETTER G WITH BREVE
-0121          ; Lowercase # L&       LATIN SMALL LETTER G WITH DOT ABOVE
-0123          ; Lowercase # L&       LATIN SMALL LETTER G WITH CEDILLA
-0125          ; Lowercase # L&       LATIN SMALL LETTER H WITH CIRCUMFLEX
-0127          ; Lowercase # L&       LATIN SMALL LETTER H WITH STROKE
-0129          ; Lowercase # L&       LATIN SMALL LETTER I WITH TILDE
-012B          ; Lowercase # L&       LATIN SMALL LETTER I WITH MACRON
-012D          ; Lowercase # L&       LATIN SMALL LETTER I WITH BREVE
-012F          ; Lowercase # L&       LATIN SMALL LETTER I WITH OGONEK
-0131          ; Lowercase # L&       LATIN SMALL LETTER DOTLESS I
-0133          ; Lowercase # L&       LATIN SMALL LIGATURE IJ
-0135          ; Lowercase # L&       LATIN SMALL LETTER J WITH CIRCUMFLEX
-0137..0138    ; Lowercase # L&   [2] LATIN SMALL LETTER K WITH CEDILLA..LATIN SMALL LETTER KRA
-013A          ; Lowercase # L&       LATIN SMALL LETTER L WITH ACUTE
-013C          ; Lowercase # L&       LATIN SMALL LETTER L WITH CEDILLA
-013E          ; Lowercase # L&       LATIN SMALL LETTER L WITH CARON
-0140          ; Lowercase # L&       LATIN SMALL LETTER L WITH MIDDLE DOT
-0142          ; Lowercase # L&       LATIN SMALL LETTER L WITH STROKE
-0144          ; Lowercase # L&       LATIN SMALL LETTER N WITH ACUTE
-0146          ; Lowercase # L&       LATIN SMALL LETTER N WITH CEDILLA
-0148..0149    ; Lowercase # L&   [2] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
-014B          ; Lowercase # L&       LATIN SMALL LETTER ENG
-014D          ; Lowercase # L&       LATIN SMALL LETTER O WITH MACRON
-014F          ; Lowercase # L&       LATIN SMALL LETTER O WITH BREVE
-0151          ; Lowercase # L&       LATIN SMALL LETTER O WITH DOUBLE ACUTE
-0153          ; Lowercase # L&       LATIN SMALL LIGATURE OE
-0155          ; Lowercase # L&       LATIN SMALL LETTER R WITH ACUTE
-0157          ; Lowercase # L&       LATIN SMALL LETTER R WITH CEDILLA
-0159          ; Lowercase # L&       LATIN SMALL LETTER R WITH CARON
-015B          ; Lowercase # L&       LATIN SMALL LETTER S WITH ACUTE
-015D          ; Lowercase # L&       LATIN SMALL LETTER S WITH CIRCUMFLEX
-015F          ; Lowercase # L&       LATIN SMALL LETTER S WITH CEDILLA
-0161          ; Lowercase # L&       LATIN SMALL LETTER S WITH CARON
-0163          ; Lowercase # L&       LATIN SMALL LETTER T WITH CEDILLA
-0165          ; Lowercase # L&       LATIN SMALL LETTER T WITH CARON
-0167          ; Lowercase # L&       LATIN SMALL LETTER T WITH STROKE
-0169          ; Lowercase # L&       LATIN SMALL LETTER U WITH TILDE
-016B          ; Lowercase # L&       LATIN SMALL LETTER U WITH MACRON
-016D          ; Lowercase # L&       LATIN SMALL LETTER U WITH BREVE
-016F          ; Lowercase # L&       LATIN SMALL LETTER U WITH RING ABOVE
-0171          ; Lowercase # L&       LATIN SMALL LETTER U WITH DOUBLE ACUTE
-0173          ; Lowercase # L&       LATIN SMALL LETTER U WITH OGONEK
-0175          ; Lowercase # L&       LATIN SMALL LETTER W WITH CIRCUMFLEX
-0177          ; Lowercase # L&       LATIN SMALL LETTER Y WITH CIRCUMFLEX
-017A          ; Lowercase # L&       LATIN SMALL LETTER Z WITH ACUTE
-017C          ; Lowercase # L&       LATIN SMALL LETTER Z WITH DOT ABOVE
-017E..0180    ; Lowercase # L&   [3] LATIN SMALL LETTER Z WITH CARON..LATIN SMALL LETTER B WITH STROKE
-0183          ; Lowercase # L&       LATIN SMALL LETTER B WITH TOPBAR
-0185          ; Lowercase # L&       LATIN SMALL LETTER TONE SIX
-0188          ; Lowercase # L&       LATIN SMALL LETTER C WITH HOOK
-018C..018D    ; Lowercase # L&   [2] LATIN SMALL LETTER D WITH TOPBAR..LATIN SMALL LETTER TURNED DELTA
-0192          ; Lowercase # L&       LATIN SMALL LETTER F WITH HOOK
-0195          ; Lowercase # L&       LATIN SMALL LETTER HV
-0199..019B    ; Lowercase # L&   [3] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER LAMBDA WITH STROKE
-019E          ; Lowercase # L&       LATIN SMALL LETTER N WITH LONG RIGHT LEG
-01A1          ; Lowercase # L&       LATIN SMALL LETTER O WITH HORN
-01A3          ; Lowercase # L&       LATIN SMALL LETTER OI
-01A5          ; Lowercase # L&       LATIN SMALL LETTER P WITH HOOK
-01A8          ; Lowercase # L&       LATIN SMALL LETTER TONE TWO
-01AA..01AB    ; Lowercase # L&   [2] LATIN LETTER REVERSED ESH LOOP..LATIN SMALL LETTER T WITH PALATAL HOOK
-01AD          ; Lowercase # L&       LATIN SMALL LETTER T WITH HOOK
-01B0          ; Lowercase # L&       LATIN SMALL LETTER U WITH HORN
-01B4          ; Lowercase # L&       LATIN SMALL LETTER Y WITH HOOK
-01B6          ; Lowercase # L&       LATIN SMALL LETTER Z WITH STROKE
-01B9..01BA    ; Lowercase # L&   [2] LATIN SMALL LETTER EZH REVERSED..LATIN SMALL LETTER EZH WITH TAIL
-01BD..01BF    ; Lowercase # L&   [3] LATIN SMALL LETTER TONE FIVE..LATIN LETTER WYNN
-01C6          ; Lowercase # L&       LATIN SMALL LETTER DZ WITH CARON
-01C9          ; Lowercase # L&       LATIN SMALL LETTER LJ
-01CC          ; Lowercase # L&       LATIN SMALL LETTER NJ
-01CE          ; Lowercase # L&       LATIN SMALL LETTER A WITH CARON
-01D0          ; Lowercase # L&       LATIN SMALL LETTER I WITH CARON
-01D2          ; Lowercase # L&       LATIN SMALL LETTER O WITH CARON
-01D4          ; Lowercase # L&       LATIN SMALL LETTER U WITH CARON
-01D6          ; Lowercase # L&       LATIN SMALL LETTER U WITH DIAERESIS AND MACRON
-01D8          ; Lowercase # L&       LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE
-01DA          ; Lowercase # L&       LATIN SMALL LETTER U WITH DIAERESIS AND CARON
-01DC..01DD    ; Lowercase # L&   [2] LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE..LATIN SMALL LETTER TURNED E
-01DF          ; Lowercase # L&       LATIN SMALL LETTER A WITH DIAERESIS AND MACRON
-01E1          ; Lowercase # L&       LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON
-01E3          ; Lowercase # L&       LATIN SMALL LETTER AE WITH MACRON
-01E5          ; Lowercase # L&       LATIN SMALL LETTER G WITH STROKE
-01E7          ; Lowercase # L&       LATIN SMALL LETTER G WITH CARON
-01E9          ; Lowercase # L&       LATIN SMALL LETTER K WITH CARON
-01EB          ; Lowercase # L&       LATIN SMALL LETTER O WITH OGONEK
-01ED          ; Lowercase # L&       LATIN SMALL LETTER O WITH OGONEK AND MACRON
-01EF..01F0    ; Lowercase # L&   [2] LATIN SMALL LETTER EZH WITH CARON..LATIN SMALL LETTER J WITH CARON
-01F3          ; Lowercase # L&       LATIN SMALL LETTER DZ
-01F5          ; Lowercase # L&       LATIN SMALL LETTER G WITH ACUTE
-01F9          ; Lowercase # L&       LATIN SMALL LETTER N WITH GRAVE
-01FB          ; Lowercase # L&       LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE
-01FD          ; Lowercase # L&       LATIN SMALL LETTER AE WITH ACUTE
-01FF          ; Lowercase # L&       LATIN SMALL LETTER O WITH STROKE AND ACUTE
-0201          ; Lowercase # L&       LATIN SMALL LETTER A WITH DOUBLE GRAVE
-0203          ; Lowercase # L&       LATIN SMALL LETTER A WITH INVERTED BREVE
-0205          ; Lowercase # L&       LATIN SMALL LETTER E WITH DOUBLE GRAVE
-0207          ; Lowercase # L&       LATIN SMALL LETTER E WITH INVERTED BREVE
-0209          ; Lowercase # L&       LATIN SMALL LETTER I WITH DOUBLE GRAVE
-020B          ; Lowercase # L&       LATIN SMALL LETTER I WITH INVERTED BREVE
-020D          ; Lowercase # L&       LATIN SMALL LETTER O WITH DOUBLE GRAVE
-020F          ; Lowercase # L&       LATIN SMALL LETTER O WITH INVERTED BREVE
-0211          ; Lowercase # L&       LATIN SMALL LETTER R WITH DOUBLE GRAVE
-0213          ; Lowercase # L&       LATIN SMALL LETTER R WITH INVERTED BREVE
-0215          ; Lowercase # L&       LATIN SMALL LETTER U WITH DOUBLE GRAVE
-0217          ; Lowercase # L&       LATIN SMALL LETTER U WITH INVERTED BREVE
-0219          ; Lowercase # L&       LATIN SMALL LETTER S WITH COMMA BELOW
-021B          ; Lowercase # L&       LATIN SMALL LETTER T WITH COMMA BELOW
-021D          ; Lowercase # L&       LATIN SMALL LETTER YOGH
-021F          ; Lowercase # L&       LATIN SMALL LETTER H WITH CARON
-0221          ; Lowercase # L&       LATIN SMALL LETTER D WITH CURL
-0223          ; Lowercase # L&       LATIN SMALL LETTER OU
-0225          ; Lowercase # L&       LATIN SMALL LETTER Z WITH HOOK
-0227          ; Lowercase # L&       LATIN SMALL LETTER A WITH DOT ABOVE
-0229          ; Lowercase # L&       LATIN SMALL LETTER E WITH CEDILLA
-022B          ; Lowercase # L&       LATIN SMALL LETTER O WITH DIAERESIS AND MACRON
-022D          ; Lowercase # L&       LATIN SMALL LETTER O WITH TILDE AND MACRON
-022F          ; Lowercase # L&       LATIN SMALL LETTER O WITH DOT ABOVE
-0231          ; Lowercase # L&       LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON
-0233..0239    ; Lowercase # L&   [7] LATIN SMALL LETTER Y WITH MACRON..LATIN SMALL LETTER QP DIGRAPH
-023C          ; Lowercase # L&       LATIN SMALL LETTER C WITH STROKE
-023F..0240    ; Lowercase # L&   [2] LATIN SMALL LETTER S WITH SWASH TAIL..LATIN SMALL LETTER Z WITH SWASH TAIL
-0242          ; Lowercase # L&       LATIN SMALL LETTER GLOTTAL STOP
-0247          ; Lowercase # L&       LATIN SMALL LETTER E WITH STROKE
-0249          ; Lowercase # L&       LATIN SMALL LETTER J WITH STROKE
-024B          ; Lowercase # L&       LATIN SMALL LETTER Q WITH HOOK TAIL
-024D          ; Lowercase # L&       LATIN SMALL LETTER R WITH STROKE
-024F..0293    ; Lowercase # L&  [69] LATIN SMALL LETTER Y WITH STROKE..LATIN SMALL LETTER EZH WITH CURL
-0295..02AF    ; Lowercase # L&  [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL
-02B0..02B8    ; Lowercase # Lm   [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y
-02C0..02C1    ; Lowercase # Lm   [2] MODIFIER LETTER GLOTTAL STOP..MODIFIER LETTER REVERSED GLOTTAL STOP
-02E0..02E4    ; Lowercase # Lm   [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP
-0345          ; Lowercase # Mn       COMBINING GREEK YPOGEGRAMMENI
-0371          ; Lowercase # L&       GREEK SMALL LETTER HETA
-0373          ; Lowercase # L&       GREEK SMALL LETTER ARCHAIC SAMPI
-0377          ; Lowercase # L&       GREEK SMALL LETTER PAMPHYLIAN DIGAMMA
-037A          ; Lowercase # Lm       GREEK YPOGEGRAMMENI
-037B..037D    ; Lowercase # L&   [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL
-0390          ; Lowercase # L&       GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
-03AC..03CE    ; Lowercase # L&  [35] GREEK SMALL LETTER ALPHA WITH TONOS..GREEK SMALL LETTER OMEGA WITH TONOS
-03D0..03D1    ; Lowercase # L&   [2] GREEK BETA SYMBOL..GREEK THETA SYMBOL
-03D5..03D7    ; Lowercase # L&   [3] GREEK PHI SYMBOL..GREEK KAI SYMBOL
-03D9          ; Lowercase # L&       GREEK SMALL LETTER ARCHAIC KOPPA
-03DB          ; Lowercase # L&       GREEK SMALL LETTER STIGMA
-03DD          ; Lowercase # L&       GREEK SMALL LETTER DIGAMMA
-03DF          ; Lowercase # L&       GREEK SMALL LETTER KOPPA
-03E1          ; Lowercase # L&       GREEK SMALL LETTER SAMPI
-03E3          ; Lowercase # L&       COPTIC SMALL LETTER SHEI
-03E5          ; Lowercase # L&       COPTIC SMALL LETTER FEI
-03E7          ; Lowercase # L&       COPTIC SMALL LETTER KHEI
-03E9          ; Lowercase # L&       COPTIC SMALL LETTER HORI
-03EB          ; Lowercase # L&       COPTIC SMALL LETTER GANGIA
-03ED          ; Lowercase # L&       COPTIC SMALL LETTER SHIMA
-03EF..03F3    ; Lowercase # L&   [5] COPTIC SMALL LETTER DEI..GREEK LETTER YOT
-03F5          ; Lowercase # L&       GREEK LUNATE EPSILON SYMBOL
-03F8          ; Lowercase # L&       GREEK SMALL LETTER SHO
-03FB..03FC    ; Lowercase # L&   [2] GREEK SMALL LETTER SAN..GREEK RHO WITH STROKE SYMBOL
-0430..045F    ; Lowercase # L&  [48] CYRILLIC SMALL LETTER A..CYRILLIC SMALL LETTER DZHE
-0461          ; Lowercase # L&       CYRILLIC SMALL LETTER OMEGA
-0463          ; Lowercase # L&       CYRILLIC SMALL LETTER YAT
-0465          ; Lowercase # L&       CYRILLIC SMALL LETTER IOTIFIED E
-0467          ; Lowercase # L&       CYRILLIC SMALL LETTER LITTLE YUS
-0469          ; Lowercase # L&       CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS
-046B          ; Lowercase # L&       CYRILLIC SMALL LETTER BIG YUS
-046D          ; Lowercase # L&       CYRILLIC SMALL LETTER IOTIFIED BIG YUS
-046F          ; Lowercase # L&       CYRILLIC SMALL LETTER KSI
-0471          ; Lowercase # L&       CYRILLIC SMALL LETTER PSI
-0473          ; Lowercase # L&       CYRILLIC SMALL LETTER FITA
-0475          ; Lowercase # L&       CYRILLIC SMALL LETTER IZHITSA
-0477          ; Lowercase # L&       CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
-0479          ; Lowercase # L&       CYRILLIC SMALL LETTER UK
-047B          ; Lowercase # L&       CYRILLIC SMALL LETTER ROUND OMEGA
-047D          ; Lowercase # L&       CYRILLIC SMALL LETTER OMEGA WITH TITLO
-047F          ; Lowercase # L&       CYRILLIC SMALL LETTER OT
-0481          ; Lowercase # L&       CYRILLIC SMALL LETTER KOPPA
-048B          ; Lowercase # L&       CYRILLIC SMALL LETTER SHORT I WITH TAIL
-048D          ; Lowercase # L&       CYRILLIC SMALL LETTER SEMISOFT SIGN
-048F          ; Lowercase # L&       CYRILLIC SMALL LETTER ER WITH TICK
-0491          ; Lowercase # L&       CYRILLIC SMALL LETTER GHE WITH UPTURN
-0493          ; Lowercase # L&       CYRILLIC SMALL LETTER GHE WITH STROKE
-0495          ; Lowercase # L&       CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK
-0497          ; Lowercase # L&       CYRILLIC SMALL LETTER ZHE WITH DESCENDER
-0499          ; Lowercase # L&       CYRILLIC SMALL LETTER ZE WITH DESCENDER
-049B          ; Lowercase # L&       CYRILLIC SMALL LETTER KA WITH DESCENDER
-049D          ; Lowercase # L&       CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE
-049F          ; Lowercase # L&       CYRILLIC SMALL LETTER KA WITH STROKE
-04A1          ; Lowercase # L&       CYRILLIC SMALL LETTER BASHKIR KA
-04A3          ; Lowercase # L&       CYRILLIC SMALL LETTER EN WITH DESCENDER
-04A5          ; Lowercase # L&       CYRILLIC SMALL LIGATURE EN GHE
-04A7          ; Lowercase # L&       CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK
-04A9          ; Lowercase # L&       CYRILLIC SMALL LETTER ABKHASIAN HA
-04AB          ; Lowercase # L&       CYRILLIC SMALL LETTER ES WITH DESCENDER
-04AD          ; Lowercase # L&       CYRILLIC SMALL LETTER TE WITH DESCENDER
-04AF          ; Lowercase # L&       CYRILLIC SMALL LETTER STRAIGHT U
-04B1          ; Lowercase # L&       CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE
-04B3          ; Lowercase # L&       CYRILLIC SMALL LETTER HA WITH DESCENDER
-04B5          ; Lowercase # L&       CYRILLIC SMALL LIGATURE TE TSE
-04B7          ; Lowercase # L&       CYRILLIC SMALL LETTER CHE WITH DESCENDER
-04B9          ; Lowercase # L&       CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE
-04BB          ; Lowercase # L&       CYRILLIC SMALL LETTER SHHA
-04BD          ; Lowercase # L&       CYRILLIC SMALL LETTER ABKHASIAN CHE
-04BF          ; Lowercase # L&       CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER
-04C2          ; Lowercase # L&       CYRILLIC SMALL LETTER ZHE WITH BREVE
-04C4          ; Lowercase # L&       CYRILLIC SMALL LETTER KA WITH HOOK
-04C6          ; Lowercase # L&       CYRILLIC SMALL LETTER EL WITH TAIL
-04C8          ; Lowercase # L&       CYRILLIC SMALL LETTER EN WITH HOOK
-04CA          ; Lowercase # L&       CYRILLIC SMALL LETTER EN WITH TAIL
-04CC          ; Lowercase # L&       CYRILLIC SMALL LETTER KHAKASSIAN CHE
-04CE..04CF    ; Lowercase # L&   [2] CYRILLIC SMALL LETTER EM WITH TAIL..CYRILLIC SMALL LETTER PALOCHKA
-04D1          ; Lowercase # L&       CYRILLIC SMALL LETTER A WITH BREVE
-04D3          ; Lowercase # L&       CYRILLIC SMALL LETTER A WITH DIAERESIS
-04D5          ; Lowercase # L&       CYRILLIC SMALL LIGATURE A IE
-04D7          ; Lowercase # L&       CYRILLIC SMALL LETTER IE WITH BREVE
-04D9          ; Lowercase # L&       CYRILLIC SMALL LETTER SCHWA
-04DB          ; Lowercase # L&       CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS
-04DD          ; Lowercase # L&       CYRILLIC SMALL LETTER ZHE WITH DIAERESIS
-04DF          ; Lowercase # L&       CYRILLIC SMALL LETTER ZE WITH DIAERESIS
-04E1          ; Lowercase # L&       CYRILLIC SMALL LETTER ABKHASIAN DZE
-04E3          ; Lowercase # L&       CYRILLIC SMALL LETTER I WITH MACRON
-04E5          ; Lowercase # L&       CYRILLIC SMALL LETTER I WITH DIAERESIS
-04E7          ; Lowercase # L&       CYRILLIC SMALL LETTER O WITH DIAERESIS
-04E9          ; Lowercase # L&       CYRILLIC SMALL LETTER BARRED O
-04EB          ; Lowercase # L&       CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS
-04ED          ; Lowercase # L&       CYRILLIC SMALL LETTER E WITH DIAERESIS
-04EF          ; Lowercase # L&       CYRILLIC SMALL LETTER U WITH MACRON
-04F1          ; Lowercase # L&       CYRILLIC SMALL LETTER U WITH DIAERESIS
-04F3          ; Lowercase # L&       CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE
-04F5          ; Lowercase # L&       CYRILLIC SMALL LETTER CHE WITH DIAERESIS
-04F7          ; Lowercase # L&       CYRILLIC SMALL LETTER GHE WITH DESCENDER
-04F9          ; Lowercase # L&       CYRILLIC SMALL LETTER YERU WITH DIAERESIS
-04FB          ; Lowercase # L&       CYRILLIC SMALL LETTER GHE WITH STROKE AND HOOK
-04FD          ; Lowercase # L&       CYRILLIC SMALL LETTER HA WITH HOOK
-04FF          ; Lowercase # L&       CYRILLIC SMALL LETTER HA WITH STROKE
-0501          ; Lowercase # L&       CYRILLIC SMALL LETTER KOMI DE
-0503          ; Lowercase # L&       CYRILLIC SMALL LETTER KOMI DJE
-0505          ; Lowercase # L&       CYRILLIC SMALL LETTER KOMI ZJE
-0507          ; Lowercase # L&       CYRILLIC SMALL LETTER KOMI DZJE
-0509          ; Lowercase # L&       CYRILLIC SMALL LETTER KOMI LJE
-050B          ; Lowercase # L&       CYRILLIC SMALL LETTER KOMI NJE
-050D          ; Lowercase # L&       CYRILLIC SMALL LETTER KOMI SJE
-050F          ; Lowercase # L&       CYRILLIC SMALL LETTER KOMI TJE
-0511          ; Lowercase # L&       CYRILLIC SMALL LETTER REVERSED ZE
-0513          ; Lowercase # L&       CYRILLIC SMALL LETTER EL WITH HOOK
-0515          ; Lowercase # L&       CYRILLIC SMALL LETTER LHA
-0517          ; Lowercase # L&       CYRILLIC SMALL LETTER RHA
-0519          ; Lowercase # L&       CYRILLIC SMALL LETTER YAE
-051B          ; Lowercase # L&       CYRILLIC SMALL LETTER QA
-051D          ; Lowercase # L&       CYRILLIC SMALL LETTER WE
-051F          ; Lowercase # L&       CYRILLIC SMALL LETTER ALEUT KA
-0521          ; Lowercase # L&       CYRILLIC SMALL LETTER EL WITH MIDDLE HOOK
-0523          ; Lowercase # L&       CYRILLIC SMALL LETTER EN WITH MIDDLE HOOK
-0525          ; Lowercase # L&       CYRILLIC SMALL LETTER PE WITH DESCENDER
-0561..0587    ; Lowercase # L&  [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN
-1D00..1D2B    ; Lowercase # L&  [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL
-1D2C..1D61    ; Lowercase # Lm  [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI
-1D62..1D77    ; Lowercase # L&  [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G
-1D78          ; Lowercase # Lm       MODIFIER LETTER CYRILLIC EN
-1D79..1D9A    ; Lowercase # L&  [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK
-1D9B..1DBF    ; Lowercase # Lm  [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA
-1E01          ; Lowercase # L&       LATIN SMALL LETTER A WITH RING BELOW
-1E03          ; Lowercase # L&       LATIN SMALL LETTER B WITH DOT ABOVE
-1E05          ; Lowercase # L&       LATIN SMALL LETTER B WITH DOT BELOW
-1E07          ; Lowercase # L&       LATIN SMALL LETTER B WITH LINE BELOW
-1E09          ; Lowercase # L&       LATIN SMALL LETTER C WITH CEDILLA AND ACUTE
-1E0B          ; Lowercase # L&       LATIN SMALL LETTER D WITH DOT ABOVE
-1E0D          ; Lowercase # L&       LATIN SMALL LETTER D WITH DOT BELOW
-1E0F          ; Lowercase # L&       LATIN SMALL LETTER D WITH LINE BELOW
-1E11          ; Lowercase # L&       LATIN SMALL LETTER D WITH CEDILLA
-1E13          ; Lowercase # L&       LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW
-1E15          ; Lowercase # L&       LATIN SMALL LETTER E WITH MACRON AND GRAVE
-1E17          ; Lowercase # L&       LATIN SMALL LETTER E WITH MACRON AND ACUTE
-1E19          ; Lowercase # L&       LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW
-1E1B          ; Lowercase # L&       LATIN SMALL LETTER E WITH TILDE BELOW
-1E1D          ; Lowercase # L&       LATIN SMALL LETTER E WITH CEDILLA AND BREVE
-1E1F          ; Lowercase # L&       LATIN SMALL LETTER F WITH DOT ABOVE
-1E21          ; Lowercase # L&       LATIN SMALL LETTER G WITH MACRON
-1E23          ; Lowercase # L&       LATIN SMALL LETTER H WITH DOT ABOVE
-1E25          ; Lowercase # L&       LATIN SMALL LETTER H WITH DOT BELOW
-1E27          ; Lowercase # L&       LATIN SMALL LETTER H WITH DIAERESIS
-1E29          ; Lowercase # L&       LATIN SMALL LETTER H WITH CEDILLA
-1E2B          ; Lowercase # L&       LATIN SMALL LETTER H WITH BREVE BELOW
-1E2D          ; Lowercase # L&       LATIN SMALL LETTER I WITH TILDE BELOW
-1E2F          ; Lowercase # L&       LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE
-1E31          ; Lowercase # L&       LATIN SMALL LETTER K WITH ACUTE
-1E33          ; Lowercase # L&       LATIN SMALL LETTER K WITH DOT BELOW
-1E35          ; Lowercase # L&       LATIN SMALL LETTER K WITH LINE BELOW
-1E37          ; Lowercase # L&       LATIN SMALL LETTER L WITH DOT BELOW
-1E39          ; Lowercase # L&       LATIN SMALL LETTER L WITH DOT BELOW AND MACRON
-1E3B          ; Lowercase # L&       LATIN SMALL LETTER L WITH LINE BELOW
-1E3D          ; Lowercase # L&       LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW
-1E3F          ; Lowercase # L&       LATIN SMALL LETTER M WITH ACUTE
-1E41          ; Lowercase # L&       LATIN SMALL LETTER M WITH DOT ABOVE
-1E43          ; Lowercase # L&       LATIN SMALL LETTER M WITH DOT BELOW
-1E45          ; Lowercase # L&       LATIN SMALL LETTER N WITH DOT ABOVE
-1E47          ; Lowercase # L&       LATIN SMALL LETTER N WITH DOT BELOW
-1E49          ; Lowercase # L&       LATIN SMALL LETTER N WITH LINE BELOW
-1E4B          ; Lowercase # L&       LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW
-1E4D          ; Lowercase # L&       LATIN SMALL LETTER O WITH TILDE AND ACUTE
-1E4F          ; Lowercase # L&       LATIN SMALL LETTER O WITH TILDE AND DIAERESIS
-1E51          ; Lowercase # L&       LATIN SMALL LETTER O WITH MACRON AND GRAVE
-1E53          ; Lowercase # L&       LATIN SMALL LETTER O WITH MACRON AND ACUTE
-1E55          ; Lowercase # L&       LATIN SMALL LETTER P WITH ACUTE
-1E57          ; Lowercase # L&       LATIN SMALL LETTER P WITH DOT ABOVE
-1E59          ; Lowercase # L&       LATIN SMALL LETTER R WITH DOT ABOVE
-1E5B          ; Lowercase # L&       LATIN SMALL LETTER R WITH DOT BELOW
-1E5D          ; Lowercase # L&       LATIN SMALL LETTER R WITH DOT BELOW AND MACRON
-1E5F          ; Lowercase # L&       LATIN SMALL LETTER R WITH LINE BELOW
-1E61          ; Lowercase # L&       LATIN SMALL LETTER S WITH DOT ABOVE
-1E63          ; Lowercase # L&       LATIN SMALL LETTER S WITH DOT BELOW
-1E65          ; Lowercase # L&       LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE
-1E67          ; Lowercase # L&       LATIN SMALL LETTER S WITH CARON AND DOT ABOVE
-1E69          ; Lowercase # L&       LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE
-1E6B          ; Lowercase # L&       LATIN SMALL LETTER T WITH DOT ABOVE
-1E6D          ; Lowercase # L&       LATIN SMALL LETTER T WITH DOT BELOW
-1E6F          ; Lowercase # L&       LATIN SMALL LETTER T WITH LINE BELOW
-1E71          ; Lowercase # L&       LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW
-1E73          ; Lowercase # L&       LATIN SMALL LETTER U WITH DIAERESIS BELOW
-1E75          ; Lowercase # L&       LATIN SMALL LETTER U WITH TILDE BELOW
-1E77          ; Lowercase # L&       LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW
-1E79          ; Lowercase # L&       LATIN SMALL LETTER U WITH TILDE AND ACUTE
-1E7B          ; Lowercase # L&       LATIN SMALL LETTER U WITH MACRON AND DIAERESIS
-1E7D          ; Lowercase # L&       LATIN SMALL LETTER V WITH TILDE
-1E7F          ; Lowercase # L&       LATIN SMALL LETTER V WITH DOT BELOW
-1E81          ; Lowercase # L&       LATIN SMALL LETTER W WITH GRAVE
-1E83          ; Lowercase # L&       LATIN SMALL LETTER W WITH ACUTE
-1E85          ; Lowercase # L&       LATIN SMALL LETTER W WITH DIAERESIS
-1E87          ; Lowercase # L&       LATIN SMALL LETTER W WITH DOT ABOVE
-1E89          ; Lowercase # L&       LATIN SMALL LETTER W WITH DOT BELOW
-1E8B          ; Lowercase # L&       LATIN SMALL LETTER X WITH DOT ABOVE
-1E8D          ; Lowercase # L&       LATIN SMALL LETTER X WITH DIAERESIS
-1E8F          ; Lowercase # L&       LATIN SMALL LETTER Y WITH DOT ABOVE
-1E91          ; Lowercase # L&       LATIN SMALL LETTER Z WITH CIRCUMFLEX
-1E93          ; Lowercase # L&       LATIN SMALL LETTER Z WITH DOT BELOW
-1E95..1E9D    ; Lowercase # L&   [9] LATIN SMALL LETTER Z WITH LINE BELOW..LATIN SMALL LETTER LONG S WITH HIGH STROKE
-1E9F          ; Lowercase # L&       LATIN SMALL LETTER DELTA
-1EA1          ; Lowercase # L&       LATIN SMALL LETTER A WITH DOT BELOW
-1EA3          ; Lowercase # L&       LATIN SMALL LETTER A WITH HOOK ABOVE
-1EA5          ; Lowercase # L&       LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE
-1EA7          ; Lowercase # L&       LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE
-1EA9          ; Lowercase # L&       LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
-1EAB          ; Lowercase # L&       LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE
-1EAD          ; Lowercase # L&       LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW
-1EAF          ; Lowercase # L&       LATIN SMALL LETTER A WITH BREVE AND ACUTE
-1EB1          ; Lowercase # L&       LATIN SMALL LETTER A WITH BREVE AND GRAVE
-1EB3          ; Lowercase # L&       LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE
-1EB5          ; Lowercase # L&       LATIN SMALL LETTER A WITH BREVE AND TILDE
-1EB7          ; Lowercase # L&       LATIN SMALL LETTER A WITH BREVE AND DOT BELOW
-1EB9          ; Lowercase # L&       LATIN SMALL LETTER E WITH DOT BELOW
-1EBB          ; Lowercase # L&       LATIN SMALL LETTER E WITH HOOK ABOVE
-1EBD          ; Lowercase # L&       LATIN SMALL LETTER E WITH TILDE
-1EBF          ; Lowercase # L&       LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE
-1EC1          ; Lowercase # L&       LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE
-1EC3          ; Lowercase # L&       LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
-1EC5          ; Lowercase # L&       LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE
-1EC7          ; Lowercase # L&       LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW
-1EC9          ; Lowercase # L&       LATIN SMALL LETTER I WITH HOOK ABOVE
-1ECB          ; Lowercase # L&       LATIN SMALL LETTER I WITH DOT BELOW
-1ECD          ; Lowercase # L&       LATIN SMALL LETTER O WITH DOT BELOW
-1ECF          ; Lowercase # L&       LATIN SMALL LETTER O WITH HOOK ABOVE
-1ED1          ; Lowercase # L&       LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE
-1ED3          ; Lowercase # L&       LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE
-1ED5          ; Lowercase # L&       LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
-1ED7          ; Lowercase # L&       LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE
-1ED9          ; Lowercase # L&       LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW
-1EDB          ; Lowercase # L&       LATIN SMALL LETTER O WITH HORN AND ACUTE
-1EDD          ; Lowercase # L&       LATIN SMALL LETTER O WITH HORN AND GRAVE
-1EDF          ; Lowercase # L&       LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE
-1EE1          ; Lowercase # L&       LATIN SMALL LETTER O WITH HORN AND TILDE
-1EE3          ; Lowercase # L&       LATIN SMALL LETTER O WITH HORN AND DOT BELOW
-1EE5          ; Lowercase # L&       LATIN SMALL LETTER U WITH DOT BELOW
-1EE7          ; Lowercase # L&       LATIN SMALL LETTER U WITH HOOK ABOVE
-1EE9          ; Lowercase # L&       LATIN SMALL LETTER U WITH HORN AND ACUTE
-1EEB          ; Lowercase # L&       LATIN SMALL LETTER U WITH HORN AND GRAVE
-1EED          ; Lowercase # L&       LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE
-1EEF          ; Lowercase # L&       LATIN SMALL LETTER U WITH HORN AND TILDE
-1EF1          ; Lowercase # L&       LATIN SMALL LETTER U WITH HORN AND DOT BELOW
-1EF3          ; Lowercase # L&       LATIN SMALL LETTER Y WITH GRAVE
-1EF5          ; Lowercase # L&       LATIN SMALL LETTER Y WITH DOT BELOW
-1EF7          ; Lowercase # L&       LATIN SMALL LETTER Y WITH HOOK ABOVE
-1EF9          ; Lowercase # L&       LATIN SMALL LETTER Y WITH TILDE
-1EFB          ; Lowercase # L&       LATIN SMALL LETTER MIDDLE-WELSH LL
-1EFD          ; Lowercase # L&       LATIN SMALL LETTER MIDDLE-WELSH V
-1EFF..1F07    ; Lowercase # L&   [9] LATIN SMALL LETTER Y WITH LOOP..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI
-1F10..1F15    ; Lowercase # L&   [6] GREEK SMALL LETTER EPSILON WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA
-1F20..1F27    ; Lowercase # L&   [8] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI
-1F30..1F37    ; Lowercase # L&   [8] GREEK SMALL LETTER IOTA WITH PSILI..GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI
-1F40..1F45    ; Lowercase # L&   [6] GREEK SMALL LETTER OMICRON WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA
-1F50..1F57    ; Lowercase # L&   [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI
-1F60..1F67    ; Lowercase # L&   [8] GREEK SMALL LETTER OMEGA WITH PSILI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI
-1F70..1F7D    ; Lowercase # L&  [14] GREEK SMALL LETTER ALPHA WITH VARIA..GREEK SMALL LETTER OMEGA WITH OXIA
-1F80..1F87    ; Lowercase # L&   [8] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
-1F90..1F97    ; Lowercase # L&   [8] GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
-1FA0..1FA7    ; Lowercase # L&   [8] GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
-1FB0..1FB4    ; Lowercase # L&   [5] GREEK SMALL LETTER ALPHA WITH VRACHY..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
-1FB6..1FB7    ; Lowercase # L&   [2] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
-1FBE          ; Lowercase # L&       GREEK PROSGEGRAMMENI
-1FC2..1FC4    ; Lowercase # L&   [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
-1FC6..1FC7    ; Lowercase # L&   [2] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
-1FD0..1FD3    ; Lowercase # L&   [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
-1FD6..1FD7    ; Lowercase # L&   [2] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
-1FE0..1FE7    ; Lowercase # L&   [8] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
-1FF2..1FF4    ; Lowercase # L&   [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
-1FF6..1FF7    ; Lowercase # L&   [2] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
-2090..2094    ; Lowercase # Lm   [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA
-210A          ; Lowercase # L&       SCRIPT SMALL G
-210E..210F    ; Lowercase # L&   [2] PLANCK CONSTANT..PLANCK CONSTANT OVER TWO PI
-2113          ; Lowercase # L&       SCRIPT SMALL L
-212F          ; Lowercase # L&       SCRIPT SMALL E
-2134          ; Lowercase # L&       SCRIPT SMALL O
-2139          ; Lowercase # L&       INFORMATION SOURCE
-213C..213D    ; Lowercase # L&   [2] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK SMALL GAMMA
-2146..2149    ; Lowercase # L&   [4] DOUBLE-STRUCK ITALIC SMALL D..DOUBLE-STRUCK ITALIC SMALL J
-214E          ; Lowercase # L&       TURNED SMALL F
-2170..217F    ; Lowercase # Nl  [16] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND
-2184          ; Lowercase # L&       LATIN SMALL LETTER REVERSED C
-24D0..24E9    ; Lowercase # So  [26] CIRCLED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z
-2C30..2C5E    ; Lowercase # L&  [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE
-2C61          ; Lowercase # L&       LATIN SMALL LETTER L WITH DOUBLE BAR
-2C65..2C66    ; Lowercase # L&   [2] LATIN SMALL LETTER A WITH STROKE..LATIN SMALL LETTER T WITH DIAGONAL STROKE
-2C68          ; Lowercase # L&       LATIN SMALL LETTER H WITH DESCENDER
-2C6A          ; Lowercase # L&       LATIN SMALL LETTER K WITH DESCENDER
-2C6C          ; Lowercase # L&       LATIN SMALL LETTER Z WITH DESCENDER
-2C71          ; Lowercase # L&       LATIN SMALL LETTER V WITH RIGHT HOOK
-2C73..2C74    ; Lowercase # L&   [2] LATIN SMALL LETTER W WITH HOOK..LATIN SMALL LETTER V WITH CURL
-2C76..2C7C    ; Lowercase # L&   [7] LATIN SMALL LETTER HALF H..LATIN SUBSCRIPT SMALL LETTER J
-2C7D          ; Lowercase # Lm       MODIFIER LETTER CAPITAL V
-2C81          ; Lowercase # L&       COPTIC SMALL LETTER ALFA
-2C83          ; Lowercase # L&       COPTIC SMALL LETTER VIDA
-2C85          ; Lowercase # L&       COPTIC SMALL LETTER GAMMA
-2C87          ; Lowercase # L&       COPTIC SMALL LETTER DALDA
-2C89          ; Lowercase # L&       COPTIC SMALL LETTER EIE
-2C8B          ; Lowercase # L&       COPTIC SMALL LETTER SOU
-2C8D          ; Lowercase # L&       COPTIC SMALL LETTER ZATA
-2C8F          ; Lowercase # L&       COPTIC SMALL LETTER HATE
-2C91          ; Lowercase # L&       COPTIC SMALL LETTER THETHE
-2C93          ; Lowercase # L&       COPTIC SMALL LETTER IAUDA
-2C95          ; Lowercase # L&       COPTIC SMALL LETTER KAPA
-2C97          ; Lowercase # L&       COPTIC SMALL LETTER LAULA
-2C99          ; Lowercase # L&       COPTIC SMALL LETTER MI
-2C9B          ; Lowercase # L&       COPTIC SMALL LETTER NI
-2C9D          ; Lowercase # L&       COPTIC SMALL LETTER KSI
-2C9F          ; Lowercase # L&       COPTIC SMALL LETTER O
-2CA1          ; Lowercase # L&       COPTIC SMALL LETTER PI
-2CA3          ; Lowercase # L&       COPTIC SMALL LETTER RO
-2CA5          ; Lowercase # L&       COPTIC SMALL LETTER SIMA
-2CA7          ; Lowercase # L&       COPTIC SMALL LETTER TAU
-2CA9          ; Lowercase # L&       COPTIC SMALL LETTER UA
-2CAB          ; Lowercase # L&       COPTIC SMALL LETTER FI
-2CAD          ; Lowercase # L&       COPTIC SMALL LETTER KHI
-2CAF          ; Lowercase # L&       COPTIC SMALL LETTER PSI
-2CB1          ; Lowercase # L&       COPTIC SMALL LETTER OOU
-2CB3          ; Lowercase # L&       COPTIC SMALL LETTER DIALECT-P ALEF
-2CB5          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC AIN
-2CB7          ; Lowercase # L&       COPTIC SMALL LETTER CRYPTOGRAMMIC EIE
-2CB9          ; Lowercase # L&       COPTIC SMALL LETTER DIALECT-P KAPA
-2CBB          ; Lowercase # L&       COPTIC SMALL LETTER DIALECT-P NI
-2CBD          ; Lowercase # L&       COPTIC SMALL LETTER CRYPTOGRAMMIC NI
-2CBF          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC OOU
-2CC1          ; Lowercase # L&       COPTIC SMALL LETTER SAMPI
-2CC3          ; Lowercase # L&       COPTIC SMALL LETTER CROSSED SHEI
-2CC5          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC SHEI
-2CC7          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC ESH
-2CC9          ; Lowercase # L&       COPTIC SMALL LETTER AKHMIMIC KHEI
-2CCB          ; Lowercase # L&       COPTIC SMALL LETTER DIALECT-P HORI
-2CCD          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC HORI
-2CCF          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC HA
-2CD1          ; Lowercase # L&       COPTIC SMALL LETTER L-SHAPED HA
-2CD3          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC HEI
-2CD5          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC HAT
-2CD7          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC GANGIA
-2CD9          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC DJA
-2CDB          ; Lowercase # L&       COPTIC SMALL LETTER OLD COPTIC SHIMA
-2CDD          ; Lowercase # L&       COPTIC SMALL LETTER OLD NUBIAN SHIMA
-2CDF          ; Lowercase # L&       COPTIC SMALL LETTER OLD NUBIAN NGI
-2CE1          ; Lowercase # L&       COPTIC SMALL LETTER OLD NUBIAN NYI
-2CE3..2CE4    ; Lowercase # L&   [2] COPTIC SMALL LETTER OLD NUBIAN WAU..COPTIC SYMBOL KAI
-2CEC          ; Lowercase # L&       COPTIC SMALL LETTER CRYPTOGRAMMIC SHEI
-2CEE          ; Lowercase # L&       COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA
-2D00..2D25    ; Lowercase # L&  [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE
-A641          ; Lowercase # L&       CYRILLIC SMALL LETTER ZEMLYA
-A643          ; Lowercase # L&       CYRILLIC SMALL LETTER DZELO
-A645          ; Lowercase # L&       CYRILLIC SMALL LETTER REVERSED DZE
-A647          ; Lowercase # L&       CYRILLIC SMALL LETTER IOTA
-A649          ; Lowercase # L&       CYRILLIC SMALL LETTER DJERV
-A64B          ; Lowercase # L&       CYRILLIC SMALL LETTER MONOGRAPH UK
-A64D          ; Lowercase # L&       CYRILLIC SMALL LETTER BROAD OMEGA
-A64F          ; Lowercase # L&       CYRILLIC SMALL LETTER NEUTRAL YER
-A651          ; Lowercase # L&       CYRILLIC SMALL LETTER YERU WITH BACK YER
-A653          ; Lowercase # L&       CYRILLIC SMALL LETTER IOTIFIED YAT
-A655          ; Lowercase # L&       CYRILLIC SMALL LETTER REVERSED YU
-A657          ; Lowercase # L&       CYRILLIC SMALL LETTER IOTIFIED A
-A659          ; Lowercase # L&       CYRILLIC SMALL LETTER CLOSED LITTLE YUS
-A65B          ; Lowercase # L&       CYRILLIC SMALL LETTER BLENDED YUS
-A65D          ; Lowercase # L&       CYRILLIC SMALL LETTER IOTIFIED CLOSED LITTLE YUS
-A65F          ; Lowercase # L&       CYRILLIC SMALL LETTER YN
-A663          ; Lowercase # L&       CYRILLIC SMALL LETTER SOFT DE
-A665          ; Lowercase # L&       CYRILLIC SMALL LETTER SOFT EL
-A667          ; Lowercase # L&       CYRILLIC SMALL LETTER SOFT EM
-A669          ; Lowercase # L&       CYRILLIC SMALL LETTER MONOCULAR O
-A66B          ; Lowercase # L&       CYRILLIC SMALL LETTER BINOCULAR O
-A66D          ; Lowercase # L&       CYRILLIC SMALL LETTER DOUBLE MONOCULAR O
-A681          ; Lowercase # L&       CYRILLIC SMALL LETTER DWE
-A683          ; Lowercase # L&       CYRILLIC SMALL LETTER DZWE
-A685          ; Lowercase # L&       CYRILLIC SMALL LETTER ZHWE
-A687          ; Lowercase # L&       CYRILLIC SMALL LETTER CCHE
-A689          ; Lowercase # L&       CYRILLIC SMALL LETTER DZZE
-A68B          ; Lowercase # L&       CYRILLIC SMALL LETTER TE WITH MIDDLE HOOK
-A68D          ; Lowercase # L&       CYRILLIC SMALL LETTER TWE
-A68F          ; Lowercase # L&       CYRILLIC SMALL LETTER TSWE
-A691          ; Lowercase # L&       CYRILLIC SMALL LETTER TSSE
-A693          ; Lowercase # L&       CYRILLIC SMALL LETTER TCHE
-A695          ; Lowercase # L&       CYRILLIC SMALL LETTER HWE
-A697          ; Lowercase # L&       CYRILLIC SMALL LETTER SHWE
-A723          ; Lowercase # L&       LATIN SMALL LETTER EGYPTOLOGICAL ALEF
-A725          ; Lowercase # L&       LATIN SMALL LETTER EGYPTOLOGICAL AIN
-A727          ; Lowercase # L&       LATIN SMALL LETTER HENG
-A729          ; Lowercase # L&       LATIN SMALL LETTER TZ
-A72B          ; Lowercase # L&       LATIN SMALL LETTER TRESILLO
-A72D          ; Lowercase # L&       LATIN SMALL LETTER CUATRILLO
-A72F..A731    ; Lowercase # L&   [3] LATIN SMALL LETTER CUATRILLO WITH COMMA..LATIN LETTER SMALL CAPITAL S
-A733          ; Lowercase # L&       LATIN SMALL LETTER AA
-A735          ; Lowercase # L&       LATIN SMALL LETTER AO
-A737          ; Lowercase # L&       LATIN SMALL LETTER AU
-A739          ; Lowercase # L&       LATIN SMALL LETTER AV
-A73B          ; Lowercase # L&       LATIN SMALL LETTER AV WITH HORIZONTAL BAR
-A73D          ; Lowercase # L&       LATIN SMALL LETTER AY
-A73F          ; Lowercase # L&       LATIN SMALL LETTER REVERSED C WITH DOT
-A741          ; Lowercase # L&       LATIN SMALL LETTER K WITH STROKE
-A743          ; Lowercase # L&       LATIN SMALL LETTER K WITH DIAGONAL STROKE
-A745          ; Lowercase # L&       LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE
-A747          ; Lowercase # L&       LATIN SMALL LETTER BROKEN L
-A749          ; Lowercase # L&       LATIN SMALL LETTER L WITH HIGH STROKE
-A74B          ; Lowercase # L&       LATIN SMALL LETTER O WITH LONG STROKE OVERLAY
-A74D          ; Lowercase # L&       LATIN SMALL LETTER O WITH LOOP
-A74F          ; Lowercase # L&       LATIN SMALL LETTER OO
-A751          ; Lowercase # L&       LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER
-A753          ; Lowercase # L&       LATIN SMALL LETTER P WITH FLOURISH
-A755          ; Lowercase # L&       LATIN SMALL LETTER P WITH SQUIRREL TAIL
-A757          ; Lowercase # L&       LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER
-A759          ; Lowercase # L&       LATIN SMALL LETTER Q WITH DIAGONAL STROKE
-A75B          ; Lowercase # L&       LATIN SMALL LETTER R ROTUNDA
-A75D          ; Lowercase # L&       LATIN SMALL LETTER RUM ROTUNDA
-A75F          ; Lowercase # L&       LATIN SMALL LETTER V WITH DIAGONAL STROKE
-A761          ; Lowercase # L&       LATIN SMALL LETTER VY
-A763          ; Lowercase # L&       LATIN SMALL LETTER VISIGOTHIC Z
-A765          ; Lowercase # L&       LATIN SMALL LETTER THORN WITH STROKE
-A767          ; Lowercase # L&       LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER
-A769          ; Lowercase # L&       LATIN SMALL LETTER VEND
-A76B          ; Lowercase # L&       LATIN SMALL LETTER ET
-A76D          ; Lowercase # L&       LATIN SMALL LETTER IS
-A76F          ; Lowercase # L&       LATIN SMALL LETTER CON
-A770          ; Lowercase # Lm       MODIFIER LETTER US
-A771..A778    ; Lowercase # L&   [8] LATIN SMALL LETTER DUM..LATIN SMALL LETTER UM
-A77A          ; Lowercase # L&       LATIN SMALL LETTER INSULAR D
-A77C          ; Lowercase # L&       LATIN SMALL LETTER INSULAR F
-A77F          ; Lowercase # L&       LATIN SMALL LETTER TURNED INSULAR G
-A781          ; Lowercase # L&       LATIN SMALL LETTER TURNED L
-A783          ; Lowercase # L&       LATIN SMALL LETTER INSULAR R
-A785          ; Lowercase # L&       LATIN SMALL LETTER INSULAR S
-A787          ; Lowercase # L&       LATIN SMALL LETTER INSULAR T
-A78C          ; Lowercase # L&       LATIN SMALL LETTER SALTILLO
-FB00..FB06    ; Lowercase # L&   [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST
-FB13..FB17    ; Lowercase # L&   [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH
-FF41..FF5A    ; Lowercase # L&  [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z
-10428..1044F  ; Lowercase # L&  [40] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER EW
-1D41A..1D433  ; Lowercase # L&  [26] MATHEMATICAL BOLD SMALL A..MATHEMATICAL BOLD SMALL Z
-1D44E..1D454  ; Lowercase # L&   [7] MATHEMATICAL ITALIC SMALL A..MATHEMATICAL ITALIC SMALL G
-1D456..1D467  ; Lowercase # L&  [18] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL ITALIC SMALL Z
-1D482..1D49B  ; Lowercase # L&  [26] MATHEMATICAL BOLD ITALIC SMALL A..MATHEMATICAL BOLD ITALIC SMALL Z
-1D4B6..1D4B9  ; Lowercase # L&   [4] MATHEMATICAL SCRIPT SMALL A..MATHEMATICAL SCRIPT SMALL D
-1D4BB         ; Lowercase # L&       MATHEMATICAL SCRIPT SMALL F
-1D4BD..1D4C3  ; Lowercase # L&   [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N
-1D4C5..1D4CF  ; Lowercase # L&  [11] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL SCRIPT SMALL Z
-1D4EA..1D503  ; Lowercase # L&  [26] MATHEMATICAL BOLD SCRIPT SMALL A..MATHEMATICAL BOLD SCRIPT SMALL Z
-1D51E..1D537  ; Lowercase # L&  [26] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL FRAKTUR SMALL Z
-1D552..1D56B  ; Lowercase # L&  [26] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL DOUBLE-STRUCK SMALL Z
-1D586..1D59F  ; Lowercase # L&  [26] MATHEMATICAL BOLD FRAKTUR SMALL A..MATHEMATICAL BOLD FRAKTUR SMALL Z
-1D5BA..1D5D3  ; Lowercase # L&  [26] MATHEMATICAL SANS-SERIF SMALL A..MATHEMATICAL SANS-SERIF SMALL Z
-1D5EE..1D607  ; Lowercase # L&  [26] MATHEMATICAL SANS-SERIF BOLD SMALL A..MATHEMATICAL SANS-SERIF BOLD SMALL Z
-1D622..1D63B  ; Lowercase # L&  [26] MATHEMATICAL SANS-SERIF ITALIC SMALL A..MATHEMATICAL SANS-SERIF ITALIC SMALL Z
-1D656..1D66F  ; Lowercase # L&  [26] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL A..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Z
-1D68A..1D6A5  ; Lowercase # L&  [28] MATHEMATICAL MONOSPACE SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J
-1D6C2..1D6DA  ; Lowercase # L&  [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA
-1D6DC..1D6E1  ; Lowercase # L&   [6] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL BOLD PI SYMBOL
-1D6FC..1D714  ; Lowercase # L&  [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA
-1D716..1D71B  ; Lowercase # L&   [6] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL ITALIC PI SYMBOL
-1D736..1D74E  ; Lowercase # L&  [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA
-1D750..1D755  ; Lowercase # L&   [6] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC PI SYMBOL
-1D770..1D788  ; Lowercase # L&  [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA
-1D78A..1D78F  ; Lowercase # L&   [6] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD PI SYMBOL
-1D7AA..1D7C2  ; Lowercase # L&  [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA
-1D7C4..1D7C9  ; Lowercase # L&   [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL
-1D7CB         ; Lowercase # L&       MATHEMATICAL BOLD SMALL DIGAMMA
-
-# Total code points: 1908
-
-# ================================================
-
-# Derived Property: Uppercase
-#  Generated from: Lu + Other_Uppercase
-
-0041..005A    ; Uppercase # L&  [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z
-00C0..00D6    ; Uppercase # L&  [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS
-00D8..00DE    ; Uppercase # L&   [7] LATIN CAPITAL LETTER O WITH STROKE..LATIN CAPITAL LETTER THORN
-0100          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH MACRON
-0102          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH BREVE
-0104          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH OGONEK
-0106          ; Uppercase # L&       LATIN CAPITAL LETTER C WITH ACUTE
-0108          ; Uppercase # L&       LATIN CAPITAL LETTER C WITH CIRCUMFLEX
-010A          ; Uppercase # L&       LATIN CAPITAL LETTER C WITH DOT ABOVE
-010C          ; Uppercase # L&       LATIN CAPITAL LETTER C WITH CARON
-010E          ; Uppercase # L&       LATIN CAPITAL LETTER D WITH CARON
-0110          ; Uppercase # L&       LATIN CAPITAL LETTER D WITH STROKE
-0112          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH MACRON
-0114          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH BREVE
-0116          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH DOT ABOVE
-0118          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH OGONEK
-011A          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH CARON
-011C          ; Uppercase # L&       LATIN CAPITAL LETTER G WITH CIRCUMFLEX
-011E          ; Uppercase # L&       LATIN CAPITAL LETTER G WITH BREVE
-0120          ; Uppercase # L&       LATIN CAPITAL LETTER G WITH DOT ABOVE
-0122          ; Uppercase # L&       LATIN CAPITAL LETTER G WITH CEDILLA
-0124          ; Uppercase # L&       LATIN CAPITAL LETTER H WITH CIRCUMFLEX
-0126          ; Uppercase # L&       LATIN CAPITAL LETTER H WITH STROKE
-0128          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH TILDE
-012A          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH MACRON
-012C          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH BREVE
-012E          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH OGONEK
-0130          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH DOT ABOVE
-0132          ; Uppercase # L&       LATIN CAPITAL LIGATURE IJ
-0134          ; Uppercase # L&       LATIN CAPITAL LETTER J WITH CIRCUMFLEX
-0136          ; Uppercase # L&       LATIN CAPITAL LETTER K WITH CEDILLA
-0139          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH ACUTE
-013B          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH CEDILLA
-013D          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH CARON
-013F          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH MIDDLE DOT
-0141          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH STROKE
-0143          ; Uppercase # L&       LATIN CAPITAL LETTER N WITH ACUTE
-0145          ; Uppercase # L&       LATIN CAPITAL LETTER N WITH CEDILLA
-0147          ; Uppercase # L&       LATIN CAPITAL LETTER N WITH CARON
-014A          ; Uppercase # L&       LATIN CAPITAL LETTER ENG
-014C          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH MACRON
-014E          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH BREVE
-0150          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
-0152          ; Uppercase # L&       LATIN CAPITAL LIGATURE OE
-0154          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH ACUTE
-0156          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH CEDILLA
-0158          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH CARON
-015A          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH ACUTE
-015C          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH CIRCUMFLEX
-015E          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH CEDILLA
-0160          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH CARON
-0162          ; Uppercase # L&       LATIN CAPITAL LETTER T WITH CEDILLA
-0164          ; Uppercase # L&       LATIN CAPITAL LETTER T WITH CARON
-0166          ; Uppercase # L&       LATIN CAPITAL LETTER T WITH STROKE
-0168          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH TILDE
-016A          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH MACRON
-016C          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH BREVE
-016E          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH RING ABOVE
-0170          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
-0172          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH OGONEK
-0174          ; Uppercase # L&       LATIN CAPITAL LETTER W WITH CIRCUMFLEX
-0176          ; Uppercase # L&       LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
-0178..0179    ; Uppercase # L&   [2] LATIN CAPITAL LETTER Y WITH DIAERESIS..LATIN CAPITAL LETTER Z WITH ACUTE
-017B          ; Uppercase # L&       LATIN CAPITAL LETTER Z WITH DOT ABOVE
-017D          ; Uppercase # L&       LATIN CAPITAL LETTER Z WITH CARON
-0181..0182    ; Uppercase # L&   [2] LATIN CAPITAL LETTER B WITH HOOK..LATIN CAPITAL LETTER B WITH TOPBAR
-0184          ; Uppercase # L&       LATIN CAPITAL LETTER TONE SIX
-0186..0187    ; Uppercase # L&   [2] LATIN CAPITAL LETTER OPEN O..LATIN CAPITAL LETTER C WITH HOOK
-0189..018B    ; Uppercase # L&   [3] LATIN CAPITAL LETTER AFRICAN D..LATIN CAPITAL LETTER D WITH TOPBAR
-018E..0191    ; Uppercase # L&   [4] LATIN CAPITAL LETTER REVERSED E..LATIN CAPITAL LETTER F WITH HOOK
-0193..0194    ; Uppercase # L&   [2] LATIN CAPITAL LETTER G WITH HOOK..LATIN CAPITAL LETTER GAMMA
-0196..0198    ; Uppercase # L&   [3] LATIN CAPITAL LETTER IOTA..LATIN CAPITAL LETTER K WITH HOOK
-019C..019D    ; Uppercase # L&   [2] LATIN CAPITAL LETTER TURNED M..LATIN CAPITAL LETTER N WITH LEFT HOOK
-019F..01A0    ; Uppercase # L&   [2] LATIN CAPITAL LETTER O WITH MIDDLE TILDE..LATIN CAPITAL LETTER O WITH HORN
-01A2          ; Uppercase # L&       LATIN CAPITAL LETTER OI
-01A4          ; Uppercase # L&       LATIN CAPITAL LETTER P WITH HOOK
-01A6..01A7    ; Uppercase # L&   [2] LATIN LETTER YR..LATIN CAPITAL LETTER TONE TWO
-01A9          ; Uppercase # L&       LATIN CAPITAL LETTER ESH
-01AC          ; Uppercase # L&       LATIN CAPITAL LETTER T WITH HOOK
-01AE..01AF    ; Uppercase # L&   [2] LATIN CAPITAL LETTER T WITH RETROFLEX HOOK..LATIN CAPITAL LETTER U WITH HORN
-01B1..01B3    ; Uppercase # L&   [3] LATIN CAPITAL LETTER UPSILON..LATIN CAPITAL LETTER Y WITH HOOK
-01B5          ; Uppercase # L&       LATIN CAPITAL LETTER Z WITH STROKE
-01B7..01B8    ; Uppercase # L&   [2] LATIN CAPITAL LETTER EZH..LATIN CAPITAL LETTER EZH REVERSED
-01BC          ; Uppercase # L&       LATIN CAPITAL LETTER TONE FIVE
-01C4          ; Uppercase # L&       LATIN CAPITAL LETTER DZ WITH CARON
-01C7          ; Uppercase # L&       LATIN CAPITAL LETTER LJ
-01CA          ; Uppercase # L&       LATIN CAPITAL LETTER NJ
-01CD          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH CARON
-01CF          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH CARON
-01D1          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH CARON
-01D3          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH CARON
-01D5          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
-01D7          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
-01D9          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
-01DB          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
-01DE          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
-01E0          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
-01E2          ; Uppercase # L&       LATIN CAPITAL LETTER AE WITH MACRON
-01E4          ; Uppercase # L&       LATIN CAPITAL LETTER G WITH STROKE
-01E6          ; Uppercase # L&       LATIN CAPITAL LETTER G WITH CARON
-01E8          ; Uppercase # L&       LATIN CAPITAL LETTER K WITH CARON
-01EA          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH OGONEK
-01EC          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
-01EE          ; Uppercase # L&       LATIN CAPITAL LETTER EZH WITH CARON
-01F1          ; Uppercase # L&       LATIN CAPITAL LETTER DZ
-01F4          ; Uppercase # L&       LATIN CAPITAL LETTER G WITH ACUTE
-01F6..01F8    ; Uppercase # L&   [3] LATIN CAPITAL LETTER HWAIR..LATIN CAPITAL LETTER N WITH GRAVE
-01FA          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
-01FC          ; Uppercase # L&       LATIN CAPITAL LETTER AE WITH ACUTE
-01FE          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
-0200          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
-0202          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH INVERTED BREVE
-0204          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
-0206          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH INVERTED BREVE
-0208          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
-020A          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH INVERTED BREVE
-020C          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
-020E          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH INVERTED BREVE
-0210          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
-0212          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH INVERTED BREVE
-0214          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
-0216          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH INVERTED BREVE
-0218          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH COMMA BELOW
-021A          ; Uppercase # L&       LATIN CAPITAL LETTER T WITH COMMA BELOW
-021C          ; Uppercase # L&       LATIN CAPITAL LETTER YOGH
-021E          ; Uppercase # L&       LATIN CAPITAL LETTER H WITH CARON
-0220          ; Uppercase # L&       LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
-0222          ; Uppercase # L&       LATIN CAPITAL LETTER OU
-0224          ; Uppercase # L&       LATIN CAPITAL LETTER Z WITH HOOK
-0226          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH DOT ABOVE
-0228          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH CEDILLA
-022A          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
-022C          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH TILDE AND MACRON
-022E          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH DOT ABOVE
-0230          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
-0232          ; Uppercase # L&       LATIN CAPITAL LETTER Y WITH MACRON
-023A..023B    ; Uppercase # L&   [2] LATIN CAPITAL LETTER A WITH STROKE..LATIN CAPITAL LETTER C WITH STROKE
-023D..023E    ; Uppercase # L&   [2] LATIN CAPITAL LETTER L WITH BAR..LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
-0241          ; Uppercase # L&       LATIN CAPITAL LETTER GLOTTAL STOP
-0243..0246    ; Uppercase # L&   [4] LATIN CAPITAL LETTER B WITH STROKE..LATIN CAPITAL LETTER E WITH STROKE
-0248          ; Uppercase # L&       LATIN CAPITAL LETTER J WITH STROKE
-024A          ; Uppercase # L&       LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
-024C          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH STROKE
-024E          ; Uppercase # L&       LATIN CAPITAL LETTER Y WITH STROKE
-0370          ; Uppercase # L&       GREEK CAPITAL LETTER HETA
-0372          ; Uppercase # L&       GREEK CAPITAL LETTER ARCHAIC SAMPI
-0376          ; Uppercase # L&       GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
-0386          ; Uppercase # L&       GREEK CAPITAL LETTER ALPHA WITH TONOS
-0388..038A    ; Uppercase # L&   [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS
-038C          ; Uppercase # L&       GREEK CAPITAL LETTER OMICRON WITH TONOS
-038E..038F    ; Uppercase # L&   [2] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER OMEGA WITH TONOS
-0391..03A1    ; Uppercase # L&  [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO
-03A3..03AB    ; Uppercase # L&   [9] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
-03CF          ; Uppercase # L&       GREEK CAPITAL KAI SYMBOL
-03D2..03D4    ; Uppercase # L&   [3] GREEK UPSILON WITH HOOK SYMBOL..GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL
-03D8          ; Uppercase # L&       GREEK LETTER ARCHAIC KOPPA
-03DA          ; Uppercase # L&       GREEK LETTER STIGMA
-03DC          ; Uppercase # L&       GREEK LETTER DIGAMMA
-03DE          ; Uppercase # L&       GREEK LETTER KOPPA
-03E0          ; Uppercase # L&       GREEK LETTER SAMPI
-03E2          ; Uppercase # L&       COPTIC CAPITAL LETTER SHEI
-03E4          ; Uppercase # L&       COPTIC CAPITAL LETTER FEI
-03E6          ; Uppercase # L&       COPTIC CAPITAL LETTER KHEI
-03E8          ; Uppercase # L&       COPTIC CAPITAL LETTER HORI
-03EA          ; Uppercase # L&       COPTIC CAPITAL LETTER GANGIA
-03EC          ; Uppercase # L&       COPTIC CAPITAL LETTER SHIMA
-03EE          ; Uppercase # L&       COPTIC CAPITAL LETTER DEI
-03F4          ; Uppercase # L&       GREEK CAPITAL THETA SYMBOL
-03F7          ; Uppercase # L&       GREEK CAPITAL LETTER SHO
-03F9..03FA    ; Uppercase # L&   [2] GREEK CAPITAL LUNATE SIGMA SYMBOL..GREEK CAPITAL LETTER SAN
-03FD..042F    ; Uppercase # L&  [51] GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL..CYRILLIC CAPITAL LETTER YA
-0460          ; Uppercase # L&       CYRILLIC CAPITAL LETTER OMEGA
-0462          ; Uppercase # L&       CYRILLIC CAPITAL LETTER YAT
-0464          ; Uppercase # L&       CYRILLIC CAPITAL LETTER IOTIFIED E
-0466          ; Uppercase # L&       CYRILLIC CAPITAL LETTER LITTLE YUS
-0468          ; Uppercase # L&       CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
-046A          ; Uppercase # L&       CYRILLIC CAPITAL LETTER BIG YUS
-046C          ; Uppercase # L&       CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
-046E          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KSI
-0470          ; Uppercase # L&       CYRILLIC CAPITAL LETTER PSI
-0472          ; Uppercase # L&       CYRILLIC CAPITAL LETTER FITA
-0474          ; Uppercase # L&       CYRILLIC CAPITAL LETTER IZHITSA
-0476          ; Uppercase # L&       CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
-0478          ; Uppercase # L&       CYRILLIC CAPITAL LETTER UK
-047A          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ROUND OMEGA
-047C          ; Uppercase # L&       CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
-047E          ; Uppercase # L&       CYRILLIC CAPITAL LETTER OT
-0480          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KOPPA
-048A          ; Uppercase # L&       CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
-048C          ; Uppercase # L&       CYRILLIC CAPITAL LETTER SEMISOFT SIGN
-048E          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ER WITH TICK
-0490          ; Uppercase # L&       CYRILLIC CAPITAL LETTER GHE WITH UPTURN
-0492          ; Uppercase # L&       CYRILLIC CAPITAL LETTER GHE WITH STROKE
-0494          ; Uppercase # L&       CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
-0496          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
-0498          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
-049A          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KA WITH DESCENDER
-049C          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
-049E          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KA WITH STROKE
-04A0          ; Uppercase # L&       CYRILLIC CAPITAL LETTER BASHKIR KA
-04A2          ; Uppercase # L&       CYRILLIC CAPITAL LETTER EN WITH DESCENDER
-04A4          ; Uppercase # L&       CYRILLIC CAPITAL LIGATURE EN GHE
-04A6          ; Uppercase # L&       CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
-04A8          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ABKHASIAN HA
-04AA          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ES WITH DESCENDER
-04AC          ; Uppercase # L&       CYRILLIC CAPITAL LETTER TE WITH DESCENDER
-04AE          ; Uppercase # L&       CYRILLIC CAPITAL LETTER STRAIGHT U
-04B0          ; Uppercase # L&       CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
-04B2          ; Uppercase # L&       CYRILLIC CAPITAL LETTER HA WITH DESCENDER
-04B4          ; Uppercase # L&       CYRILLIC CAPITAL LIGATURE TE TSE
-04B6          ; Uppercase # L&       CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
-04B8          ; Uppercase # L&       CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
-04BA          ; Uppercase # L&       CYRILLIC CAPITAL LETTER SHHA
-04BC          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ABKHASIAN CHE
-04BE          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
-04C0..04C1    ; Uppercase # L&   [2] CYRILLIC LETTER PALOCHKA..CYRILLIC CAPITAL LETTER ZHE WITH BREVE
-04C3          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KA WITH HOOK
-04C5          ; Uppercase # L&       CYRILLIC CAPITAL LETTER EL WITH TAIL
-04C7          ; Uppercase # L&       CYRILLIC CAPITAL LETTER EN WITH HOOK
-04C9          ; Uppercase # L&       CYRILLIC CAPITAL LETTER EN WITH TAIL
-04CB          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
-04CD          ; Uppercase # L&       CYRILLIC CAPITAL LETTER EM WITH TAIL
-04D0          ; Uppercase # L&       CYRILLIC CAPITAL LETTER A WITH BREVE
-04D2          ; Uppercase # L&       CYRILLIC CAPITAL LETTER A WITH DIAERESIS
-04D4          ; Uppercase # L&       CYRILLIC CAPITAL LIGATURE A IE
-04D6          ; Uppercase # L&       CYRILLIC CAPITAL LETTER IE WITH BREVE
-04D8          ; Uppercase # L&       CYRILLIC CAPITAL LETTER SCHWA
-04DA          ; Uppercase # L&       CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
-04DC          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
-04DE          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
-04E0          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ABKHASIAN DZE
-04E2          ; Uppercase # L&       CYRILLIC CAPITAL LETTER I WITH MACRON
-04E4          ; Uppercase # L&       CYRILLIC CAPITAL LETTER I WITH DIAERESIS
-04E6          ; Uppercase # L&       CYRILLIC CAPITAL LETTER O WITH DIAERESIS
-04E8          ; Uppercase # L&       CYRILLIC CAPITAL LETTER BARRED O
-04EA          ; Uppercase # L&       CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
-04EC          ; Uppercase # L&       CYRILLIC CAPITAL LETTER E WITH DIAERESIS
-04EE          ; Uppercase # L&       CYRILLIC CAPITAL LETTER U WITH MACRON
-04F0          ; Uppercase # L&       CYRILLIC CAPITAL LETTER U WITH DIAERESIS
-04F2          ; Uppercase # L&       CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
-04F4          ; Uppercase # L&       CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
-04F6          ; Uppercase # L&       CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
-04F8          ; Uppercase # L&       CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
-04FA          ; Uppercase # L&       CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
-04FC          ; Uppercase # L&       CYRILLIC CAPITAL LETTER HA WITH HOOK
-04FE          ; Uppercase # L&       CYRILLIC CAPITAL LETTER HA WITH STROKE
-0500          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KOMI DE
-0502          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KOMI DJE
-0504          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KOMI ZJE
-0506          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KOMI DZJE
-0508          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KOMI LJE
-050A          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KOMI NJE
-050C          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KOMI SJE
-050E          ; Uppercase # L&       CYRILLIC CAPITAL LETTER KOMI TJE
-0510          ; Uppercase # L&       CYRILLIC CAPITAL LETTER REVERSED ZE
-0512          ; Uppercase # L&       CYRILLIC CAPITAL LETTER EL WITH HOOK
-0514          ; Uppercase # L&       CYRILLIC CAPITAL LETTER LHA
-0516          ; Uppercase # L&       CYRILLIC CAPITAL LETTER RHA
-0518          ; Uppercase # L&       CYRILLIC CAPITAL LETTER YAE
-051A          ; Uppercase # L&       CYRILLIC CAPITAL LETTER QA
-051C          ; Uppercase # L&       CYRILLIC CAPITAL LETTER WE
-051E          ; Uppercase # L&       CYRILLIC CAPITAL LETTER ALEUT KA
-0520          ; Uppercase # L&       CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
-0522          ; Uppercase # L&       CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
-0524          ; Uppercase # L&       CYRILLIC CAPITAL LETTER PE WITH DESCENDER
-0531..0556    ; Uppercase # L&  [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH
-10A0..10C5    ; Uppercase # L&  [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE
-1E00          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH RING BELOW
-1E02          ; Uppercase # L&       LATIN CAPITAL LETTER B WITH DOT ABOVE
-1E04          ; Uppercase # L&       LATIN CAPITAL LETTER B WITH DOT BELOW
-1E06          ; Uppercase # L&       LATIN CAPITAL LETTER B WITH LINE BELOW
-1E08          ; Uppercase # L&       LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
-1E0A          ; Uppercase # L&       LATIN CAPITAL LETTER D WITH DOT ABOVE
-1E0C          ; Uppercase # L&       LATIN CAPITAL LETTER D WITH DOT BELOW
-1E0E          ; Uppercase # L&       LATIN CAPITAL LETTER D WITH LINE BELOW
-1E10          ; Uppercase # L&       LATIN CAPITAL LETTER D WITH CEDILLA
-1E12          ; Uppercase # L&       LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
-1E14          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
-1E16          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
-1E18          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
-1E1A          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH TILDE BELOW
-1E1C          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
-1E1E          ; Uppercase # L&       LATIN CAPITAL LETTER F WITH DOT ABOVE
-1E20          ; Uppercase # L&       LATIN CAPITAL LETTER G WITH MACRON
-1E22          ; Uppercase # L&       LATIN CAPITAL LETTER H WITH DOT ABOVE
-1E24          ; Uppercase # L&       LATIN CAPITAL LETTER H WITH DOT BELOW
-1E26          ; Uppercase # L&       LATIN CAPITAL LETTER H WITH DIAERESIS
-1E28          ; Uppercase # L&       LATIN CAPITAL LETTER H WITH CEDILLA
-1E2A          ; Uppercase # L&       LATIN CAPITAL LETTER H WITH BREVE BELOW
-1E2C          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH TILDE BELOW
-1E2E          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
-1E30          ; Uppercase # L&       LATIN CAPITAL LETTER K WITH ACUTE
-1E32          ; Uppercase # L&       LATIN CAPITAL LETTER K WITH DOT BELOW
-1E34          ; Uppercase # L&       LATIN CAPITAL LETTER K WITH LINE BELOW
-1E36          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH DOT BELOW
-1E38          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
-1E3A          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH LINE BELOW
-1E3C          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
-1E3E          ; Uppercase # L&       LATIN CAPITAL LETTER M WITH ACUTE
-1E40          ; Uppercase # L&       LATIN CAPITAL LETTER M WITH DOT ABOVE
-1E42          ; Uppercase # L&       LATIN CAPITAL LETTER M WITH DOT BELOW
-1E44          ; Uppercase # L&       LATIN CAPITAL LETTER N WITH DOT ABOVE
-1E46          ; Uppercase # L&       LATIN CAPITAL LETTER N WITH DOT BELOW
-1E48          ; Uppercase # L&       LATIN CAPITAL LETTER N WITH LINE BELOW
-1E4A          ; Uppercase # L&       LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
-1E4C          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
-1E4E          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
-1E50          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
-1E52          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
-1E54          ; Uppercase # L&       LATIN CAPITAL LETTER P WITH ACUTE
-1E56          ; Uppercase # L&       LATIN CAPITAL LETTER P WITH DOT ABOVE
-1E58          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH DOT ABOVE
-1E5A          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH DOT BELOW
-1E5C          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
-1E5E          ; Uppercase # L&       LATIN CAPITAL LETTER R WITH LINE BELOW
-1E60          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH DOT ABOVE
-1E62          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH DOT BELOW
-1E64          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
-1E66          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
-1E68          ; Uppercase # L&       LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
-1E6A          ; Uppercase # L&       LATIN CAPITAL LETTER T WITH DOT ABOVE
-1E6C          ; Uppercase # L&       LATIN CAPITAL LETTER T WITH DOT BELOW
-1E6E          ; Uppercase # L&       LATIN CAPITAL LETTER T WITH LINE BELOW
-1E70          ; Uppercase # L&       LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
-1E72          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
-1E74          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH TILDE BELOW
-1E76          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
-1E78          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
-1E7A          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
-1E7C          ; Uppercase # L&       LATIN CAPITAL LETTER V WITH TILDE
-1E7E          ; Uppercase # L&       LATIN CAPITAL LETTER V WITH DOT BELOW
-1E80          ; Uppercase # L&       LATIN CAPITAL LETTER W WITH GRAVE
-1E82          ; Uppercase # L&       LATIN CAPITAL LETTER W WITH ACUTE
-1E84          ; Uppercase # L&       LATIN CAPITAL LETTER W WITH DIAERESIS
-1E86          ; Uppercase # L&       LATIN CAPITAL LETTER W WITH DOT ABOVE
-1E88          ; Uppercase # L&       LATIN CAPITAL LETTER W WITH DOT BELOW
-1E8A          ; Uppercase # L&       LATIN CAPITAL LETTER X WITH DOT ABOVE
-1E8C          ; Uppercase # L&       LATIN CAPITAL LETTER X WITH DIAERESIS
-1E8E          ; Uppercase # L&       LATIN CAPITAL LETTER Y WITH DOT ABOVE
-1E90          ; Uppercase # L&       LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
-1E92          ; Uppercase # L&       LATIN CAPITAL LETTER Z WITH DOT BELOW
-1E94          ; Uppercase # L&       LATIN CAPITAL LETTER Z WITH LINE BELOW
-1E9E          ; Uppercase # L&       LATIN CAPITAL LETTER SHARP S
-1EA0          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH DOT BELOW
-1EA2          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH HOOK ABOVE
-1EA4          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
-1EA6          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
-1EA8          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
-1EAA          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
-1EAC          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
-1EAE          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
-1EB0          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
-1EB2          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
-1EB4          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH BREVE AND TILDE
-1EB6          ; Uppercase # L&       LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
-1EB8          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH DOT BELOW
-1EBA          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH HOOK ABOVE
-1EBC          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH TILDE
-1EBE          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
-1EC0          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
-1EC2          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
-1EC4          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
-1EC6          ; Uppercase # L&       LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
-1EC8          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH HOOK ABOVE
-1ECA          ; Uppercase # L&       LATIN CAPITAL LETTER I WITH DOT BELOW
-1ECC          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH DOT BELOW
-1ECE          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH HOOK ABOVE
-1ED0          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
-1ED2          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
-1ED4          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
-1ED6          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
-1ED8          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
-1EDA          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH HORN AND ACUTE
-1EDC          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH HORN AND GRAVE
-1EDE          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
-1EE0          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH HORN AND TILDE
-1EE2          ; Uppercase # L&       LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
-1EE4          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH DOT BELOW
-1EE6          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH HOOK ABOVE
-1EE8          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH HORN AND ACUTE
-1EEA          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH HORN AND GRAVE
-1EEC          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
-1EEE          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH HORN AND TILDE
-1EF0          ; Uppercase # L&       LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
-1EF2          ; Uppercase # L&       LATIN CAPITAL LETTER Y WITH GRAVE
-1EF4          ; Uppercase # L&       LATIN CAPITAL LETTER Y WITH DOT BELOW
-1EF6          ; Uppercase # L&       LATIN CAPITAL LETTER Y WITH HOOK ABOVE
-1EF8          ; Uppercase # L&       LATIN CAPITAL LETTER Y WITH TILDE
-1EFA          ; Uppercase # L&       LATIN CAPITAL LETTER MIDDLE-WELSH LL
-1EFC          ; Uppercase # L&       LATIN CAPITAL LETTER MIDDLE-WELSH V
-1EFE          ; Uppercase # L&       LATIN CAPITAL LETTER Y WITH LOOP
-1F08..1F0F    ; Uppercase # L&   [8] GREEK CAPITAL LETTER ALPHA WITH PSILI..GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
-1F18..1F1D    ; Uppercase # L&   [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
-1F28..1F2F    ; Uppercase # L&   [8] GREEK CAPITAL LETTER ETA WITH PSILI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
-1F38..1F3F    ; Uppercase # L&   [8] GREEK CAPITAL LETTER IOTA WITH PSILI..GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
-1F48..1F4D    ; Uppercase # L&   [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
-1F59          ; Uppercase # L&       GREEK CAPITAL LETTER UPSILON WITH DASIA
-1F5B          ; Uppercase # L&       GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
-1F5D          ; Uppercase # L&       GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
-1F5F          ; Uppercase # L&       GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
-1F68..1F6F    ; Uppercase # L&   [8] GREEK CAPITAL LETTER OMEGA WITH PSILI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
-1FB8..1FBB    ; Uppercase # L&   [4] GREEK CAPITAL LETTER ALPHA WITH VRACHY..GREEK CAPITAL LETTER ALPHA WITH OXIA
-1FC8..1FCB    ; Uppercase # L&   [4] GREEK CAPITAL LETTER EPSILON WITH VARIA..GREEK CAPITAL LETTER ETA WITH OXIA
-1FD8..1FDB    ; Uppercase # L&   [4] GREEK CAPITAL LETTER IOTA WITH VRACHY..GREEK CAPITAL LETTER IOTA WITH OXIA
-1FE8..1FEC    ; Uppercase # L&   [5] GREEK CAPITAL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA
-1FF8..1FFB    ; Uppercase # L&   [4] GREEK CAPITAL LETTER OMICRON WITH VARIA..GREEK CAPITAL LETTER OMEGA WITH OXIA
-2102          ; Uppercase # L&       DOUBLE-STRUCK CAPITAL C
-2107          ; Uppercase # L&       EULER CONSTANT
-210B..210D    ; Uppercase # L&   [3] SCRIPT CAPITAL H..DOUBLE-STRUCK CAPITAL H
-2110..2112    ; Uppercase # L&   [3] SCRIPT CAPITAL I..SCRIPT CAPITAL L
-2115          ; Uppercase # L&       DOUBLE-STRUCK CAPITAL N
-2119..211D    ; Uppercase # L&   [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R
-2124          ; Uppercase # L&       DOUBLE-STRUCK CAPITAL Z
-2126          ; Uppercase # L&       OHM SIGN
-2128          ; Uppercase # L&       BLACK-LETTER CAPITAL Z
-212A..212D    ; Uppercase # L&   [4] KELVIN SIGN..BLACK-LETTER CAPITAL C
-2130..2133    ; Uppercase # L&   [4] SCRIPT CAPITAL E..SCRIPT CAPITAL M
-213E..213F    ; Uppercase # L&   [2] DOUBLE-STRUCK CAPITAL GAMMA..DOUBLE-STRUCK CAPITAL PI
-2145          ; Uppercase # L&       DOUBLE-STRUCK ITALIC CAPITAL D
-2160..216F    ; Uppercase # Nl  [16] ROMAN NUMERAL ONE..ROMAN NUMERAL ONE THOUSAND
-2183          ; Uppercase # L&       ROMAN NUMERAL REVERSED ONE HUNDRED
-24B6..24CF    ; Uppercase # So  [26] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN CAPITAL LETTER Z
-2C00..2C2E    ; Uppercase # L&  [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE
-2C60          ; Uppercase # L&       LATIN CAPITAL LETTER L WITH DOUBLE BAR
-2C62..2C64    ; Uppercase # L&   [3] LATIN CAPITAL LETTER L WITH MIDDLE TILDE..LATIN CAPITAL LETTER R WITH TAIL
-2C67          ; Uppercase # L&       LATIN CAPITAL LETTER H WITH DESCENDER
-2C69          ; Uppercase # L&       LATIN CAPITAL LETTER K WITH DESCENDER
-2C6B          ; Uppercase # L&       LATIN CAPITAL LETTER Z WITH DESCENDER
-2C6D..2C70    ; Uppercase # L&   [4] LATIN CAPITAL LETTER ALPHA..LATIN CAPITAL LETTER TURNED ALPHA
-2C72          ; Uppercase # L&       LATIN CAPITAL LETTER W WITH HOOK
-2C75          ; Uppercase # L&       LATIN CAPITAL LETTER HALF H
-2C7E..2C80    ; Uppercase # L&   [3] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC CAPITAL LETTER ALFA
-2C82          ; Uppercase # L&       COPTIC CAPITAL LETTER VIDA
-2C84          ; Uppercase # L&       COPTIC CAPITAL LETTER GAMMA
-2C86          ; Uppercase # L&       COPTIC CAPITAL LETTER DALDA
-2C88          ; Uppercase # L&       COPTIC CAPITAL LETTER EIE
-2C8A          ; Uppercase # L&       COPTIC CAPITAL LETTER SOU

From pypy.commits at gmail.com  Mon Jan  9 13:01:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 09 Jan 2017 10:01:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5873d013.820bc30a.5970b.a261@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89451:3708a9cce7d6
Date: 2017-01-09 19:00 +0100
http://bitbucket.org/pypy/pypy/changeset/3708a9cce7d6/

Log:	hg merge default

diff too long, truncating to 2000 out of 70334 lines

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -14,7 +14,7 @@
         #assert api.PyObject_CheckBuffer(w_hello)
         w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
-        assert space.eq_w(w_char, space.wrap(ord('h')))
+        assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
diff --git a/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
@@ -0,0 +1,1414 @@
+# CaseFolding-8.0.0.txt
+# Date: 2015-01-13, 18:16:36 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#223;e" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding 
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+# Property: Case_Folding
+
+#  All code points not explicitly listed for Case_Folding
+#  have the value C for the status field, and the code point itself for the mapping field.
+
+# =================================================================
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE
+023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE
+023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR
+023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
+0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP
+0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE
+0244; C; 0289; # LATIN CAPITAL LETTER U BAR
+0245; C; 028C; # LATIN CAPITAL LETTER TURNED V
+0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE
+0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE
+024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
+024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE
+024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0370; C; 0371; # GREEK CAPITAL LETTER HETA
+0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI
+0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
+037F; C; 03F3; # GREEK CAPITAL LETTER YOT
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+03F7; C; 03F8; # GREEK CAPITAL LETTER SHO
+03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL
+03FA; C; 03FB; # GREEK CAPITAL LETTER SAN
+03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
+03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
+03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
+04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK
+04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE
+0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK
+0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA
+0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA
+0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE
+051A; C; 051B; # CYRILLIC CAPITAL LETTER QA
+051C; C; 051D; # CYRILLIC CAPITAL LETTER WE
+051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA
+0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
+0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
+0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER
+0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER
+0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK
+052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE
+052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE
+052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN
+10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN
+10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN
+10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON
+10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN
+10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN
+10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN
+10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN
+10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN
+10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN
+10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS
+10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN
+10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR
+10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON
+10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR
+10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR
+10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE
+10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN
+10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR
+10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN
+10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR
+10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR
+10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN
+10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR
+10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN
+10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN
+10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN
+10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL
+10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL
+10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR
+10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN
+10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN
+10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE
+10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE
+10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE
+10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE
+10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR
+10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE
+10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN
+10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN
+13F8; C; 13F0; # CHEROKEE SMALL LETTER YE
+13F9; C; 13F1; # CHEROKEE SMALL LETTER YI
+13FA; C; 13F2; # CHEROKEE SMALL LETTER YO
+13FB; C; 13F3; # CHEROKEE SMALL LETTER YU
+13FC; C; 13F4; # CHEROKEE SMALL LETTER YV
+13FD; C; 13F5; # CHEROKEE SMALL LETTER MV
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S
+1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL
+1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V
+1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2132; C; 214E; # TURNED CAPITAL F
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N
+24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O
+24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P
+24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q
+24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R
+24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S
+24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T
+24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U
+24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V
+24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W
+24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X
+24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y
+24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z
+2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU
+2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY
+2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE
+2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI
+2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO
+2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU
+2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE
+2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO
+2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA
+2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE
+2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE
+2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I
+2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI
+2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO
+2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE
+2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE
+2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI
+2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU
+2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI
+2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI
+2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO
+2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO
+2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU
+2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU
+2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU
+2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU
+2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE
+2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA
+2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI
+2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI
+2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA
+2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU
+2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI
+2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI
+2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA
+2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU
+2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS
+2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL
+2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO
+2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS
+2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS
+2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS
+2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA
+2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA
+2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC
+2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A
+2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE
+2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR
+2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE
+2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE
+2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL
+2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER
+2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER
+2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER
+2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA
+2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK
+2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A
+2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA
+2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK
+2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H
+2C7E; C; 023F; # LATIN CAPITAL LETTER S WITH SWASH TAIL
+2C7F; C; 0240; # LATIN CAPITAL LETTER Z WITH SWASH TAIL
+2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA
+2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA
+2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA
+2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA
+2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE
+2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU
+2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA
+2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE
+2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE
+2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA
+2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA
+2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA
+2C98; C; 2C99; # COPTIC CAPITAL LETTER MI
+2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI
+2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI
+2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O
+2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI
+2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO
+2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA
+2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU
+2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA
+2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI
+2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI
+2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI
+2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU
+2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF
+2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN
+2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE
+2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA
+2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI
+2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI
+2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU
+2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI
+2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI
+2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI
+2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH
+2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI
+2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI
+2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI
+2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA
+2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA
+2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI
+2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT
+2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA
+2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA
+2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA
+2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA
+2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI
+2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI
+2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU
+2CEB; C; 2CEC; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI
+2CED; C; 2CEE; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA
+2CF2; C; 2CF3; # COPTIC CAPITAL LETTER BOHAIRIC KHEI
+A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA
+A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO
+A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE
+A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA
+A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV
+A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK
+A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA
+A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER
+A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER
+A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT
+A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU
+A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A
+A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS
+A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS
+A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS
+A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN
+A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE
+A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE
+A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL
+A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM
+A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O
+A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O
+A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O
+A680; C; A681; # CYRILLIC CAPITAL LETTER DWE
+A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE
+A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE
+A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE
+A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE
+A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK
+A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE
+A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE
+A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE
+A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE
+A694; C; A695; # CYRILLIC CAPITAL LETTER HWE
+A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE
+A698; C; A699; # CYRILLIC CAPITAL LETTER DOUBLE O
+A69A; C; A69B; # CYRILLIC CAPITAL LETTER CROSSED O
+A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF
+A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN
+A726; C; A727; # LATIN CAPITAL LETTER HENG
+A728; C; A729; # LATIN CAPITAL LETTER TZ
+A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO
+A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO
+A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA
+A732; C; A733; # LATIN CAPITAL LETTER AA
+A734; C; A735; # LATIN CAPITAL LETTER AO
+A736; C; A737; # LATIN CAPITAL LETTER AU
+A738; C; A739; # LATIN CAPITAL LETTER AV
+A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR
+A73C; C; A73D; # LATIN CAPITAL LETTER AY
+A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT
+A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE
+A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE
+A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE
+A746; C; A747; # LATIN CAPITAL LETTER BROKEN L
+A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE
+A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY
+A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP
+A74E; C; A74F; # LATIN CAPITAL LETTER OO
+A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER
+A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH
+A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL
+A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER
+A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE
+A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA
+A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA
+A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE
+A760; C; A761; # LATIN CAPITAL LETTER VY
+A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z
+A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE
+A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER
+A768; C; A769; # LATIN CAPITAL LETTER VEND
+A76A; C; A76B; # LATIN CAPITAL LETTER ET
+A76C; C; A76D; # LATIN CAPITAL LETTER IS
+A76E; C; A76F; # LATIN CAPITAL LETTER CON
+A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D
+A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F
+A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G
+A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G
+A780; C; A781; # LATIN CAPITAL LETTER TURNED L
+A782; C; A783; # LATIN CAPITAL LETTER INSULAR R
+A784; C; A785; # LATIN CAPITAL LETTER INSULAR S
+A786; C; A787; # LATIN CAPITAL LETTER INSULAR T
+A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO
+A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H
+A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER
+A792; C; A793; # LATIN CAPITAL LETTER C WITH BAR
+A796; C; A797; # LATIN CAPITAL LETTER B WITH FLOURISH
+A798; C; A799; # LATIN CAPITAL LETTER F WITH STROKE
+A79A; C; A79B; # LATIN CAPITAL LETTER VOLAPUK AE
+A79C; C; A79D; # LATIN CAPITAL LETTER VOLAPUK OE
+A79E; C; A79F; # LATIN CAPITAL LETTER VOLAPUK UE
+A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE
+A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE
+A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE
+A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE
+A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE
+A7AA; C; 0266; # LATIN CAPITAL LETTER H WITH HOOK
+A7AB; C; 025C; # LATIN CAPITAL LETTER REVERSED OPEN E
+A7AC; C; 0261; # LATIN CAPITAL LETTER SCRIPT G
+A7AD; C; 026C; # LATIN CAPITAL LETTER L WITH BELT
+A7B0; C; 029E; # LATIN CAPITAL LETTER TURNED K
+A7B1; C; 0287; # LATIN CAPITAL LETTER TURNED T
+A7B2; C; 029D; # LATIN CAPITAL LETTER J WITH CROSSED-TAIL
+A7B3; C; AB53; # LATIN CAPITAL LETTER CHI
+A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA
+A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA
+AB70; C; 13A0; # CHEROKEE SMALL LETTER A
+AB71; C; 13A1; # CHEROKEE SMALL LETTER E
+AB72; C; 13A2; # CHEROKEE SMALL LETTER I
+AB73; C; 13A3; # CHEROKEE SMALL LETTER O
+AB74; C; 13A4; # CHEROKEE SMALL LETTER U
+AB75; C; 13A5; # CHEROKEE SMALL LETTER V
+AB76; C; 13A6; # CHEROKEE SMALL LETTER GA
+AB77; C; 13A7; # CHEROKEE SMALL LETTER KA
+AB78; C; 13A8; # CHEROKEE SMALL LETTER GE
+AB79; C; 13A9; # CHEROKEE SMALL LETTER GI
+AB7A; C; 13AA; # CHEROKEE SMALL LETTER GO
+AB7B; C; 13AB; # CHEROKEE SMALL LETTER GU
+AB7C; C; 13AC; # CHEROKEE SMALL LETTER GV
+AB7D; C; 13AD; # CHEROKEE SMALL LETTER HA
+AB7E; C; 13AE; # CHEROKEE SMALL LETTER HE
+AB7F; C; 13AF; # CHEROKEE SMALL LETTER HI
+AB80; C; 13B0; # CHEROKEE SMALL LETTER HO
+AB81; C; 13B1; # CHEROKEE SMALL LETTER HU
+AB82; C; 13B2; # CHEROKEE SMALL LETTER HV
+AB83; C; 13B3; # CHEROKEE SMALL LETTER LA
+AB84; C; 13B4; # CHEROKEE SMALL LETTER LE
+AB85; C; 13B5; # CHEROKEE SMALL LETTER LI
+AB86; C; 13B6; # CHEROKEE SMALL LETTER LO
+AB87; C; 13B7; # CHEROKEE SMALL LETTER LU
+AB88; C; 13B8; # CHEROKEE SMALL LETTER LV
+AB89; C; 13B9; # CHEROKEE SMALL LETTER MA
+AB8A; C; 13BA; # CHEROKEE SMALL LETTER ME
+AB8B; C; 13BB; # CHEROKEE SMALL LETTER MI
+AB8C; C; 13BC; # CHEROKEE SMALL LETTER MO
+AB8D; C; 13BD; # CHEROKEE SMALL LETTER MU
+AB8E; C; 13BE; # CHEROKEE SMALL LETTER NA
+AB8F; C; 13BF; # CHEROKEE SMALL LETTER HNA
+AB90; C; 13C0; # CHEROKEE SMALL LETTER NAH
+AB91; C; 13C1; # CHEROKEE SMALL LETTER NE
+AB92; C; 13C2; # CHEROKEE SMALL LETTER NI
+AB93; C; 13C3; # CHEROKEE SMALL LETTER NO
+AB94; C; 13C4; # CHEROKEE SMALL LETTER NU
+AB95; C; 13C5; # CHEROKEE SMALL LETTER NV
+AB96; C; 13C6; # CHEROKEE SMALL LETTER QUA
+AB97; C; 13C7; # CHEROKEE SMALL LETTER QUE
+AB98; C; 13C8; # CHEROKEE SMALL LETTER QUI
+AB99; C; 13C9; # CHEROKEE SMALL LETTER QUO
+AB9A; C; 13CA; # CHEROKEE SMALL LETTER QUU
+AB9B; C; 13CB; # CHEROKEE SMALL LETTER QUV
+AB9C; C; 13CC; # CHEROKEE SMALL LETTER SA
+AB9D; C; 13CD; # CHEROKEE SMALL LETTER S
+AB9E; C; 13CE; # CHEROKEE SMALL LETTER SE
+AB9F; C; 13CF; # CHEROKEE SMALL LETTER SI
+ABA0; C; 13D0; # CHEROKEE SMALL LETTER SO
+ABA1; C; 13D1; # CHEROKEE SMALL LETTER SU
+ABA2; C; 13D2; # CHEROKEE SMALL LETTER SV
+ABA3; C; 13D3; # CHEROKEE SMALL LETTER DA
+ABA4; C; 13D4; # CHEROKEE SMALL LETTER TA
+ABA5; C; 13D5; # CHEROKEE SMALL LETTER DE
+ABA6; C; 13D6; # CHEROKEE SMALL LETTER TE
+ABA7; C; 13D7; # CHEROKEE SMALL LETTER DI
+ABA8; C; 13D8; # CHEROKEE SMALL LETTER TI
+ABA9; C; 13D9; # CHEROKEE SMALL LETTER DO
+ABAA; C; 13DA; # CHEROKEE SMALL LETTER DU
+ABAB; C; 13DB; # CHEROKEE SMALL LETTER DV
+ABAC; C; 13DC; # CHEROKEE SMALL LETTER DLA
+ABAD; C; 13DD; # CHEROKEE SMALL LETTER TLA
+ABAE; C; 13DE; # CHEROKEE SMALL LETTER TLE
+ABAF; C; 13DF; # CHEROKEE SMALL LETTER TLI
+ABB0; C; 13E0; # CHEROKEE SMALL LETTER TLO
+ABB1; C; 13E1; # CHEROKEE SMALL LETTER TLU
+ABB2; C; 13E2; # CHEROKEE SMALL LETTER TLV
+ABB3; C; 13E3; # CHEROKEE SMALL LETTER TSA
+ABB4; C; 13E4; # CHEROKEE SMALL LETTER TSE
+ABB5; C; 13E5; # CHEROKEE SMALL LETTER TSI
+ABB6; C; 13E6; # CHEROKEE SMALL LETTER TSO
+ABB7; C; 13E7; # CHEROKEE SMALL LETTER TSU
+ABB8; C; 13E8; # CHEROKEE SMALL LETTER TSV
+ABB9; C; 13E9; # CHEROKEE SMALL LETTER WA
+ABBA; C; 13EA; # CHEROKEE SMALL LETTER WE
+ABBB; C; 13EB; # CHEROKEE SMALL LETTER WI
+ABBC; C; 13EC; # CHEROKEE SMALL LETTER WO
+ABBD; C; 13ED; # CHEROKEE SMALL LETTER WU
+ABBE; C; 13EE; # CHEROKEE SMALL LETTER WV
+ABBF; C; 13EF; # CHEROKEE SMALL LETTER YA
+FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF
+FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI
+FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
+FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI
+FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL
+FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T
+FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST
+FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW
+FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH
+FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI
+FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW
+FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH
+FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A
+FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B
+FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C
+FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D
+FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E
+FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F
+FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G
+FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H
+FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I
+FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J
+FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K
+FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L
+FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M
+FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N
+FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O
+FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P
+FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q
+FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R
+FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S
+FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T
+FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U
+FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V
+FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W
+FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X
+FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y
+FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
+10400; C; 10428; # DESERET CAPITAL LETTER LONG I
+10401; C; 10429; # DESERET CAPITAL LETTER LONG E
+10402; C; 1042A; # DESERET CAPITAL LETTER LONG A
+10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH
+10404; C; 1042C; # DESERET CAPITAL LETTER LONG O
+10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO
+10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I
+10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E
+10408; C; 10430; # DESERET CAPITAL LETTER SHORT A
+10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH
+1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O
+1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO
+1040C; C; 10434; # DESERET CAPITAL LETTER AY
+1040D; C; 10435; # DESERET CAPITAL LETTER OW
+1040E; C; 10436; # DESERET CAPITAL LETTER WU
+1040F; C; 10437; # DESERET CAPITAL LETTER YEE
+10410; C; 10438; # DESERET CAPITAL LETTER H
+10411; C; 10439; # DESERET CAPITAL LETTER PEE
+10412; C; 1043A; # DESERET CAPITAL LETTER BEE
+10413; C; 1043B; # DESERET CAPITAL LETTER TEE
+10414; C; 1043C; # DESERET CAPITAL LETTER DEE
+10415; C; 1043D; # DESERET CAPITAL LETTER CHEE
+10416; C; 1043E; # DESERET CAPITAL LETTER JEE
+10417; C; 1043F; # DESERET CAPITAL LETTER KAY
+10418; C; 10440; # DESERET CAPITAL LETTER GAY
+10419; C; 10441; # DESERET CAPITAL LETTER EF
+1041A; C; 10442; # DESERET CAPITAL LETTER VEE
+1041B; C; 10443; # DESERET CAPITAL LETTER ETH
+1041C; C; 10444; # DESERET CAPITAL LETTER THEE
+1041D; C; 10445; # DESERET CAPITAL LETTER ES
+1041E; C; 10446; # DESERET CAPITAL LETTER ZEE
+1041F; C; 10447; # DESERET CAPITAL LETTER ESH
+10420; C; 10448; # DESERET CAPITAL LETTER ZHEE
+10421; C; 10449; # DESERET CAPITAL LETTER ER
+10422; C; 1044A; # DESERET CAPITAL LETTER EL
+10423; C; 1044B; # DESERET CAPITAL LETTER EM
+10424; C; 1044C; # DESERET CAPITAL LETTER EN
+10425; C; 1044D; # DESERET CAPITAL LETTER ENG
+10426; C; 1044E; # DESERET CAPITAL LETTER OI
+10427; C; 1044F; # DESERET CAPITAL LETTER EW
+10C80; C; 10CC0; # OLD HUNGARIAN CAPITAL LETTER A
+10C81; C; 10CC1; # OLD HUNGARIAN CAPITAL LETTER AA
+10C82; C; 10CC2; # OLD HUNGARIAN CAPITAL LETTER EB
+10C83; C; 10CC3; # OLD HUNGARIAN CAPITAL LETTER AMB
+10C84; C; 10CC4; # OLD HUNGARIAN CAPITAL LETTER EC
+10C85; C; 10CC5; # OLD HUNGARIAN CAPITAL LETTER ENC
+10C86; C; 10CC6; # OLD HUNGARIAN CAPITAL LETTER ECS
+10C87; C; 10CC7; # OLD HUNGARIAN CAPITAL LETTER ED
+10C88; C; 10CC8; # OLD HUNGARIAN CAPITAL LETTER AND
+10C89; C; 10CC9; # OLD HUNGARIAN CAPITAL LETTER E
+10C8A; C; 10CCA; # OLD HUNGARIAN CAPITAL LETTER CLOSE E
+10C8B; C; 10CCB; # OLD HUNGARIAN CAPITAL LETTER EE
+10C8C; C; 10CCC; # OLD HUNGARIAN CAPITAL LETTER EF
+10C8D; C; 10CCD; # OLD HUNGARIAN CAPITAL LETTER EG
+10C8E; C; 10CCE; # OLD HUNGARIAN CAPITAL LETTER EGY
+10C8F; C; 10CCF; # OLD HUNGARIAN CAPITAL LETTER EH
+10C90; C; 10CD0; # OLD HUNGARIAN CAPITAL LETTER I
+10C91; C; 10CD1; # OLD HUNGARIAN CAPITAL LETTER II
+10C92; C; 10CD2; # OLD HUNGARIAN CAPITAL LETTER EJ
+10C93; C; 10CD3; # OLD HUNGARIAN CAPITAL LETTER EK
+10C94; C; 10CD4; # OLD HUNGARIAN CAPITAL LETTER AK
+10C95; C; 10CD5; # OLD HUNGARIAN CAPITAL LETTER UNK
+10C96; C; 10CD6; # OLD HUNGARIAN CAPITAL LETTER EL
+10C97; C; 10CD7; # OLD HUNGARIAN CAPITAL LETTER ELY
+10C98; C; 10CD8; # OLD HUNGARIAN CAPITAL LETTER EM
+10C99; C; 10CD9; # OLD HUNGARIAN CAPITAL LETTER EN
+10C9A; C; 10CDA; # OLD HUNGARIAN CAPITAL LETTER ENY
+10C9B; C; 10CDB; # OLD HUNGARIAN CAPITAL LETTER O
+10C9C; C; 10CDC; # OLD HUNGARIAN CAPITAL LETTER OO
+10C9D; C; 10CDD; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE
+10C9E; C; 10CDE; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE
+10C9F; C; 10CDF; # OLD HUNGARIAN CAPITAL LETTER OEE
+10CA0; C; 10CE0; # OLD HUNGARIAN CAPITAL LETTER EP
+10CA1; C; 10CE1; # OLD HUNGARIAN CAPITAL LETTER EMP
+10CA2; C; 10CE2; # OLD HUNGARIAN CAPITAL LETTER ER
+10CA3; C; 10CE3; # OLD HUNGARIAN CAPITAL LETTER SHORT ER
+10CA4; C; 10CE4; # OLD HUNGARIAN CAPITAL LETTER ES
+10CA5; C; 10CE5; # OLD HUNGARIAN CAPITAL LETTER ESZ
+10CA6; C; 10CE6; # OLD HUNGARIAN CAPITAL LETTER ET
+10CA7; C; 10CE7; # OLD HUNGARIAN CAPITAL LETTER ENT
+10CA8; C; 10CE8; # OLD HUNGARIAN CAPITAL LETTER ETY
+10CA9; C; 10CE9; # OLD HUNGARIAN CAPITAL LETTER ECH
+10CAA; C; 10CEA; # OLD HUNGARIAN CAPITAL LETTER U
+10CAB; C; 10CEB; # OLD HUNGARIAN CAPITAL LETTER UU
+10CAC; C; 10CEC; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE
+10CAD; C; 10CED; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE
+10CAE; C; 10CEE; # OLD HUNGARIAN CAPITAL LETTER EV
+10CAF; C; 10CEF; # OLD HUNGARIAN CAPITAL LETTER EZ
+10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS
+10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN
+10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US
+118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA
+118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A
+118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI
+118A3; C; 118C3; # WARANG CITI CAPITAL LETTER YU
+118A4; C; 118C4; # WARANG CITI CAPITAL LETTER YA
+118A5; C; 118C5; # WARANG CITI CAPITAL LETTER YO
+118A6; C; 118C6; # WARANG CITI CAPITAL LETTER II
+118A7; C; 118C7; # WARANG CITI CAPITAL LETTER UU
+118A8; C; 118C8; # WARANG CITI CAPITAL LETTER E
+118A9; C; 118C9; # WARANG CITI CAPITAL LETTER O
+118AA; C; 118CA; # WARANG CITI CAPITAL LETTER ANG
+118AB; C; 118CB; # WARANG CITI CAPITAL LETTER GA
+118AC; C; 118CC; # WARANG CITI CAPITAL LETTER KO
+118AD; C; 118CD; # WARANG CITI CAPITAL LETTER ENY
+118AE; C; 118CE; # WARANG CITI CAPITAL LETTER YUJ
+118AF; C; 118CF; # WARANG CITI CAPITAL LETTER UC
+118B0; C; 118D0; # WARANG CITI CAPITAL LETTER ENN
+118B1; C; 118D1; # WARANG CITI CAPITAL LETTER ODD
+118B2; C; 118D2; # WARANG CITI CAPITAL LETTER TTE
+118B3; C; 118D3; # WARANG CITI CAPITAL LETTER NUNG
+118B4; C; 118D4; # WARANG CITI CAPITAL LETTER DA
+118B5; C; 118D5; # WARANG CITI CAPITAL LETTER AT
+118B6; C; 118D6; # WARANG CITI CAPITAL LETTER AM
+118B7; C; 118D7; # WARANG CITI CAPITAL LETTER BU
+118B8; C; 118D8; # WARANG CITI CAPITAL LETTER PU
+118B9; C; 118D9; # WARANG CITI CAPITAL LETTER HIYO
+118BA; C; 118DA; # WARANG CITI CAPITAL LETTER HOLO
+118BB; C; 118DB; # WARANG CITI CAPITAL LETTER HORR
+118BC; C; 118DC; # WARANG CITI CAPITAL LETTER HAR
+118BD; C; 118DD; # WARANG CITI CAPITAL LETTER SSUU
+118BE; C; 118DE; # WARANG CITI CAPITAL LETTER SII
+118BF; C; 118DF; # WARANG CITI CAPITAL LETTER VIYO
+#
+# EOF
diff --git a/rpython/rlib/unicodedata/CompositionExclusions-8.0.0.txt b/rpython/rlib/unicodedata/CompositionExclusions-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CompositionExclusions-8.0.0.txt
@@ -0,0 +1,206 @@
+# CompositionExclusions-8.0.0.txt
+# Date: 2015-02-19, 00:30:00 GMT [KW, LI]
+#
+# This file lists the characters for the Composition Exclusion Table
+# defined in UAX #15, Unicode Normalization Forms.
+#
+# This file is a normative contributory data file in the
+# Unicode Character Database.
+#
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+#
+# For more information, see
+# http://www.unicode.org/unicode/reports/tr15/#Primary_Exclusion_List_Table
+#
+# For a full derivation of composition exclusions, see the derived property
+# Full_Composition_Exclusion in DerivedNormalizationProps.txt
+#
+
+# ================================================
+# (1) Script Specifics
+#
+# This list of characters cannot be derived from the UnicodeData.txt file.
+# ================================================
+
+0958    #  DEVANAGARI LETTER QA
+0959    #  DEVANAGARI LETTER KHHA
+095A    #  DEVANAGARI LETTER GHHA
+095B    #  DEVANAGARI LETTER ZA
+095C    #  DEVANAGARI LETTER DDDHA
+095D    #  DEVANAGARI LETTER RHA
+095E    #  DEVANAGARI LETTER FA
+095F    #  DEVANAGARI LETTER YYA
+09DC    #  BENGALI LETTER RRA
+09DD    #  BENGALI LETTER RHA
+09DF    #  BENGALI LETTER YYA
+0A33    #  GURMUKHI LETTER LLA
+0A36    #  GURMUKHI LETTER SHA
+0A59    #  GURMUKHI LETTER KHHA
+0A5A    #  GURMUKHI LETTER GHHA
+0A5B    #  GURMUKHI LETTER ZA
+0A5E    #  GURMUKHI LETTER FA
+0B5C    #  ORIYA LETTER RRA
+0B5D    #  ORIYA LETTER RHA
+0F43    #  TIBETAN LETTER GHA
+0F4D    #  TIBETAN LETTER DDHA
+0F52    #  TIBETAN LETTER DHA
+0F57    #  TIBETAN LETTER BHA
+0F5C    #  TIBETAN LETTER DZHA
+0F69    #  TIBETAN LETTER KSSA
+0F76    #  TIBETAN VOWEL SIGN VOCALIC R
+0F78    #  TIBETAN VOWEL SIGN VOCALIC L
+0F93    #  TIBETAN SUBJOINED LETTER GHA
+0F9D    #  TIBETAN SUBJOINED LETTER DDHA
+0FA2    #  TIBETAN SUBJOINED LETTER DHA
+0FA7    #  TIBETAN SUBJOINED LETTER BHA
+0FAC    #  TIBETAN SUBJOINED LETTER DZHA
+0FB9    #  TIBETAN SUBJOINED LETTER KSSA
+FB1D    #  HEBREW LETTER YOD WITH HIRIQ
+FB1F    #  HEBREW LIGATURE YIDDISH YOD YOD PATAH
+FB2A    #  HEBREW LETTER SHIN WITH SHIN DOT
+FB2B    #  HEBREW LETTER SHIN WITH SIN DOT
+FB2C    #  HEBREW LETTER SHIN WITH DAGESH AND SHIN DOT
+FB2D    #  HEBREW LETTER SHIN WITH DAGESH AND SIN DOT
+FB2E    #  HEBREW LETTER ALEF WITH PATAH
+FB2F    #  HEBREW LETTER ALEF WITH QAMATS
+FB30    #  HEBREW LETTER ALEF WITH MAPIQ
+FB31    #  HEBREW LETTER BET WITH DAGESH
+FB32    #  HEBREW LETTER GIMEL WITH DAGESH
+FB33    #  HEBREW LETTER DALET WITH DAGESH
+FB34    #  HEBREW LETTER HE WITH MAPIQ
+FB35    #  HEBREW LETTER VAV WITH DAGESH
+FB36    #  HEBREW LETTER ZAYIN WITH DAGESH
+FB38    #  HEBREW LETTER TET WITH DAGESH
+FB39    #  HEBREW LETTER YOD WITH DAGESH
+FB3A    #  HEBREW LETTER FINAL KAF WITH DAGESH
+FB3B    #  HEBREW LETTER KAF WITH DAGESH
+FB3C    #  HEBREW LETTER LAMED WITH DAGESH
+FB3E    #  HEBREW LETTER MEM WITH DAGESH
+FB40    #  HEBREW LETTER NUN WITH DAGESH
+FB41    #  HEBREW LETTER SAMEKH WITH DAGESH
+FB43    #  HEBREW LETTER FINAL PE WITH DAGESH
+FB44    #  HEBREW LETTER PE WITH DAGESH
+FB46    #  HEBREW LETTER TSADI WITH DAGESH
+FB47    #  HEBREW LETTER QOF WITH DAGESH
+FB48    #  HEBREW LETTER RESH WITH DAGESH
+FB49    #  HEBREW LETTER SHIN WITH DAGESH
+FB4A    #  HEBREW LETTER TAV WITH DAGESH
+FB4B    #  HEBREW LETTER VAV WITH HOLAM
+FB4C    #  HEBREW LETTER BET WITH RAFE
+FB4D    #  HEBREW LETTER KAF WITH RAFE
+FB4E    #  HEBREW LETTER PE WITH RAFE
+
+# Total code points: 67
+
+# ================================================
+# (2) Post Composition Version precomposed characters
+#
+# These characters cannot be derived solely from the UnicodeData.txt file
+# in this version of Unicode.
+#
+# Note that characters added to the standard after the
+# Composition Version and which have canonical decomposition mappings
+# are not automatically added to this list of Post Composition
+# Version precomposed characters.
+# ================================================
+
+2ADC    #  FORKING
+1D15E   #  MUSICAL SYMBOL HALF NOTE
+1D15F   #  MUSICAL SYMBOL QUARTER NOTE
+1D160   #  MUSICAL SYMBOL EIGHTH NOTE
+1D161   #  MUSICAL SYMBOL SIXTEENTH NOTE
+1D162   #  MUSICAL SYMBOL THIRTY-SECOND NOTE
+1D163   #  MUSICAL SYMBOL SIXTY-FOURTH NOTE
+1D164   #  MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE
+1D1BB   #  MUSICAL SYMBOL MINIMA
+1D1BC   #  MUSICAL SYMBOL MINIMA BLACK
+1D1BD   #  MUSICAL SYMBOL SEMIMINIMA WHITE
+1D1BE   #  MUSICAL SYMBOL SEMIMINIMA BLACK
+1D1BF   #  MUSICAL SYMBOL FUSA WHITE
+1D1C0   #  MUSICAL SYMBOL FUSA BLACK
+
+# Total code points: 14
+
+# ================================================
+# (3) Singleton Decompositions
+#
+# These characters can be derived from the UnicodeData.txt file
+# by including all canonically decomposable characters whose
+# canonical decomposition consists of a single character.
+#
+# These characters are simply quoted here for reference.
+# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt
+# ================================================
+
+# 0340..0341       [2] COMBINING GRAVE TONE MARK..COMBINING ACUTE TONE MARK
+# 0343                 COMBINING GREEK KORONIS
+# 0374                 GREEK NUMERAL SIGN
+# 037E                 GREEK QUESTION MARK
+# 0387                 GREEK ANO TELEIA
+# 1F71                 GREEK SMALL LETTER ALPHA WITH OXIA
+# 1F73                 GREEK SMALL LETTER EPSILON WITH OXIA
+# 1F75                 GREEK SMALL LETTER ETA WITH OXIA
+# 1F77                 GREEK SMALL LETTER IOTA WITH OXIA
+# 1F79                 GREEK SMALL LETTER OMICRON WITH OXIA
+# 1F7B                 GREEK SMALL LETTER UPSILON WITH OXIA
+# 1F7D                 GREEK SMALL LETTER OMEGA WITH OXIA
+# 1FBB                 GREEK CAPITAL LETTER ALPHA WITH OXIA
+# 1FBE                 GREEK PROSGEGRAMMENI
+# 1FC9                 GREEK CAPITAL LETTER EPSILON WITH OXIA
+# 1FCB                 GREEK CAPITAL LETTER ETA WITH OXIA
+# 1FD3                 GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+# 1FDB                 GREEK CAPITAL LETTER IOTA WITH OXIA
+# 1FE3                 GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+# 1FEB                 GREEK CAPITAL LETTER UPSILON WITH OXIA
+# 1FEE..1FEF       [2] GREEK DIALYTIKA AND OXIA..GREEK VARIA
+# 1FF9                 GREEK CAPITAL LETTER OMICRON WITH OXIA
+# 1FFB                 GREEK CAPITAL LETTER OMEGA WITH OXIA
+# 1FFD                 GREEK OXIA
+# 2000..2001       [2] EN QUAD..EM QUAD
+# 2126                 OHM SIGN
+# 212A..212B       [2] KELVIN SIGN..ANGSTROM SIGN
+# 2329                 LEFT-POINTING ANGLE BRACKET
+# 232A                 RIGHT-POINTING ANGLE BRACKET
+# F900..FA0D     [270] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA0D
+# FA10                 CJK COMPATIBILITY IDEOGRAPH-FA10
+# FA12                 CJK COMPATIBILITY IDEOGRAPH-FA12
+# FA15..FA1E      [10] CJK COMPATIBILITY IDEOGRAPH-FA15..CJK COMPATIBILITY IDEOGRAPH-FA1E
+# FA20                 CJK COMPATIBILITY IDEOGRAPH-FA20
+# FA22                 CJK COMPATIBILITY IDEOGRAPH-FA22
+# FA25..FA26       [2] CJK COMPATIBILITY IDEOGRAPH-FA25..CJK COMPATIBILITY IDEOGRAPH-FA26
+# FA2A..FA6D      [68] CJK COMPATIBILITY IDEOGRAPH-FA2A..CJK COMPATIBILITY IDEOGRAPH-FA6D
+# FA70..FAD9     [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9
+# 2F800..2FA1D   [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
+
+# Total code points: 1035
+
+# ================================================
+# (4) Non-Starter Decompositions
+#
+# These characters can be derived from the UnicodeData.txt file
+# by including each expanding canonical decomposition
+# (i.e., those which canonically decompose to a sequence
+# of characters instead of a single character), such that:
+#
+# A. The character is not a Starter.
+#
+# OR (inclusive)
+#
+# B. The character's canonical decomposition begins
+# with a character that is not a Starter.
+#
+# Note that a "Starter" is any character with a zero combining class.
+#
+# These characters are simply quoted here for reference.
+# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt
+# ================================================
+
+# 0344                 COMBINING GREEK DIALYTIKA TONOS
+# 0F73                 TIBETAN VOWEL SIGN II
+# 0F75                 TIBETAN VOWEL SIGN UU
+# 0F81                 TIBETAN VOWEL SIGN REVERSED II
+
+# Total code points: 4
+
+# EOF
diff --git a/rpython/rlib/unicodedata/DerivedCoreProperties-8.0.0.txt b/rpython/rlib/unicodedata/DerivedCoreProperties-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/DerivedCoreProperties-8.0.0.txt
@@ -0,0 +1,11029 @@
+# DerivedCoreProperties-8.0.0.txt
+# Date: 2015-03-11, 22:29:21 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+
+# ================================================
+
+# Derived Property: Math
+#  Generated from: Sm + Other_Math
+
+002B          ; Math # Sm       PLUS SIGN
+003C..003E    ; Math # Sm   [3] LESS-THAN SIGN..GREATER-THAN SIGN
+005E          ; Math # Sk       CIRCUMFLEX ACCENT
+007C          ; Math # Sm       VERTICAL LINE
+007E          ; Math # Sm       TILDE
+00AC          ; Math # Sm       NOT SIGN
+00B1          ; Math # Sm       PLUS-MINUS SIGN
+00D7          ; Math # Sm       MULTIPLICATION SIGN
+00F7          ; Math # Sm       DIVISION SIGN
+03D0..03D2    ; Math # L&   [3] GREEK BETA SYMBOL..GREEK UPSILON WITH HOOK SYMBOL
+03D5          ; Math # L&       GREEK PHI SYMBOL
+03F0..03F1    ; Math # L&   [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL
+03F4..03F5    ; Math # L&   [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL
+03F6          ; Math # Sm       GREEK REVERSED LUNATE EPSILON SYMBOL
+0606..0608    ; Math # Sm   [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY
+2016          ; Math # Po       DOUBLE VERTICAL LINE
+2032..2034    ; Math # Po   [3] PRIME..TRIPLE PRIME
+2040          ; Math # Pc       CHARACTER TIE
+2044          ; Math # Sm       FRACTION SLASH
+2052          ; Math # Sm       COMMERCIAL MINUS SIGN
+2061..2064    ; Math # Cf   [4] FUNCTION APPLICATION..INVISIBLE PLUS
+207A..207C    ; Math # Sm   [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN
+207D          ; Math # Ps       SUPERSCRIPT LEFT PARENTHESIS
+207E          ; Math # Pe       SUPERSCRIPT RIGHT PARENTHESIS
+208A..208C    ; Math # Sm   [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN
+208D          ; Math # Ps       SUBSCRIPT LEFT PARENTHESIS
+208E          ; Math # Pe       SUBSCRIPT RIGHT PARENTHESIS
+20D0..20DC    ; Math # Mn  [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE
+20E1          ; Math # Mn       COMBINING LEFT RIGHT ARROW ABOVE
+20E5..20E6    ; Math # Mn   [2] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING DOUBLE VERTICAL STROKE OVERLAY
+20EB..20EF    ; Math # Mn   [5] COMBINING LONG DOUBLE SOLIDUS OVERLAY..COMBINING RIGHT ARROW BELOW
+2102          ; Math # L&       DOUBLE-STRUCK CAPITAL C
+2107          ; Math # L&       EULER CONSTANT
+210A..2113    ; Math # L&  [10] SCRIPT SMALL G..SCRIPT SMALL L
+2115          ; Math # L&       DOUBLE-STRUCK CAPITAL N
+2118          ; Math # Sm       SCRIPT CAPITAL P
+2119..211D    ; Math # L&   [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R
+2124          ; Math # L&       DOUBLE-STRUCK CAPITAL Z
+2128          ; Math # L&       BLACK-LETTER CAPITAL Z
+2129          ; Math # So       TURNED GREEK SMALL LETTER IOTA
+212C..212D    ; Math # L&   [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C
+212F..2131    ; Math # L&   [3] SCRIPT SMALL E..SCRIPT CAPITAL F
+2133..2134    ; Math # L&   [2] SCRIPT CAPITAL M..SCRIPT SMALL O
+2135..2138    ; Math # Lo   [4] ALEF SYMBOL..DALET SYMBOL
+213C..213F    ; Math # L&   [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI
+2140..2144    ; Math # Sm   [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y
+2145..2149    ; Math # L&   [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J
+214B          ; Math # Sm       TURNED AMPERSAND
+2190..2194    ; Math # Sm   [5] LEFTWARDS ARROW..LEFT RIGHT ARROW
+2195..2199    ; Math # So   [5] UP DOWN ARROW..SOUTH WEST ARROW
+219A..219B    ; Math # Sm   [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE
+219C..219F    ; Math # So   [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW
+21A0          ; Math # Sm       RIGHTWARDS TWO HEADED ARROW
+21A1..21A2    ; Math # So   [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL
+21A3          ; Math # Sm       RIGHTWARDS ARROW WITH TAIL
+21A4..21A5    ; Math # So   [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR
+21A6          ; Math # Sm       RIGHTWARDS ARROW FROM BAR
+21A7          ; Math # So       DOWNWARDS ARROW FROM BAR
+21A9..21AD    ; Math # So   [5] LEFTWARDS ARROW WITH HOOK..LEFT RIGHT WAVE ARROW
+21AE          ; Math # Sm       LEFT RIGHT ARROW WITH STROKE
+21B0..21B1    ; Math # So   [2] UPWARDS ARROW WITH TIP LEFTWARDS..UPWARDS ARROW WITH TIP RIGHTWARDS
+21B6..21B7    ; Math # So   [2] ANTICLOCKWISE TOP SEMICIRCLE ARROW..CLOCKWISE TOP SEMICIRCLE ARROW
+21BC..21CD    ; Math # So  [18] LEFTWARDS HARPOON WITH BARB UPWARDS..LEFTWARDS DOUBLE ARROW WITH STROKE
+21CE..21CF    ; Math # Sm   [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE
+21D0..21D1    ; Math # So   [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW
+21D2          ; Math # Sm       RIGHTWARDS DOUBLE ARROW
+21D3          ; Math # So       DOWNWARDS DOUBLE ARROW
+21D4          ; Math # Sm       LEFT RIGHT DOUBLE ARROW
+21D5..21DB    ; Math # So   [7] UP DOWN DOUBLE ARROW..RIGHTWARDS TRIPLE ARROW
+21DD          ; Math # So       RIGHTWARDS SQUIGGLE ARROW
+21E4..21E5    ; Math # So   [2] LEFTWARDS ARROW TO BAR..RIGHTWARDS ARROW TO BAR
+21F4..22FF    ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP
+2308          ; Math # Ps       LEFT CEILING
+2309          ; Math # Pe       RIGHT CEILING
+230A          ; Math # Ps       LEFT FLOOR
+230B          ; Math # Pe       RIGHT FLOOR
+2320..2321    ; Math # Sm   [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL
+237C          ; Math # Sm       RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW
+239B..23B3    ; Math # Sm  [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM
+23B4..23B5    ; Math # So   [2] TOP SQUARE BRACKET..BOTTOM SQUARE BRACKET
+23B7          ; Math # So       RADICAL SYMBOL BOTTOM
+23D0          ; Math # So       VERTICAL LINE EXTENSION
+23DC..23E1    ; Math # Sm   [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET
+23E2          ; Math # So       WHITE TRAPEZIUM
+25A0..25A1    ; Math # So   [2] BLACK SQUARE..WHITE SQUARE
+25AE..25B6    ; Math # So   [9] BLACK VERTICAL RECTANGLE..BLACK RIGHT-POINTING TRIANGLE
+25B7          ; Math # Sm       WHITE RIGHT-POINTING TRIANGLE
+25BC..25C0    ; Math # So   [5] BLACK DOWN-POINTING TRIANGLE..BLACK LEFT-POINTING TRIANGLE
+25C1          ; Math # Sm       WHITE LEFT-POINTING TRIANGLE
+25C6..25C7    ; Math # So   [2] BLACK DIAMOND..WHITE DIAMOND
+25CA..25CB    ; Math # So   [2] LOZENGE..WHITE CIRCLE
+25CF..25D3    ; Math # So   [5] BLACK CIRCLE..CIRCLE WITH UPPER HALF BLACK
+25E2          ; Math # So       BLACK LOWER RIGHT TRIANGLE
+25E4          ; Math # So       BLACK UPPER LEFT TRIANGLE
+25E7..25EC    ; Math # So   [6] SQUARE WITH LEFT HALF BLACK..WHITE UP-POINTING TRIANGLE WITH DOT
+25F8..25FF    ; Math # Sm   [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE
+2605..2606    ; Math # So   [2] BLACK STAR..WHITE STAR
+2640          ; Math # So       FEMALE SIGN
+2642          ; Math # So       MALE SIGN
+2660..2663    ; Math # So   [4] BLACK SPADE SUIT..BLACK CLUB SUIT
+266D..266E    ; Math # So   [2] MUSIC FLAT SIGN..MUSIC NATURAL SIGN
+266F          ; Math # Sm       MUSIC SHARP SIGN
+27C0..27C4    ; Math # Sm   [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET
+27C5          ; Math # Ps       LEFT S-SHAPED BAG DELIMITER
+27C6          ; Math # Pe       RIGHT S-SHAPED BAG DELIMITER
+27C7..27E5    ; Math # Sm  [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK
+27E6          ; Math # Ps       MATHEMATICAL LEFT WHITE SQUARE BRACKET
+27E7          ; Math # Pe       MATHEMATICAL RIGHT WHITE SQUARE BRACKET
+27E8          ; Math # Ps       MATHEMATICAL LEFT ANGLE BRACKET
+27E9          ; Math # Pe       MATHEMATICAL RIGHT ANGLE BRACKET
+27EA          ; Math # Ps       MATHEMATICAL LEFT DOUBLE ANGLE BRACKET
+27EB          ; Math # Pe       MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET
+27EC          ; Math # Ps       MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET
+27ED          ; Math # Pe       MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET
+27EE          ; Math # Ps       MATHEMATICAL LEFT FLATTENED PARENTHESIS
+27EF          ; Math # Pe       MATHEMATICAL RIGHT FLATTENED PARENTHESIS
+27F0..27FF    ; Math # Sm  [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW
+2900..2982    ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON
+2983          ; Math # Ps       LEFT WHITE CURLY BRACKET
+2984          ; Math # Pe       RIGHT WHITE CURLY BRACKET
+2985          ; Math # Ps       LEFT WHITE PARENTHESIS
+2986          ; Math # Pe       RIGHT WHITE PARENTHESIS
+2987          ; Math # Ps       Z NOTATION LEFT IMAGE BRACKET
+2988          ; Math # Pe       Z NOTATION RIGHT IMAGE BRACKET
+2989          ; Math # Ps       Z NOTATION LEFT BINDING BRACKET
+298A          ; Math # Pe       Z NOTATION RIGHT BINDING BRACKET
+298B          ; Math # Ps       LEFT SQUARE BRACKET WITH UNDERBAR
+298C          ; Math # Pe       RIGHT SQUARE BRACKET WITH UNDERBAR
+298D          ; Math # Ps       LEFT SQUARE BRACKET WITH TICK IN TOP CORNER
+298E          ; Math # Pe       RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER
+298F          ; Math # Ps       LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER
+2990          ; Math # Pe       RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER
+2991          ; Math # Ps       LEFT ANGLE BRACKET WITH DOT
+2992          ; Math # Pe       RIGHT ANGLE BRACKET WITH DOT
+2993          ; Math # Ps       LEFT ARC LESS-THAN BRACKET
+2994          ; Math # Pe       RIGHT ARC GREATER-THAN BRACKET
+2995          ; Math # Ps       DOUBLE LEFT ARC GREATER-THAN BRACKET
+2996          ; Math # Pe       DOUBLE RIGHT ARC LESS-THAN BRACKET
+2997          ; Math # Ps       LEFT BLACK TORTOISE SHELL BRACKET
+2998          ; Math # Pe       RIGHT BLACK TORTOISE SHELL BRACKET
+2999..29D7    ; Math # Sm  [63] DOTTED FENCE..BLACK HOURGLASS
+29D8          ; Math # Ps       LEFT WIGGLY FENCE
+29D9          ; Math # Pe       RIGHT WIGGLY FENCE
+29DA          ; Math # Ps       LEFT DOUBLE WIGGLY FENCE
+29DB          ; Math # Pe       RIGHT DOUBLE WIGGLY FENCE
+29DC..29FB    ; Math # Sm  [32] INCOMPLETE INFINITY..TRIPLE PLUS
+29FC          ; Math # Ps       LEFT-POINTING CURVED ANGLE BRACKET
+29FD          ; Math # Pe       RIGHT-POINTING CURVED ANGLE BRACKET
+29FE..2AFF    ; Math # Sm [258] TINY..N-ARY WHITE VERTICAL BAR
+2B30..2B44    ; Math # Sm  [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET
+2B47..2B4C    ; Math # Sm   [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR
+FB29          ; Math # Sm       HEBREW LETTER ALTERNATIVE PLUS SIGN
+FE61          ; Math # Po       SMALL ASTERISK
+FE62          ; Math # Sm       SMALL PLUS SIGN
+FE63          ; Math # Pd       SMALL HYPHEN-MINUS
+FE64..FE66    ; Math # Sm   [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN
+FE68          ; Math # Po       SMALL REVERSE SOLIDUS
+FF0B          ; Math # Sm       FULLWIDTH PLUS SIGN
+FF1C..FF1E    ; Math # Sm   [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN
+FF3C          ; Math # Po       FULLWIDTH REVERSE SOLIDUS
+FF3E          ; Math # Sk       FULLWIDTH CIRCUMFLEX ACCENT
+FF5C          ; Math # Sm       FULLWIDTH VERTICAL LINE
+FF5E          ; Math # Sm       FULLWIDTH TILDE
+FFE2          ; Math # Sm       FULLWIDTH NOT SIGN
+FFE9..FFEC    ; Math # Sm   [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW
+1D400..1D454  ; Math # L&  [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G
+1D456..1D49C  ; Math # L&  [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A
+1D49E..1D49F  ; Math # L&   [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D
+1D4A2         ; Math # L&       MATHEMATICAL SCRIPT CAPITAL G
+1D4A5..1D4A6  ; Math # L&   [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K
+1D4A9..1D4AC  ; Math # L&   [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q
+1D4AE..1D4B9  ; Math # L&  [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D
+1D4BB         ; Math # L&       MATHEMATICAL SCRIPT SMALL F
+1D4BD..1D4C3  ; Math # L&   [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N
+1D4C5..1D505  ; Math # L&  [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B
+1D507..1D50A  ; Math # L&   [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G
+1D50D..1D514  ; Math # L&   [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q
+1D516..1D51C  ; Math # L&   [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y
+1D51E..1D539  ; Math # L&  [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B
+1D53B..1D53E  ; Math # L&   [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G
+1D540..1D544  ; Math # L&   [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M
+1D546         ; Math # L&       MATHEMATICAL DOUBLE-STRUCK CAPITAL O
+1D54A..1D550  ; Math # L&   [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y
+1D552..1D6A5  ; Math # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J
+1D6A8..1D6C0  ; Math # L&  [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA
+1D6C1         ; Math # Sm       MATHEMATICAL BOLD NABLA
+1D6C2..1D6DA  ; Math # L&  [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA
+1D6DB         ; Math # Sm       MATHEMATICAL BOLD PARTIAL DIFFERENTIAL
+1D6DC..1D6FA  ; Math # L&  [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA
+1D6FB         ; Math # Sm       MATHEMATICAL ITALIC NABLA
+1D6FC..1D714  ; Math # L&  [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA
+1D715         ; Math # Sm       MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL
+1D716..1D734  ; Math # L&  [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA
+1D735         ; Math # Sm       MATHEMATICAL BOLD ITALIC NABLA
+1D736..1D74E  ; Math # L&  [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA
+1D74F         ; Math # Sm       MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL
+1D750..1D76E  ; Math # L&  [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA
+1D76F         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD NABLA
+1D770..1D788  ; Math # L&  [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA
+1D789         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL
+1D78A..1D7A8  ; Math # L&  [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA
+1D7A9         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA
+1D7AA..1D7C2  ; Math # L&  [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA
+1D7C3         ; Math # Sm       MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL
+1D7C4..1D7CB  ; Math # L&   [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA
+1D7CE..1D7FF  ; Math # Nd  [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE
+1EE00..1EE03  ; Math # Lo   [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL
+1EE05..1EE1F  ; Math # Lo  [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF
+1EE21..1EE22  ; Math # Lo   [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM
+1EE24         ; Math # Lo       ARABIC MATHEMATICAL INITIAL HEH
+1EE27         ; Math # Lo       ARABIC MATHEMATICAL INITIAL HAH
+1EE29..1EE32  ; Math # Lo  [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF
+1EE34..1EE37  ; Math # Lo   [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH
+1EE39         ; Math # Lo       ARABIC MATHEMATICAL INITIAL DAD
+1EE3B         ; Math # Lo       ARABIC MATHEMATICAL INITIAL GHAIN
+1EE42         ; Math # Lo       ARABIC MATHEMATICAL TAILED JEEM
+1EE47         ; Math # Lo       ARABIC MATHEMATICAL TAILED HAH
+1EE49         ; Math # Lo       ARABIC MATHEMATICAL TAILED YEH
+1EE4B         ; Math # Lo       ARABIC MATHEMATICAL TAILED LAM
+1EE4D..1EE4F  ; Math # Lo   [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN
+1EE51..1EE52  ; Math # Lo   [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF
+1EE54         ; Math # Lo       ARABIC MATHEMATICAL TAILED SHEEN
+1EE57         ; Math # Lo       ARABIC MATHEMATICAL TAILED KHAH
+1EE59         ; Math # Lo       ARABIC MATHEMATICAL TAILED DAD
+1EE5B         ; Math # Lo       ARABIC MATHEMATICAL TAILED GHAIN
+1EE5D         ; Math # Lo       ARABIC MATHEMATICAL TAILED DOTLESS NOON
+1EE5F         ; Math # Lo       ARABIC MATHEMATICAL TAILED DOTLESS QAF
+1EE61..1EE62  ; Math # Lo   [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM
+1EE64         ; Math # Lo       ARABIC MATHEMATICAL STRETCHED HEH
+1EE67..1EE6A  ; Math # Lo   [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF
+1EE6C..1EE72  ; Math # Lo   [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF
+1EE74..1EE77  ; Math # Lo   [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH
+1EE79..1EE7C  ; Math # Lo   [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH
+1EE7E         ; Math # Lo       ARABIC MATHEMATICAL STRETCHED DOTLESS FEH
+1EE80..1EE89  ; Math # Lo  [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH
+1EE8B..1EE9B  ; Math # Lo  [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN
+1EEA1..1EEA3  ; Math # Lo   [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL
+1EEA5..1EEA9  ; Math # Lo   [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH
+1EEAB..1EEBB  ; Math # Lo  [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN
+1EEF0..1EEF1  ; Math # Sm   [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL
+
+# Total code points: 2310
+
+# ================================================
+
+# Derived Property: Alphabetic
+#  Generated from: Uppercase + Lowercase + Lt + Lm + Lo + Nl + Other_Alphabetic
+
+0041..005A    ; Alphabetic # L&  [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z
+0061..007A    ; Alphabetic # L&  [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z
+00AA          ; Alphabetic # Lo       FEMININE ORDINAL INDICATOR
+00B5          ; Alphabetic # L&       MICRO SIGN
+00BA          ; Alphabetic # Lo       MASCULINE ORDINAL INDICATOR
+00C0..00D6    ; Alphabetic # L&  [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8..00F6    ; Alphabetic # L&  [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS
+00F8..01BA    ; Alphabetic # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL
+01BB          ; Alphabetic # Lo       LATIN LETTER TWO WITH STROKE
+01BC..01BF    ; Alphabetic # L&   [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN
+01C0..01C3    ; Alphabetic # Lo   [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK
+01C4..0293    ; Alphabetic # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL
+0294          ; Alphabetic # Lo       LATIN LETTER GLOTTAL STOP
+0295..02AF    ; Alphabetic # L&  [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL
+02B0..02C1    ; Alphabetic # Lm  [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP
+02C6..02D1    ; Alphabetic # Lm  [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON
+02E0..02E4    ; Alphabetic # Lm   [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP
+02EC          ; Alphabetic # Lm       MODIFIER LETTER VOICING
+02EE          ; Alphabetic # Lm       MODIFIER LETTER DOUBLE APOSTROPHE
+0345          ; Alphabetic # Mn       COMBINING GREEK YPOGEGRAMMENI
+0370..0373    ; Alphabetic # L&   [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI
+0374          ; Alphabetic # Lm       GREEK NUMERAL SIGN
+0376..0377    ; Alphabetic # L&   [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA
+037A          ; Alphabetic # Lm       GREEK YPOGEGRAMMENI
+037B..037D    ; Alphabetic # L&   [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL
+037F          ; Alphabetic # L&       GREEK CAPITAL LETTER YOT
+0386          ; Alphabetic # L&       GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388..038A    ; Alphabetic # L&   [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS
+038C          ; Alphabetic # L&       GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E..03A1    ; Alphabetic # L&  [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO
+03A3..03F5    ; Alphabetic # L&  [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL
+03F7..0481    ; Alphabetic # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA
+048A..052F    ; Alphabetic # L& [166] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EL WITH DESCENDER
+0531..0556    ; Alphabetic # L&  [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH
+0559          ; Alphabetic # Lm       ARMENIAN MODIFIER LETTER LEFT HALF RING
+0561..0587    ; Alphabetic # L&  [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN
+05B0..05BD    ; Alphabetic # Mn  [14] HEBREW POINT SHEVA..HEBREW POINT METEG
+05BF          ; Alphabetic # Mn       HEBREW POINT RAFE
+05C1..05C2    ; Alphabetic # Mn   [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT
+05C4..05C5    ; Alphabetic # Mn   [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT
+05C7          ; Alphabetic # Mn       HEBREW POINT QAMATS QATAN
+05D0..05EA    ; Alphabetic # Lo  [27] HEBREW LETTER ALEF..HEBREW LETTER TAV
+05F0..05F2    ; Alphabetic # Lo   [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD
+0610..061A    ; Alphabetic # Mn  [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA
+0620..063F    ; Alphabetic # Lo  [32] ARABIC LETTER KASHMIRI YEH..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE
+0640          ; Alphabetic # Lm       ARABIC TATWEEL
+0641..064A    ; Alphabetic # Lo  [10] ARABIC LETTER FEH..ARABIC LETTER YEH
+064B..0657    ; Alphabetic # Mn  [13] ARABIC FATHATAN..ARABIC INVERTED DAMMA
+0659..065F    ; Alphabetic # Mn   [7] ARABIC ZWARAKAY..ARABIC WAVY HAMZA BELOW
+066E..066F    ; Alphabetic # Lo   [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF
+0670          ; Alphabetic # Mn       ARABIC LETTER SUPERSCRIPT ALEF
+0671..06D3    ; Alphabetic # Lo  [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE
+06D5          ; Alphabetic # Lo       ARABIC LETTER AE
+06D6..06DC    ; Alphabetic # Mn   [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN
+06E1..06E4    ; Alphabetic # Mn   [4] ARABIC SMALL HIGH DOTLESS HEAD OF KHAH..ARABIC SMALL HIGH MADDA
+06E5..06E6    ; Alphabetic # Lm   [2] ARABIC SMALL WAW..ARABIC SMALL YEH
+06E7..06E8    ; Alphabetic # Mn   [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON
+06ED          ; Alphabetic # Mn       ARABIC SMALL LOW MEEM
+06EE..06EF    ; Alphabetic # Lo   [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V
+06FA..06FC    ; Alphabetic # Lo   [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW
+06FF          ; Alphabetic # Lo       ARABIC LETTER HEH WITH INVERTED V
+0710          ; Alphabetic # Lo       SYRIAC LETTER ALAPH
+0711          ; Alphabetic # Mn       SYRIAC LETTER SUPERSCRIPT ALAPH
+0712..072F    ; Alphabetic # Lo  [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH
+0730..073F    ; Alphabetic # Mn  [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA
+074D..07A5    ; Alphabetic # Lo  [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU
+07A6..07B0    ; Alphabetic # Mn  [11] THAANA ABAFILI..THAANA SUKUN
+07B1          ; Alphabetic # Lo       THAANA LETTER NAA
+07CA..07EA    ; Alphabetic # Lo  [33] NKO LETTER A..NKO LETTER JONA RA
+07F4..07F5    ; Alphabetic # Lm   [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE
+07FA          ; Alphabetic # Lm       NKO LAJANYALAN
+0800..0815    ; Alphabetic # Lo  [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF
+0816..0817    ; Alphabetic # Mn   [2] SAMARITAN MARK IN..SAMARITAN MARK IN-ALAF
+081A          ; Alphabetic # Lm       SAMARITAN MODIFIER LETTER EPENTHETIC YUT
+081B..0823    ; Alphabetic # Mn   [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A
+0824          ; Alphabetic # Lm       SAMARITAN MODIFIER LETTER SHORT A
+0825..0827    ; Alphabetic # Mn   [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U
+0828          ; Alphabetic # Lm       SAMARITAN MODIFIER LETTER I
+0829..082C    ; Alphabetic # Mn   [4] SAMARITAN VOWEL SIGN LONG I..SAMARITAN VOWEL SIGN SUKUN
+0840..0858    ; Alphabetic # Lo  [25] MANDAIC LETTER HALQA..MANDAIC LETTER AIN
+08A0..08B4    ; Alphabetic # Lo  [21] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER KAF WITH DOT BELOW
+08E3..08E9    ; Alphabetic # Mn   [7] ARABIC TURNED DAMMA BELOW..ARABIC CURLY KASRATAN
+08F0..0902    ; Alphabetic # Mn  [19] ARABIC OPEN FATHATAN..DEVANAGARI SIGN ANUSVARA
+0903          ; Alphabetic # Mc       DEVANAGARI SIGN VISARGA
+0904..0939    ; Alphabetic # Lo  [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA
+093A          ; Alphabetic # Mn       DEVANAGARI VOWEL SIGN OE
+093B          ; Alphabetic # Mc       DEVANAGARI VOWEL SIGN OOE
+093D          ; Alphabetic # Lo       DEVANAGARI SIGN AVAGRAHA
+093E..0940    ; Alphabetic # Mc   [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II
+0941..0948    ; Alphabetic # Mn   [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI
+0949..094C    ; Alphabetic # Mc   [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU
+094E..094F    ; Alphabetic # Mc   [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW

From pypy.commits at gmail.com  Mon Jan  9 13:05:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 09 Jan 2017 10:05:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Update to use version 8.0.0 of the
 unicode database
Message-ID: <5873d0dc.ce181c0a.14d74.f9c4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89452:b69ffcf2b98e
Date: 2017-01-09 19:04 +0100
http://bitbucket.org/pypy/pypy/changeset/b69ffcf2b98e/

Log:	Update to use version 8.0.0 of the unicode database

diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -9,7 +9,7 @@
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.runicode import MAXUNICODE
-from rpython.rlib.unicodedata import unicodedb_6_1_0, unicodedb_3_2_0
+from rpython.rlib.unicodedata import unicodedb_8_0_0, unicodedb_3_2_0
 from rpython.rlib.runicode import code_to_unichr, ord_accepts_surrogate
 import sys
 
@@ -334,5 +334,5 @@
                       **methods)
 
 ucd_3_2_0 = UCD(unicodedb_3_2_0)
-ucd_6_1_0 = UCD(unicodedb_6_1_0)
-ucd = ucd_6_1_0
+ucd_8_0_0 = UCD(unicodedb_8_0_0)
+ucd = ucd_8_0_0
diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py
--- a/pypy/module/unicodedata/test/test_unicodedata.py
+++ b/pypy/module/unicodedata/test/test_unicodedata.py
@@ -46,18 +46,18 @@
     def test_cjk(self):
         import sys
         import unicodedata
-        cases = ((0x3400, 0x4DB5),
-                 (0x4E00, 0x9FA5))
-        if unicodedata.unidata_version >= "5":    # don't know the exact limit
-            cases = ((0x3400, 0x4DB5),
-                     (0x4E00, 0x9FCB),
-                     (0x20000, 0x2A6D6),
-                     (0x2A700, 0x2B734))
-        elif unicodedata.unidata_version >= "4.1":
-            cases = ((0x3400, 0x4DB5),
-                     (0x4E00, 0x9FBB),
-                     (0x20000, 0x2A6D6))
+        assert unicodedata.unidata_version >= "8"
+        cases = [
+            ('3400', '4DB5'),
+            ('4E00', '9FD5'),
+            ('20000', '2A6D6'),
+            ('2A700', '2B734'),
+            ('2B740', '2B81D'),
+            ('2B820', '2CEA1'),
+        ]
         for first, last in cases:
+            first = int(first, 16)
+            last = int(last, 16)
             # Test at and inside the boundary
             for i in (first, first + 1, last - 1, last):
                 charname = 'CJK UNIFIED IDEOGRAPH-%X'%i

From pypy.commits at gmail.com  Mon Jan  9 14:07:03 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 09 Jan 2017 11:07:03 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Move unwrapper creation to a
 method of ApiFunction()
Message-ID: <5873df57.2105c30a.98f04.5f0a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89453:8d6b8422c3d2
Date: 2017-01-09 19:06 +0000
http://bitbucket.org/pypy/pypy/changeset/8d6b8422c3d2/

Log:	Move unwrapper creation to a method of ApiFunction()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -329,6 +329,53 @@
             wrapper.c_name = cpyext_namespace.uniquename(self.c_name)
         return wrapper
 
+    def get_unwrapper(self):
+        names = self.argnames
+        argtypesw = zip(self.argtypes,
+                        [_name.startswith("w_") for _name in self.argnames])
+        types_names_enum_ui = unrolling_iterable(enumerate(argtypesw))
+
+        @specialize.ll()
+        def unwrapper(space, *args):
+            from pypy.module.cpyext.pyobject import is_pyobj
+            from pypy.module.cpyext.pyobject import from_ref, as_pyobj
+            newargs = ()
+            keepalives = ()
+            assert len(args) == len(self.argtypes)
+            for i, (ARG, is_wrapped) in types_names_enum_ui:
+                input_arg = args[i]
+                if is_PyObject(ARG) and not is_wrapped:
+                    # build a 'PyObject *' (not holding a reference)
+                    if not is_pyobj(input_arg):
+                        keepalives += (input_arg,)
+                        arg = rffi.cast(ARG, as_pyobj(space, input_arg))
+                    else:
+                        arg = rffi.cast(ARG, input_arg)
+                elif ARG == rffi.VOIDP and not is_wrapped:
+                    # unlike is_PyObject case above, we allow any kind of
+                    # argument -- just, if it's an object, we assume the
+                    # caller meant for it to become a PyObject*.
+                    if input_arg is None or isinstance(input_arg, W_Root):
+                        keepalives += (input_arg,)
+                        arg = rffi.cast(ARG, as_pyobj(space, input_arg))
+                    else:
+                        arg = rffi.cast(ARG, input_arg)
+                elif (is_PyObject(ARG) or ARG == rffi.VOIDP) and is_wrapped:
+                    # build a W_Root, possibly from a 'PyObject *'
+                    if is_pyobj(input_arg):
+                        arg = from_ref(space, input_arg)
+                    else:
+                        arg = input_arg
+                else:
+                    # arg is not declared as PyObject, no magic
+                    arg = input_arg
+                newargs += (arg, )
+            try:
+                return self.callable(space, *newargs)
+            finally:
+                keepalive_until_here(*keepalives)
+        return unwrapper
+
     def get_c_restype(self, c_writer):
         return c_writer.gettype(self.restype).replace('@', '').strip()
 
@@ -452,67 +499,7 @@
                                 c_name=c_name, gil=gil,
                                 result_borrowed=result_borrowed,
                                 result_is_ll=result_is_ll)
-    names = api_function.argnames
-    types_names_enum_ui = unrolling_iterable(enumerate(
-        zip(api_function.argtypes,
-            [tp_name.startswith("w_") for tp_name in names])))
-
-    @specialize.ll()
-    def unwrapper(space, *args):
-        from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj
-        from pypy.module.cpyext.pyobject import from_ref, as_pyobj
-        newargs = ()
-        keepalives = ()
-        assert len(args) == len(api_function.argtypes)
-        for i, (ARG, is_wrapped) in types_names_enum_ui:
-            input_arg = args[i]
-            if is_PyObject(ARG) and not is_wrapped:
-                # build a 'PyObject *' (not holding a reference)
-                if not is_pyobj(input_arg):
-                    keepalives += (input_arg,)
-                    arg = rffi.cast(ARG, as_pyobj(space, input_arg))
-                else:
-                    arg = rffi.cast(ARG, input_arg)
-            elif ARG == rffi.VOIDP and not is_wrapped:
-                # unlike is_PyObject case above, we allow any kind of
-                # argument -- just, if it's an object, we assume the
-                # caller meant for it to become a PyObject*.
-                if input_arg is None or isinstance(input_arg, W_Root):
-                    keepalives += (input_arg,)
-                    arg = rffi.cast(ARG, as_pyobj(space, input_arg))
-                else:
-                    arg = rffi.cast(ARG, input_arg)
-            elif (is_PyObject(ARG) or ARG == rffi.VOIDP) and is_wrapped:
-                # build a W_Root, possibly from a 'PyObject *'
-                if is_pyobj(input_arg):
-                    arg = from_ref(space, input_arg)
-                else:
-                    arg = input_arg
-
-                    ## ZZZ: for is_pyobj:
-                    ## try:
-                    ##     arg = from_ref(space,
-                    ##                rffi.cast(PyObject, input_arg))
-                    ## except TypeError, e:
-                    ##     err = oefmt(space.w_TypeError,
-                    ##                 "could not cast arg to PyObject")
-                    ##     if not catch_exception:
-                    ##         raise err
-                    ##     state = space.fromcache(State)
-                    ##     state.set_exception(err)
-                    ##     if is_PyObject(restype):
-                    ##         return None
-                    ##     else:
-                    ##         return api_function.error_value
-            else:
-                # arg is not declared as PyObject, no magic
-                arg = input_arg
-            newargs += (arg, )
-        try:
-            return func(space, *newargs)
-        finally:
-            keepalive_until_here(*keepalives)
-
+    unwrapper = api_function.get_unwrapper()
     unwrapper.func = func
     unwrapper.api_func = api_function
     return unwrapper

From pypy.commits at gmail.com  Mon Jan  9 15:01:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 09 Jan 2017 12:01:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add two cases,
 one passing and one obscure marked "XXX not implemented".
Message-ID: <5873ec05.a351c20a.48d77.e8ca@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89454:665c5772bb8a
Date: 2017-01-09 21:00 +0100
http://bitbucket.org/pypy/pypy/changeset/665c5772bb8a/

Log:	Add two cases, one passing and one obscure marked "XXX not
	implemented". Comment out the corresponding obscure case inside lib-
	python/3/test/test_super.py.

diff --git a/lib-python/3/test/test_super.py b/lib-python/3/test/test_super.py
--- a/lib-python/3/test/test_super.py
+++ b/lib-python/3/test/test_super.py
@@ -94,11 +94,14 @@
         x = X()
         self.assertEqual(x.f(), 'A')
         self.assertEqual(x.__class__, 413)
-        class X:
-            x = __class__
-            def f():
-                __class__
-        self.assertIs(X.x, type(self))
+        # XXX the following reads the __class__ from a class body, which
+        # XXX gives the one in the *parent* class (here, TestSuper).
+        # XXX with PyPy it fails with a NameError instead for now.
+        #class X:
+        #    x = __class__
+        #    def f():
+        #        __class__
+        #self.assertIs(X.x, type(self))
         with self.assertRaises(NameError) as e:
             exec("""class X:
                 __class__
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -405,7 +405,25 @@
     __class__ = 42
 def testing():
     return Y.__dict__['__class__']
-'''
+''', '''
+class X:
+    foobar = 42
+    def f(self):
+        return __class__.__dict__['foobar']
+def testing():
+    return X().f()
+''',
+#--------XXX the following case is not implemented for now
+#'''
+#class X:
+#    foobar = 42
+#    def f(self):
+#        class Y:
+#            Xcls = __class__
+#        return Y.Xcls.__dict__['foobar']
+#def testing():
+#    return X().f()
+#'''
         ]:
             space.call_args(w_filterwarnings, filter_arg)
             pycode = self.compiler.compile(code, '<tmp>', 'exec', 0)

From pypy.commits at gmail.com  Mon Jan  9 15:21:25 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 09 Jan 2017 12:21:25 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-FromBuffer: failing test,
 flesh out FillInfo
Message-ID: <5873f0c5.0f341c0a.77cea.38ab@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-FromBuffer
Changeset: r89455:24f3f8187bfa
Date: 2017-01-09 22:17 +0200
http://bitbucket.org/pypy/pypy/changeset/24f3f8187bfa/

Log:	failing test, flesh out FillInfo

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -124,6 +124,7 @@
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t,
+    PyVarObject, Py_buffer, size_t, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -486,22 +486,28 @@
     Fills in a buffer-info structure correctly for an exporter that can only
     share a contiguous chunk of memory of "unsigned bytes" of the given
     length. Returns 0 on success and -1 (with raising an error) on error.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
     """
     if flags & PyBUF_WRITABLE and readonly:
         raise oefmt(space.w_ValueError, "Object is not writable")
     view.c_buf = buf
     view.c_len = length
     view.c_obj = obj
-    Py_IncRef(space, obj)
+    if obj:
+        Py_IncRef(space, obj)
     view.c_itemsize = 1
     rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
+    rffi.setintfield(view, 'c_ndim', 1)
     view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
     view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
     view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
 
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -44,6 +44,7 @@
                 ("fillinfo", "METH_VARARGS",
                  """
                  Py_buffer buf;
+                 PyObject * ret = NULL;
                  PyObject *str = PyBytes_FromString("hello, world.");
                  if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
                                        0, 0)) {
@@ -55,7 +56,14 @@
                   */
                  Py_DECREF(str);
 
-                 return PyMemoryView_FromBuffer(&buf);
+                 ret = PyMemoryView_FromBuffer(&buf);
+                 if (((PyMemoryViewObject*)ret)->view.obj != buf.obj)
+                 {
+                    PyErr_SetString(PyExc_ValueError, "leaked ref");
+                    Py_DECREF(ret);
+                    return NULL;
+                 }
+                 return ret;
                  """)])
         result = module.fillinfo()
         assert b"hello, world." == result

From pypy.commits at gmail.com  Mon Jan  9 15:21:28 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 09 Jan 2017 12:21:28 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-FromBuffer: document a plan
Message-ID: <5873f0c8.c4251c0a.a005a.3cea@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-FromBuffer
Changeset: r89456:b568c0a116df
Date: 2017-01-09 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/b568c0a116df/

Log:	document a plan

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -214,6 +214,8 @@
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
+    # XXX this should allocate a PyMemoryViewObject and
+    # copy view into obj.c_view, without creating a new view.c_obj
     assert view.c_obj
     w_obj = from_ref(space, view.c_obj)
     if isinstance(w_obj, W_MemoryView):

From pypy.commits at gmail.com  Mon Jan  9 15:31:49 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 09 Jan 2017 12:31:49 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Returning an ApiFunction
 object from _create_api_func() seems to make more sense
Message-ID: <5873f335.2972c20a.d3693.f50d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89457:1106b7b9f614
Date: 2017-01-09 20:31 +0000
http://bitbucket.org/pypy/pypy/changeset/1106b7b9f614/

Log:	Returning an ApiFunction object from _create_api_func() seems to
	make more sense

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -447,9 +447,12 @@
         else:
             c_name = func.__name__
 
-        unwrapper = _create_api_func(
+        api_function = _create_api_func(
             func, argtypes, restype, error, c_name, gil, result_borrowed,
             result_is_ll)
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
 
         # ZZZ is this whole logic really needed???  It seems to be only
         # for RPython code calling PyXxx() functions directly.  I would
@@ -473,7 +476,7 @@
             return res
 
         if header is not None:
-            FUNCTIONS_BY_HEADER[header][func.__name__] = unwrapper.api_func
+            FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
             INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
         return unwrapper
     return decorate
@@ -495,14 +498,9 @@
         error = rffi.cast(real_restype, error)
 
     func._always_inline_ = 'try'
-    api_function = ApiFunction(argtypes, restype, func, error,
-                                c_name=c_name, gil=gil,
-                                result_borrowed=result_borrowed,
-                                result_is_ll=result_is_ll)
-    unwrapper = api_function.get_unwrapper()
-    unwrapper.func = func
-    unwrapper.api_func = api_function
-    return unwrapper
+    return ApiFunction(
+        argtypes, restype, func, error, c_name=c_name, gil=gil,
+        result_borrowed=result_borrowed, result_is_ll=result_is_ll)
 
 
 def cpython_struct(name, fields, forward=None, level=1):

From pypy.commits at gmail.com  Mon Jan  9 16:41:06 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 09 Jan 2017 13:41:06 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-FromBuffer: implement missing cpyext
 memoryobject pieces
Message-ID: <58740372.8878c20a.1ab7e.952f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-FromBuffer
Changeset: r89458:8653ad15527c
Date: 2017-01-09 23:29 +0200
http://bitbucket.org/pypy/pypy/changeset/8653ad15527c/

Log:	implement missing cpyext memoryobject pieces

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -3,7 +3,7 @@
                          Py_ssize_tP, PyObjectFields, cpython_struct,
                          bootstrap_function, Py_bufferP)
 from pypy.module.cpyext.pyobject import (PyObject, make_ref, as_pyobj, incref,
-             decref, from_ref, make_typedescr)
+             decref, from_ref, make_typedescr, get_typedescr, track_reference)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -28,7 +28,7 @@
                    basestruct=PyMemoryViewObject.TO,
                    attach=memory_attach,
                    dealloc=memory_dealloc,
-                   #realize=memory_realize,
+                   realize=memory_realize,
                   )
 
 def memory_attach(space, py_obj, w_obj, w_userdata=None):
@@ -54,11 +54,31 @@
                                              track_allocation=False))
         rffi.setintfield(view, 'c_readonly', 1)
 
-def memory_realize(space, py_obj):
+def memory_realize(space, obj):
     """
     Creates the memory object in the interpreter
     """
-    raise oefmt(space.w_NotImplementedError, "cannot call this yet")
+    from pypy.module.cpyext.slotdefs import CPyBuffer
+    py_mem = rffi.cast(PyMemoryViewObject, obj)
+    view = py_mem.c_view
+    shape = None
+    if view.c_shape:
+        shape = [view.c_shape[i] for i in range(view.c_ndim)]
+    strides = None
+    if view.c_strides:
+        strides = [view.c_strides[i] for i in range(view.c_ndim)]
+    format = None
+    if view.c_format:
+        format = rffi.charp2str(view.c_format)
+    buf = CPyBuffer(space, view.c_buf, view.c_len, from_ref(space, view.c_obj),
+                    format=format, shape=shape, strides=strides,
+                    ndim=view.c_ndim, itemsize=view.c_itemsize,
+                    readonly=view.c_readonly)
+    w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
+    w_obj = space.allocate_instance(W_MemoryView, w_type)
+    w_obj.__init__(buf)
+    track_reference(space, obj, w_obj)
+    return w_obj
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def memory_dealloc(space, py_obj):
@@ -208,7 +228,7 @@
     py_memview = make_ref(space, w_memview, w_obj)
     return py_memview
 
- at cpython_api([Py_bufferP], PyObject)
+ at cpython_api([Py_bufferP], PyObject, result_is_ll=True)
 def PyMemoryView_FromBuffer(space, view):
     """Create a memoryview object wrapping the given buffer-info structure view.
     The memoryview object then owns the buffer, which means you shouldn't
@@ -216,11 +236,27 @@
     memoryview object."""
     # XXX this should allocate a PyMemoryViewObject and
     # copy view into obj.c_view, without creating a new view.c_obj
-    assert view.c_obj
-    w_obj = from_ref(space, view.c_obj)
-    if isinstance(w_obj, W_MemoryView):
-        return w_obj
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    typedescr = get_typedescr(W_MemoryView.typedef)
+    py_obj = typedescr.allocate(space, space.w_memoryview)
+    py_mem = rffi.cast(PyMemoryViewObject, py_obj)
+    for f in ('c_buf', 'c_obj', 'c_len', 'c_itemsize', 'c_readonly', 'c_ndim', 'c_format'):
+        setattr(py_mem.c_view, f, getattr(view, f))
+    if view.c_strides == rffi.cast(Py_ssize_tP, view.c__strides):
+        py_mem.c_view.c_strides = rffi.cast(Py_ssize_tP, py_mem.c_view.c__strides)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_strides[i] = view.c_strides[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_strides = view.c_strides
+    if view.c_shape == rffi.cast(Py_ssize_tP, view.c__shape):
+        py_mem.c_view.c_shape = rffi.cast(Py_ssize_tP, py_mem.c_view.c__shape)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_shape[i] = view.c_shape[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_shape = view.c_shape
+    # XXX ignore suboffsets?
+    return py_obj
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -30,7 +30,7 @@
         assert view.c_len == 5
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
-        w_mv = api.PyMemoryView_FromBuffer(view)
+        w_mv = from_ref(space, api.PyMemoryView_FromBuffer(view))
         for f in ('format', 'itemsize', 'ndim', 'readonly', 
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)

From pypy.commits at gmail.com  Mon Jan  9 21:10:14 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 09 Jan 2017 18:10:14 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Split off header=None case
 from @cpython_api into new decorator @slot_function
Message-ID: <58744286.06891c0a.fffe2.59ec@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89459:9a171817ba6d
Date: 2017-01-10 02:09 +0000
http://bitbucket.org/pypy/pypy/changeset/9a171817ba6d/

Log:	Split off header=None case from @cpython_api into new decorator
	@slot_function

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -427,29 +427,17 @@
       special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual
       exception into a wrapped SystemError.  Unwrapped exceptions also cause a
       SytemError.
-    - `header` is the header file to export the function in, Set to None to get
-      a C function pointer, but not exported by the API headers.
+    - `header` is the header file to export the function in.
     - set `gil` to "acquire", "release" or "around" to acquire the GIL,
       release the GIL, or both
     """
-    if isinstance(restype, lltype.Typedef):
-        real_restype = restype.OF
-    else:
-        real_restype = restype
-    expect_integer = (isinstance(real_restype, lltype.Primitive) and
-                      rffi.cast(restype, 0) == 0)
+    assert header is not None
     def decorate(func):
-        if header is not None:
-            if func.__name__ in FUNCTIONS_BY_HEADER[header]:
-                raise ValueError("%s already registered" % func.__name__)
-        if header is not None:
-            c_name = None
-        else:
-            c_name = func.__name__
-
+        if func.__name__ in FUNCTIONS_BY_HEADER[header]:
+            raise ValueError("%s already registered" % func.__name__)
         api_function = _create_api_func(
-            func, argtypes, restype, error, c_name, gil, result_borrowed,
-            result_is_ll)
+            func, argtypes, restype, error, gil=gil,
+            result_borrowed=result_borrowed, result_is_ll=result_is_ll)
         unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
@@ -471,6 +459,12 @@
                 else:
                     return unwrapper.api_func.error_value
             got_integer = isinstance(res, (int, long, float))
+            if isinstance(restype, lltype.Typedef):
+                real_restype = restype.OF
+            else:
+                real_restype = restype
+            expect_integer = (isinstance(real_restype, lltype.Primitive) and
+                            rffi.cast(restype, 0) == 0)
             assert got_integer == expect_integer, (
                 'got %r not integer' % (res,))
             return res
@@ -481,6 +475,17 @@
         return unwrapper
     return decorate
 
+def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
+    def decorate(func):
+        c_name = func.__name__
+        api_function = _create_api_func(func, argtypes, restype, error, c_name)
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
+
+
 def _create_api_func(
         func, argtypes, restype, error=_NOT_SPECIFIED, c_name=None,
         gil=None, result_borrowed=False, result_is_ll=False):
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
+    cpython_api, Py_ssize_t, cpython_struct, bootstrap_function, slot_function,
     PyObjectFields, PyObject)
 from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.module.array.interp_array import ArrayBuffer
@@ -72,7 +72,7 @@
                 "Don't know how to realize a buffer")
 
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
     if py_buf.c_b_base:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL)
+    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -25,14 +25,14 @@
 ##
 ## In the PyBytesObject returned, the ob_sval buffer may be modified as
 ## long as the freshly allocated PyBytesObject is not "forced" via a call
-## to any of the more sophisticated C-API functions. 
+## to any of the more sophisticated C-API functions.
 ##
 ## Care has been taken in implementing the functions below, so that
-## if they are called with a non-forced PyBytesObject, they will not 
+## if they are called with a non-forced PyBytesObject, they will not
 ## unintentionally force the creation of a RPython object. As long as only these
 ## are used, the ob_sval buffer is still modifiable:
-## 
-## PyBytes_AsString / PyString_AsString 
+##
+## PyBytes_AsString / PyString_AsString
 ## PyBytes_AS_STRING / PyString_AS_STRING
 ## PyBytes_AsStringAndSize / PyString_AsStringAndSize
 ## PyBytes_Size / PyString_Size
@@ -40,7 +40,7 @@
 ## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string, 
+##   corresponds to the pypy gc-managed string,
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -105,7 +105,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def bytes_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -6,8 +6,8 @@
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
-    bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj, 
+    bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj,
         make_typedescr, track_reference, create_ref, from_ref, decref,
         Py_IncRef)
 from pypy.module.cpyext.object import _dealloc
@@ -36,7 +36,7 @@
     py_dict.c__tmpkeys = lltype.nullptr(PyObject.TO)
     # Problems: if this dict is a typedict, we may have unbound GetSetProperty
     # functions in the dict. The corresponding PyGetSetDescrObject must be
-    # bound to a class, but the actual w_type will be unavailable later on. 
+    # bound to a class, but the actual w_type will be unavailable later on.
     # Solution: use the w_userdata argument when assigning a PyTypeObject's
     # tp_dict slot to pass a w_type in, and force creation of the pair here
     if not space.is_w(w_userdata, space.gettypefor(GetSetProperty)):
@@ -55,7 +55,7 @@
     w_obj = space.newdict()
     track_reference(space, py_obj, w_obj)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def dict_dealloc(space, py_obj):
     py_dict = rffi.cast(PyDictObject, py_obj)
     decref(space, py_dict.c__tmpkeys)
@@ -287,7 +287,7 @@
     if space not in _frozendict_cache:
         _frozendict_cache[space] = _make_frozendict(space)
     return _frozendict_cache[space]
-        
+
 _frozendict_cache = {}
 def _make_frozendict(space):
     return space.appexec([], '''():
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObjectFields, cpython_struct,
-    CANNOT_FAIL)
+    CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, Py_DecRef, make_ref, from_ref, track_reference,
     make_typedescr, get_typedescr)
@@ -39,7 +39,7 @@
     py_frame.c_f_locals = make_ref(space, frame.get_w_locals())
     rffi.setintfield(py_frame, 'c_f_lineno', frame.getorcreatedebug().f_lineno)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def frame_dealloc(space, py_obj):
     py_frame = rffi.cast(PyFrameObject, py_obj)
     py_code = rffi.cast(PyObject, py_frame.c_f_code)
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from rpython.rlib.unroll import unrolling_iterable
@@ -56,7 +57,7 @@
     assert isinstance(w_obj, Function)
     py_func.c_func_name = make_ref(space, space.wrap(w_obj.name))
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def function_dealloc(space, py_obj):
     py_func = rffi.cast(PyFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_func_name)
@@ -75,7 +76,7 @@
     rffi.setintfield(py_code, 'c_co_flags', co_flags)
     rffi.setintfield(py_code, 'c_co_argcount', w_obj.co_argcount)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def code_dealloc(space, py_obj):
     py_code = rffi.cast(PyCodeObject, py_obj)
     Py_DecRef(space, py_code.c_co_name)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,9 +1,9 @@
-from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers,
-                         Py_ssize_tP, PyObjectFields, cpython_struct,
-                         bootstrap_function, Py_bufferP)
-from pypy.module.cpyext.pyobject import (PyObject, make_ref, as_pyobj, incref,
-             decref, from_ref, make_typedescr)
+from pypy.module.cpyext.api import (
+    cpython_api, Py_buffer, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS,
+    build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct,
+    bootstrap_function, Py_bufferP, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -44,7 +44,7 @@
     """
     raise oefmt(space.w_NotImplementedError, "cannot call this yet")
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def memory_dealloc(space, py_obj):
     mem_obj = rffi.cast(PyMemoryViewObject, py_obj)
     if mem_obj.c_view.c_obj:
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,7 +11,7 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr)
+    PyTypeObjectPtr, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -51,7 +51,7 @@
     py_func.c_m_self = make_ref(space, w_obj.w_self)
     py_func.c_m_module = make_ref(space, w_obj.w_module)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def cfunction_dealloc(space, py_obj):
     py_func = rffi.cast(PyCFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_m_self)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -87,7 +87,7 @@
     alloc     : allocate and basic initialization of a raw PyObject
     attach    : Function called to tie a raw structure to a pypy object
     realize   : Function called to create a pypy object from a raw struct
-    dealloc   : a cpython_api(header=None), similar to PyObject_dealloc
+    dealloc   : a @slot_function(), similar to PyObject_dealloc
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
@@ -40,7 +41,7 @@
     rffi.setintfield(py_traceback, 'c_tb_lasti', traceback.lasti)
     rffi.setintfield(py_traceback, 'c_tb_lineno',traceback.get_lineno())
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def traceback_dealloc(space, py_obj):
     py_traceback = rffi.cast(PyTracebackObject, py_obj)
     Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next))
diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py
--- a/pypy/module/cpyext/sliceobject.py
+++ b/pypy/module/cpyext/sliceobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields)
+    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, PyObject, make_ref, make_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
@@ -36,7 +36,7 @@
     py_slice.c_stop = make_ref(space, w_obj.w_stop)
     py_slice.c_step = make_ref(space, w_obj.w_step)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def slice_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -6,7 +6,7 @@
 from rpython.rlib.rarithmetic import widen
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
+    slot_function, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
     pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
@@ -64,7 +64,7 @@
 
 @not_rpython
 def llslot(space, func):
-    return llhelper(func.api_func.functype, func.api_func.get_wrapper(space))
+    return func.api_func.get_llhelper(space)
 
 @register_flow_sc(llslot)
 def sc_llslot(ctx, v_space, v_func):
@@ -521,9 +521,6 @@
 def build_slot_tp_function(space, typedef, name):
     w_type = space.gettypeobject(typedef)
 
-    header = pypy_decl
-    if not (name.startswith('Py') or name.startswith('_Py')):
-        header = None
     handled = False
     # unary functions
     for tp_name, attr in [('tp_as_number.c_nb_int', '__int__'),
@@ -545,7 +542,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject], PyObject, header=header)
+            @slot_function([PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self):
                 return space.call_function(slot_fn, w_self)
@@ -571,7 +568,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg):
                 return space.call_function(slot_fn, w_self, w_arg)
@@ -588,7 +585,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, Py_ssize_t], PyObject, header=header)
+            @slot_function([PyObject, Py_ssize_t], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, arg):
                 return space.call_function(slot_fn, w_self, space.wrap(arg))
@@ -602,7 +599,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
@@ -616,8 +613,8 @@
         if setattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real,
-                     error=-1, header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real,
+                     error=-1)
         @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,))
         def slot_tp_setattro(space, w_self, w_name, w_value):
             if w_value is not None:
@@ -631,7 +628,7 @@
         if getattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject], PyObject)
         @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,))
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
@@ -641,7 +638,7 @@
         if call_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_call(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -654,7 +651,7 @@
         if iternext_fn is None:
             return
 
-        @cpython_api([PyObject], PyObject, header=header)
+        @slot_function([PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_iternext(space, w_self):
             try:
@@ -670,8 +667,7 @@
         if init_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1,
-                     header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_init(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -684,7 +680,7 @@
         if new_fn is None:
             return
 
-        @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
+        @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_new(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -695,8 +691,8 @@
         buff_fn = w_type.getdictvalue(space, '__buffer__')
         if buff_fn is None:
             return
-        @cpython_api([PyObject, Py_bufferP, rffi.INT_real],
-                rffi.INT_real, header=None, error=-1)
+        @slot_function([PyObject, Py_bufferP, rffi.INT_real],
+                rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def buff_w(space, w_self, view, flags):
             args = Arguments(space, [space.newint(flags)])
diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test/test_translate.py
--- a/pypy/module/cpyext/test/test_translate.py
+++ b/pypy/module/cpyext/test/test_translate.py
@@ -1,6 +1,6 @@
 from rpython.translator.c.test.test_genc import compile
 import pypy.module.cpyext.api
-from pypy.module.cpyext.api import cpython_api
+from pypy.module.cpyext.api import slot_function
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.objectmodel import specialize
@@ -19,7 +19,7 @@
 
     @specialize.memo()
     def get_tp_function(space, typedef):
-        @cpython_api([], lltype.Signed, error=-1, header=None)
+        @slot_function([], lltype.Signed, error=-1)
         def slot_tp_function(space):
             return typedef.value
 
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -1,11 +1,11 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import fatalerror_notb
-from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL,
-                                    build_type_checkers, PyVarObjectFields,
-                                    cpython_struct, bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef,
-    make_ref, from_ref, decref, incref, pyobj_has_w_obj,
+from pypy.module.cpyext.api import (
+    cpython_api, Py_ssize_t, build_type_checkers,
+    PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, PyObjectP, make_ref, from_ref, decref, incref,
     track_reference, make_typedescr, get_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
@@ -74,7 +74,7 @@
     if py_tup.c_ob_size < length:
         raise oefmt(space.w_ValueError,
             "tuple_attach called on object with ob_size %d but trying to store %d",
-            py_tup.c_ob_size, length) 
+            py_tup.c_ob_size, length)
     i = 0
     try:
         while i < length:
@@ -113,7 +113,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def tuple_dealloc(space, py_obj):
     """Frees allocated PyTupleObject resources.
     """
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -2,21 +2,20 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rstring import rsplit
 from rpython.rtyper.lltypesystem import rffi, lltype
 
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.typedef import (GetSetProperty, TypeDef,
-        interp_attrproperty, interp_attrproperty, interp2app)
+from pypy.interpreter.typedef import (
+    GetSetProperty, TypeDef, interp_attrproperty, interp2app)
 from pypy.module.__builtin__.abstractinst import abstract_issubclass_w
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
+    PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
     Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
@@ -346,7 +345,7 @@
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
 
- at cpython_api([PyObject, PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
 def tp_new_wrapper(space, self, w_args, w_kwds):
     self_pytype = rffi.cast(PyTypeObjectPtr, self)
     tp_new = self_pytype.c_tp_new
@@ -507,7 +506,7 @@
                    realize=type_realize,
                    dealloc=type_dealloc)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def subtype_dealloc(space, obj):
     pto = obj.c_ob_type
     base = pto
@@ -522,15 +521,13 @@
     # hopefully this does not clash with the memory model assumed in
     # extension modules
 
- at cpython_api([PyObject, Py_ssize_tP], lltype.Signed, header=None,
-             error=CANNOT_FAIL)
+ at slot_function([PyObject, Py_ssize_tP], lltype.Signed, error=CANNOT_FAIL)
 def bf_segcount(space, w_obj, ref):
     if ref:
         ref[0] = space.len_w(w_obj)
     return 1
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def bf_getreadbuffer(space, w_buf, segment, ref):
     from rpython.rlib.buffer import StringBuffer
     if segment != 0:
@@ -543,13 +540,11 @@
     ref[0] = address
     return len(buf)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def bf_getcharbuffer(space, w_buf, segment, ref):
     return bf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def bf_getwritebuffer(space, w_buf, segment, ref):
     if segment != 0:
         raise oefmt(space.w_SystemError,
@@ -558,8 +553,7 @@
     ref[0] = buf.get_raw_address()
     return len(buf)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def str_getreadbuffer(space, w_str, segment, ref):
     from pypy.module.cpyext.bytesobject import PyString_AsString
     if segment != 0:
@@ -571,11 +565,10 @@
     Py_DecRef(space, pyref)
     return space.len_w(w_str)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def unicode_getreadbuffer(space, w_str, segment, ref):
     from pypy.module.cpyext.unicodeobject import (
-                PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
+        PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
     if segment != 0:
         raise oefmt(space.w_SystemError,
                     "accessing non-existent unicode segment")
@@ -585,13 +578,11 @@
     Py_DecRef(space, pyref)
     return PyUnicode_GET_DATA_SIZE(space, w_str)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def str_getcharbuffer(space, w_buf, segment, ref):
     return str_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def buf_getreadbuffer(space, pyref, segment, ref):
     from pypy.module.cpyext.bufferobject import PyBufferObject
     if segment != 0:
@@ -601,8 +592,7 @@
     ref[0] = py_buf.c_b_ptr
     return py_buf.c_b_size
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def buf_getcharbuffer(space, w_buf, segment, ref):
     return buf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
@@ -642,7 +632,7 @@
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def type_dealloc(space, obj):
     from pypy.module.cpyext.object import _dealloc
     obj_pto = rffi.cast(PyTypeObjectPtr, obj)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
     bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
-    CONST_WSTRING)
+    CONST_WSTRING, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -84,7 +84,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def unicode_dealloc(space, py_obj):
     py_unicode = rffi.cast(PyUnicodeObject, py_obj)
     Py_DecRef(space, py_unicode.c_defenc)

From pypy.commits at gmail.com  Tue Jan 10 01:46:04 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 09 Jan 2017 22:46:04 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-FromBuffer: name clarification,
 translation fixes
Message-ID: <5874832c.05371c0a.4a225.dcd0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-FromBuffer
Changeset: r89460:69a5e99dcc8e
Date: 2017-01-10 08:45 +0200
http://bitbucket.org/pypy/pypy/changeset/69a5e99dcc8e/

Log:	name clarification, translation fixes

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -58,7 +58,7 @@
     """
     Creates the memory object in the interpreter
     """
-    from pypy.module.cpyext.slotdefs import CPyBuffer
+    from pypy.module.cpyext.slotdefs import CPyBuffer, fq
     py_mem = rffi.cast(PyMemoryViewObject, obj)
     view = py_mem.c_view
     shape = None
@@ -67,13 +67,16 @@
     strides = None
     if view.c_strides:
         strides = [view.c_strides[i] for i in range(view.c_ndim)]
-    format = None
+    format = 'B'
     if view.c_format:
         format = rffi.charp2str(view.c_format)
     buf = CPyBuffer(space, view.c_buf, view.c_len, from_ref(space, view.c_obj),
                     format=format, shape=shape, strides=strides,
                     ndim=view.c_ndim, itemsize=view.c_itemsize,
                     readonly=view.c_readonly)
+    # Ensure view.c_buf is released upon object finalization
+    fq.register_finalizer(buf)
+    # Allow subclassing W_MemeoryView
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
     w_obj = space.allocate_instance(W_MemoryView, w_type)
     w_obj.__init__(buf)
@@ -239,8 +242,14 @@
     typedescr = get_typedescr(W_MemoryView.typedef)
     py_obj = typedescr.allocate(space, space.w_memoryview)
     py_mem = rffi.cast(PyMemoryViewObject, py_obj)
-    for f in ('c_buf', 'c_obj', 'c_len', 'c_itemsize', 'c_readonly', 'c_ndim', 'c_format'):
-        setattr(py_mem.c_view, f, getattr(view, f))
+    mview = py_mem.c_view
+    mview.c_buf = view.c_buf
+    mview.c_obj = view.c_obj
+    mview.c_len = view.c_len
+    mview.c_itemsize = view.c_itemsize
+    mview.c_readonly = view.c_readonly
+    mview.c_ndim = view.c_ndim
+    mview.c_format = view.c_format
     if view.c_strides == rffi.cast(Py_ssize_tP, view.c__strides):
         py_mem.c_view.c_strides = rffi.cast(Py_ssize_tP, py_mem.c_view.c__strides)
         for i in range(view.c_ndim):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -324,7 +324,7 @@
 
     def __init__(self, space, ptr, size, w_obj, format='B', shape=None,
                 strides=None, ndim=1, itemsize=1, readonly=True,
-                releasebuffer=None):
+                releasebufferproc=rffi.cast(rffi.VOIDP, 0)):
         self.space = space
         self.ptr = ptr
         self.size = size
@@ -342,7 +342,7 @@
         self.ndim = ndim
         self.itemsize = itemsize
         self.readonly = readonly
-        self.releasebufferproc = releasebuffer
+        self.releasebufferproc = releasebufferproc
 
     def releasebuffer(self):
         if self.pyobj:
@@ -360,7 +360,10 @@
                 for i in range(self.ndim):
                     pybuf.c_shape[i] = self.shape[i]
                     pybuf.c_strides[i] = self.strides[i]
-                pybuf.c_format = rffi.str2charp(self.format)
+                if self.format:
+                    pybuf.c_format = rffi.str2charp(self.format)
+                else:
+                    pybuf.c_format = rffi.str2charp("B")
                 generic_cpy_call(self.space, func_target, self.pyobj, pybuf)
             self.releasebufferproc = rffi.cast(rffi.VOIDP, 0)
 
@@ -407,9 +410,9 @@
     func_target = rffi.cast(readbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
@@ -417,7 +420,7 @@
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -426,16 +429,16 @@
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
     decref(space, py_obj)
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -443,9 +446,9 @@
     func_target = rffi.cast(getbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(Py_buffer) as pybuf:
         _flags = 0
@@ -471,7 +474,7 @@
                             ndim=ndim, shape=shape, strides=strides,
                             itemsize=pybuf.c_itemsize,
                             readonly=widen(pybuf.c_readonly),
-                            releasebuffer = releasebuffer)
+                            releasebufferproc = rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 

From pypy.commits at gmail.com  Tue Jan 10 02:43:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 09 Jan 2017 23:43:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <5874909c.8878c20a.8cd6d.8135@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89461:ecb699d457d3
Date: 2017-01-10 08:42 +0100
http://bitbucket.org/pypy/pypy/changeset/ecb699d457d3/

Log:	fix

diff --git a/pypy/module/unicodedata/__init__.py b/pypy/module/unicodedata/__init__.py
--- a/pypy/module/unicodedata/__init__.py
+++ b/pypy/module/unicodedata/__init__.py
@@ -14,7 +14,7 @@
     interpleveldefs = {
         'unidata_version' : 'space.wrap(interp_ucd.ucd.version)',
         'ucd_3_2_0'       : 'space.wrap(interp_ucd.ucd_3_2_0)',
-        'ucd_6_1_0'       : 'space.wrap(interp_ucd.ucd_6_1_0)',
+        'ucd_8_0_0'       : 'space.wrap(interp_ucd.ucd_8_0_0)',
         'ucd'             : 'space.wrap(interp_ucd.ucd)',
         '__doc__'         : "space.wrap('unicode character database')",
     }

From pypy.commits at gmail.com  Tue Jan 10 03:19:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 00:19:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix a test for pypy
Message-ID: <58749926.0d1a1c0a.d4e06.0c06@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89462:85928a9ccfac
Date: 2017-01-10 08:54 +0100
http://bitbucket.org/pypy/pypy/changeset/85928a9ccfac/

Log:	fix a test for pypy

diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -230,6 +230,9 @@
              26 RETURN_VALUE
 """
 
+# XXX PyPy change: the original output has SETUP_EXCEPT, SETUP_FINALLY,
+# POP_EXCEPT, END_FINALLY in that order.  The pops don't match the
+# setups.  In PyPy the POP_EXCEPT is moved after the END_FINALLY.
 dis_traceback = """\
 %3d           0 SETUP_EXCEPT            12 (to 15)
 
@@ -247,18 +250,18 @@
              25 POP_TOP
              26 STORE_FAST               0 (e)
              29 POP_TOP
-             30 SETUP_FINALLY           14 (to 47)
+             30 SETUP_FINALLY           13 (to 46)
 
 %3d          33 LOAD_FAST                0 (e)
              36 LOAD_ATTR                1 (__traceback__)
              39 STORE_FAST               1 (tb)
              42 POP_BLOCK
-             43 POP_EXCEPT
-             44 LOAD_CONST               0 (None)
-        >>   47 LOAD_CONST               0 (None)
-             50 STORE_FAST               0 (e)
-             53 DELETE_FAST              0 (e)
-             56 END_FINALLY
+             43 LOAD_CONST               0 (None)
+        >>   46 LOAD_CONST               0 (None)
+             49 STORE_FAST               0 (e)
+             52 DELETE_FAST              0 (e)
+             55 END_FINALLY
+             56 POP_EXCEPT
              57 JUMP_FORWARD             1 (to 61)
         >>   60 END_FINALLY
 

From pypy.commits at gmail.com  Tue Jan 10 03:19:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 00:19:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Detail: CO_GENERATOR is set in addition
 to CO_COROUTINE in some (nonsensical imho) cases
Message-ID: <58749928.aa1ac20a.d4bc7.8fc5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89463:e0ba73be669b
Date: 2017-01-10 09:19 +0100
http://bitbucket.org/pypy/pypy/changeset/e0ba73be669b/

Log:	Detail: CO_GENERATOR is set in addition to CO_COROUTINE in some
	(nonsensical imho) cases

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -375,7 +375,7 @@
             l += 1
         return l
 
-    def _visit_function(self, func, function_code_generator, extra_flag):
+    def _visit_function(self, func, function_code_generator):
         self.update_position(func.lineno, True)
         # Load decorators first, but apply them after the function is created.
         self.visit_sequence(func.decorator_list)
@@ -392,7 +392,6 @@
         oparg |= num_annotations << 16
         code, qualname = self.sub_scope(function_code_generator, func.name,
                                         func, func.lineno)
-        code.co_flags |= extra_flag
         self._make_function(code, oparg, qualname=qualname)
         # Apply decorators.
         if func.decorator_list:
@@ -401,11 +400,10 @@
         self.name_op(func.name, ast.Store)
 
     def visit_FunctionDef(self, func):
-        self._visit_function(func, FunctionCodeGenerator, 0)
+        self._visit_function(func, FunctionCodeGenerator)
 
     def visit_AsyncFunctionDef(self, func):
-        self._visit_function(func, AsyncFunctionCodeGenerator,
-                             consts.CO_COROUTINE)
+        self._visit_function(func, AsyncFunctionCodeGenerator)
 
     def visit_Lambda(self, lam):
         self.update_position(lam.lineno)
@@ -1569,6 +1567,10 @@
             for i in range(start, len(func.body)):
                 func.body[i].walkabout(self)
 
+    def _get_code_flags(self):
+        flags = AbstractFunctionCodeGenerator._get_code_flags(self)
+        return flags | consts.CO_COROUTINE
+
 class LambdaCodeGenerator(AbstractFunctionCodeGenerator):
 
     def _compile(self, lam):
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -317,7 +317,11 @@
                           yield_node.col_offset)
         
     def note_await(self, await_node):
-        pass
+        # Compatibility with CPython 3.5: set the CO_GENERATOR flag in
+        # addition to the CO_COROUTINE flag if the function uses the
+        # "await" keyword.  Don't do it if the function does not.  In
+        # that case, CO_GENERATOR is ignored anyway.
+        self.is_generator = True
 
 
 class ClassScope(Scope):
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -364,6 +364,13 @@
             scp = self.func_scope(input)
         scp = self.func_scope("def f():\n    return\n    yield x")
 
+    def test_async_def(self):
+        # CPython compatibility only; "is_generator" is otherwise ignored
+        scp = self.func_scope("async def f(): pass")
+        assert not scp.is_generator
+        scp = self.func_scope("async def f(): await 5")
+        assert scp.is_generator
+
     def test_yield_inside_try(self):
         scp = self.func_scope("def f(): yield x")
         assert not scp.has_yield_inside_try

From pypy.commits at gmail.com  Tue Jan 10 03:49:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 00:49:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix remaining tests
Message-ID: <5874a02c.ca06c20a.66d30.9e55@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89464:aa6799139d13
Date: 2017-01-10 09:49 +0100
http://bitbucket.org/pypy/pypy/changeset/aa6799139d13/

Log:	Fix remaining tests

diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -414,13 +414,15 @@
 
 
 
+# CPython 3.5 gives a stack size of 4 whereas it really needs only 3
+# http://bugs.python.org/issue24340
 code_info_code_info = """\
 Name:              code_info
 Filename:          (.*)
 Argument count:    1
 Kw-only arguments: 0
 Number of locals:  1
-Stack size:        4
+Stack size:        (4|3)
 Flags:             OPTIMIZED, NEWLOCALS, NOFREE, 0x100000
 Constants:
    0: %r
@@ -539,13 +541,15 @@
     async for a in b: pass
     async with c as d: pass
 
+# CPython 3.5 gives a stack size of 17 whereas it really needs only 7
+# http://bugs.python.org/issue24340
 code_info_async_def = """\
 Name:              async_def
 Filename:          (.*)
 Argument count:    0
 Kw-only arguments: 0
 Number of locals:  2
-Stack size:        17
+Stack size:        (17|7)
 Flags:             OPTIMIZED, NEWLOCALS, GENERATOR, NOFREE, COROUTINE
 Constants:
    0: None
@@ -714,13 +718,15 @@
   Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=28, starts_line=None, is_jump_target=False),
 ]
 
+# Regenerated for PyPy (difference: some extra dead jumps and jumps-to-jumps
+# in PyPy, not worth optimizing away imho)
 expected_opinfo_jumpy = [
-  Instruction(opname='SETUP_LOOP', opcode=120, arg=68, argval=71, argrepr='to 71', offset=0, starts_line=3, is_jump_target=False),
+  Instruction(opname='SETUP_LOOP', opcode=120, arg=74, argval=77, argrepr='to 77', offset=0, starts_line=3, is_jump_target=False),
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='range', argrepr='range', offset=3, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=6, starts_line=None, is_jump_target=False),
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=9, starts_line=None, is_jump_target=False),
   Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=12, starts_line=None, is_jump_target=False),
-  Instruction(opname='FOR_ITER', opcode=93, arg=44, argval=60, argrepr='to 60', offset=13, starts_line=None, is_jump_target=True),
+  Instruction(opname='FOR_ITER', opcode=93, arg=50, argval=66, argrepr='to 66', offset=13, starts_line=None, is_jump_target=True),
   Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=16, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=19, starts_line=4, is_jump_target=False),
   Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=22, starts_line=None, is_jump_target=False),
@@ -729,89 +735,93 @@
   Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=29, starts_line=5, is_jump_target=False),
   Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=32, starts_line=None, is_jump_target=False),
   Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=35, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=44, argval=44, argrepr='', offset=38, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=47, argval=47, argrepr='', offset=38, starts_line=None, is_jump_target=False),
   Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=41, starts_line=6, is_jump_target=False),
-  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=44, starts_line=7, is_jump_target=True),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=47, starts_line=None, is_jump_target=False),
-  Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=50, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=13, argval=13, argrepr='', offset=53, starts_line=None, is_jump_target=False),
-  Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=56, starts_line=8, is_jump_target=False),
-  Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=57, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=60, starts_line=None, is_jump_target=True),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=61, starts_line=10, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=64, starts_line=None, is_jump_target=False),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=67, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=70, starts_line=None, is_jump_target=False),
-  Instruction(opname='SETUP_LOOP', opcode=120, arg=68, argval=142, argrepr='to 142', offset=71, starts_line=11, is_jump_target=True),
-  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=74, starts_line=None, is_jump_target=True),
-  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=131, argval=131, argrepr='', offset=77, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=80, starts_line=12, is_jump_target=False),
-  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=83, starts_line=None, is_jump_target=False),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=86, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=89, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=90, starts_line=13, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=93, starts_line=None, is_jump_target=False),
-  Instruction(opname='INPLACE_SUBTRACT', opcode=56, arg=None, argval=None, argrepr='', offset=96, starts_line=None, is_jump_target=False),
-  Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=97, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=100, starts_line=14, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=103, starts_line=None, is_jump_target=False),
-  Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=106, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=115, argval=115, argrepr='', offset=109, starts_line=None, is_jump_target=False),
-  Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=74, argval=74, argrepr='', offset=112, starts_line=15, is_jump_target=False),
-  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=115, starts_line=16, is_jump_target=True),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=118, starts_line=None, is_jump_target=False),
-  Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=121, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=74, argval=74, argrepr='', offset=124, starts_line=None, is_jump_target=False),
-  Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=127, starts_line=17, is_jump_target=False),
-  Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=74, argval=74, argrepr='', offset=128, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=131, starts_line=None, is_jump_target=True),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=132, starts_line=19, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=135, starts_line=None, is_jump_target=False),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=138, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=141, starts_line=None, is_jump_target=False),
-  Instruction(opname='SETUP_FINALLY', opcode=122, arg=73, argval=218, argrepr='to 218', offset=142, starts_line=20, is_jump_target=True),
-  Instruction(opname='SETUP_EXCEPT', opcode=121, arg=12, argval=160, argrepr='to 160', offset=145, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=148, starts_line=21, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=151, starts_line=None, is_jump_target=False),
-  Instruction(opname='BINARY_TRUE_DIVIDE', opcode=27, arg=None, argval=None, argrepr='', offset=154, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=155, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=156, starts_line=None, is_jump_target=False),
-  Instruction(opname='JUMP_FORWARD', opcode=110, arg=28, argval=188, argrepr='to 188', offset=157, starts_line=None, is_jump_target=False),
-  Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=160, starts_line=22, is_jump_target=True),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=2, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=161, starts_line=None, is_jump_target=False),
-  Instruction(opname='COMPARE_OP', opcode=107, arg=10, argval='exception match', argrepr='exception match', offset=164, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=187, argval=187, argrepr='', offset=167, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=170, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=171, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=172, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=173, starts_line=23, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=176, starts_line=None, is_jump_target=False),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=179, starts_line=None, is_jump_target=False),
+  Instruction(opname='JUMP_FORWARD', opcode=110, arg=0, argval=47, argrepr='to 47', offset=44, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=47, starts_line=7, is_jump_target=True),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=50, starts_line=None, is_jump_target=False),
+  Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=53, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=63, argval=63, argrepr='', offset=56, starts_line=None, is_jump_target=False),
+  Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=59, starts_line=8, is_jump_target=False),
+  Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=60, starts_line=None, is_jump_target=False),
+  Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=63, starts_line=None, is_jump_target=True),
+  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=66, starts_line=None, is_jump_target=True),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=67, starts_line=10, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=70, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=73, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=76, starts_line=None, is_jump_target=False),
+  Instruction(opname='SETUP_LOOP', opcode=120, arg=74, argval=154, argrepr='to 154', offset=77, starts_line=11, is_jump_target=True),
+  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=80, starts_line=None, is_jump_target=True),
+  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=143, argval=143, argrepr='', offset=83, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=86, starts_line=12, is_jump_target=False),
+  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=89, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=92, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=95, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=96, starts_line=13, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=99, starts_line=None, is_jump_target=False),
+  Instruction(opname='INPLACE_SUBTRACT', opcode=56, arg=None, argval=None, argrepr='', offset=102, starts_line=None, is_jump_target=False),
+  Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=103, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=106, starts_line=14, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=109, starts_line=None, is_jump_target=False),
+  Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=112, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=124, argval=124, argrepr='', offset=115, starts_line=None, is_jump_target=False),
+  Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=80, argval=80, argrepr='', offset=118, starts_line=15, is_jump_target=False),
+  Instruction(opname='JUMP_FORWARD', opcode=110, arg=0, argval=124, argrepr='to 124', offset=121, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=124, starts_line=16, is_jump_target=True),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=127, starts_line=None, is_jump_target=False),
+  Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=130, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=140, argval=140, argrepr='', offset=133, starts_line=None, is_jump_target=False),
+  Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=136, starts_line=17, is_jump_target=False),
+  Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=80, argval=80, argrepr='', offset=137, starts_line=None, is_jump_target=False),
+  Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=80, argval=80, argrepr='', offset=140, starts_line=None, is_jump_target=True),
+  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=143, starts_line=None, is_jump_target=True),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=144, starts_line=19, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=147, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=150, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=153, starts_line=None, is_jump_target=False),
+  Instruction(opname='SETUP_FINALLY', opcode=122, arg=73, argval=230, argrepr='to 230', offset=154, starts_line=20, is_jump_target=True),
+  Instruction(opname='SETUP_EXCEPT', opcode=121, arg=12, argval=172, argrepr='to 172', offset=157, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=160, starts_line=21, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=163, starts_line=None, is_jump_target=False),
+  Instruction(opname='BINARY_TRUE_DIVIDE', opcode=27, arg=None, argval=None, argrepr='', offset=166, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=167, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=168, starts_line=None, is_jump_target=False),
+  Instruction(opname='JUMP_FORWARD', opcode=110, arg=28, argval=200, argrepr='to 200', offset=169, starts_line=None, is_jump_target=False),
+  Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=172, starts_line=22, is_jump_target=True),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=2, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=173, starts_line=None, is_jump_target=False),
+  Instruction(opname='COMPARE_OP', opcode=107, arg=10, argval='exception match', argrepr='exception match', offset=176, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=199, argval=199, argrepr='', offset=179, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=183, starts_line=None, is_jump_target=False),
-  Instruction(opname='JUMP_FORWARD', opcode=110, arg=27, argval=214, argrepr='to 214', offset=184, starts_line=None, is_jump_target=False),
-  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=187, starts_line=None, is_jump_target=True),
-  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=188, starts_line=25, is_jump_target=True),
-  Instruction(opname='SETUP_WITH', opcode=143, arg=17, argval=211, argrepr='to 211', offset=191, starts_line=None, is_jump_target=False),
-  Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=194, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=197, starts_line=26, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Never reach this', argrepr="'Never reach this'", offset=200, starts_line=None, is_jump_target=False),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=203, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=206, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=207, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=208, starts_line=None, is_jump_target=False),
-  Instruction(opname='WITH_CLEANUP_START', opcode=81, arg=None, argval=None, argrepr='', offset=211, starts_line=None, is_jump_target=True),
-  Instruction(opname='WITH_CLEANUP_FINISH', opcode=82, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False),
-  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=213, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=214, starts_line=None, is_jump_target=True),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=215, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=218, starts_line=28, is_jump_target=True),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=221, starts_line=None, is_jump_target=False),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=224, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=227, starts_line=None, is_jump_target=False),
-  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=228, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=229, starts_line=None, is_jump_target=False),
-  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=232, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=183, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=184, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=185, starts_line=23, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=188, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=191, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=194, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=195, starts_line=None, is_jump_target=False),
+  Instruction(opname='JUMP_FORWARD', opcode=110, arg=27, argval=226, argrepr='to 226', offset=196, starts_line=None, is_jump_target=False),
+  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=199, starts_line=None, is_jump_target=True),
+  Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=200, starts_line=25, is_jump_target=True),
+  Instruction(opname='SETUP_WITH', opcode=143, arg=17, argval=223, argrepr='to 223', offset=203, starts_line=None, is_jump_target=False),
+  Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=206, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=209, starts_line=26, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Never reach this', argrepr="'Never reach this'", offset=212, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=215, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=218, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=219, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=220, starts_line=None, is_jump_target=False),
+  Instruction(opname='WITH_CLEANUP_START', opcode=81, arg=None, argval=None, argrepr='', offset=223, starts_line=None, is_jump_target=True),
+  Instruction(opname='WITH_CLEANUP_FINISH', opcode=82, arg=None, argval=None, argrepr='', offset=224, starts_line=None, is_jump_target=False),
+  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=225, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=226, starts_line=None, is_jump_target=True),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=227, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=230, starts_line=28, is_jump_target=True),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=233, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=236, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=239, starts_line=None, is_jump_target=False),
+  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=240, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=241, starts_line=None, is_jump_target=False),
+  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=244, starts_line=None, is_jump_target=False),
 ]
 
 # One last piece of inspect fodder to check the default line number handling

From pypy.commits at gmail.com  Tue Jan 10 04:29:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 01:29:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: More strange cases of bytes.__mod__
Message-ID: <5874a966.0d1a1c0a.d4e06.2814@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89465:a003fe61afd9
Date: 2017-01-10 10:28 +0100
http://bitbucket.org/pypy/pypy/changeset/a003fe61afd9/

Log:	More strange cases of bytes.__mod__

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -583,12 +583,22 @@
         # we check directly for dict to avoid obscure checking
         # in simplest case
         if space.isinstance_w(w_values, space.w_dict) or \
-           (space.lookup(w_values, '__getitem__') and
-           not space.isinstance_w(w_values, space.w_unicode)):
+             _looks_like_a_mapping(space, w_values, fmt_type):
             return format(space, w_format, [w_values], w_values, fmt_type)
         else:
             return format(space, w_format, [w_values], None, fmt_type)
 
+def _looks_like_a_mapping(space, w_x, fmt_type):
+    if not space.lookup(w_x, '__getitem__'):
+        return False
+    if space.isinstance_w(w_x, space.w_unicode):
+        return False
+    if fmt_type != FORMAT_UNICODE:  # (S6) in http://bugs.python.org/issue28885
+        if (space.isinstance_w(w_x, space.w_bytes) or
+            space.isinstance_w(w_x, space.w_bytearray)):
+            return False
+    return True
+
 # ____________________________________________________________
 # Formatting helpers
 
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -604,6 +604,17 @@
     def test_format_bytes(self):
         assert bytearray(b'<%s>') % b'abc' == b'<abc>'
 
+    def test_formatting_not_tuple(self):
+        class mydict(dict):
+            pass
+        xxx = bytearray(b'xxx')
+        assert xxx % mydict() == xxx
+        assert xxx % [] == xxx       # [] considered as a mapping(!)
+        raises(TypeError, "xxx % 'foo'")
+        raises(TypeError, "xxx % b'foo'")
+        raises(TypeError, "xxx % bytearray()")
+        raises(TypeError, "xxx % 53")
+
     def test___alloc__(self):
         # pypy: always returns len()+1; cpython: may be bigger
         assert bytearray(b'123456').__alloc__() >= 7
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -176,6 +176,16 @@
     def test_format_bytes(self):
         assert b'<%s>' % b'abc' == b'<abc>'
 
+    def test_formatting_not_tuple(self):
+        class mydict(dict):
+            pass
+        assert b'xxx' % mydict() == b'xxx'
+        assert b'xxx' % [] == b'xxx'       # [] considered as a mapping(!)
+        raises(TypeError, "b'xxx' % 'foo'")
+        raises(TypeError, "b'xxx' % b'foo'")
+        raises(TypeError, "b'xxx' % bytearray()")
+        raises(TypeError, "b'xxx' % 53")
+
     def test_split(self):
         assert b"".split() == []
         assert b"".split(b'x') == [b'']
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -912,6 +912,16 @@
         # Printable character
         assert '%r' % chr(0xe9) == "'\xe9'"
 
+    def test_formatting_not_tuple(self):
+        class mydict(dict):
+            pass
+        assert 'xxx' % mydict() == 'xxx'
+        assert 'xxx' % b'foo' == 'xxx'   # b'foo' considered as a mapping(!)
+        assert 'xxx' % bytearray() == 'xxx'   # same
+        assert 'xxx' % [] == 'xxx'       # [] considered as a mapping(!)
+        raises(TypeError, "'xxx' % 'foo'")
+        raises(TypeError, "'xxx' % 53")
+
     def test_str_subclass(self):
         class Foo9(str):
             def __str__(self):

From pypy.commits at gmail.com  Tue Jan 10 04:57:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 01:57:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: test and fix
Message-ID: <5874b004.c80e1c0a.5c1a8.e49f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89466:9124eb47aa9d
Date: 2017-01-10 10:56 +0100
http://bitbucket.org/pypy/pypy/changeset/9124eb47aa9d/

Log:	test and fix

diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -18,25 +18,6 @@
     import copyreg
     return copyreg._reduce_ex(obj, proto)
 
-def _getnewargs(obj):
-
-    try:
-        getnewargs = obj.__getnewargs_ex__
-    except AttributeError:
-        try:
-            getnewargs = obj.__getnewargs__
-        except AttributeError:
-            args = ()
-        else:
-            args = getnewargs()
-        kwargs = None
-    else:
-        args, kwargs = getnewargs()
-
-    if not isinstance(args, tuple):
-        raise TypeError("__getnewargs__ should return a tuple")
-    return args, kwargs
-
 def _getstate(obj):
     cls = obj.__class__
 
@@ -60,13 +41,13 @@
         state = getstate()
     return state
 
-def reduce_2(obj, proto):
+def reduce_2(obj, proto, args, kwargs):
     cls = obj.__class__
 
     import copyreg
 
-    args, kwargs = _getnewargs(obj)
-
+    if not isinstance(args, tuple):
+        raise TypeError("__getnewargs__ should return a tuple")
     if not kwargs:
        newobj = copyreg.__newobj__
        args2 = (cls,) + args
@@ -187,7 +168,18 @@
 def descr__reduce__(space, w_obj, proto=0):
     w_proto = space.wrap(proto)
     if proto >= 2:
-        return reduce_2(space, w_obj, w_proto)
+        w_descr = space.lookup(w_obj, '__getnewargs_ex__')
+        if w_descr is not None:
+            w_result = space.get_and_call_function(w_descr, w_obj)
+            w_args, w_kwargs = space.fixedview(w_result, 2)
+        else:
+            w_descr = space.lookup(w_obj, '__getnewargs__')
+            if w_descr is not None:
+                w_args = space.get_and_call_function(w_descr, w_obj)
+            else:
+                w_args = space.newtuple([])
+            w_kwargs = space.w_None
+        return reduce_2(space, w_obj, w_proto, w_args, w_kwargs)
     return reduce_1(space, w_obj, w_proto)
 
 @unwrap_spec(proto=int)
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -75,6 +75,20 @@
             (NamedInt, ('Name',), dict(value=42)),
             dict(_name='Name'), None, None)
 
+    def test_reduce_ex_does_getattr(self):
+        seen = []
+        class X:
+            def __getattribute__(self, name):
+                seen.append(name)
+                return object.__getattribute__(self, name)
+        X().__reduce_ex__(2)
+        # it is the case at least on CPython 3.5.2, like PyPy:
+        assert '__reduce__' in seen
+        # but these methods, which are also called, are not looked up
+        # with getattr:
+        assert '__getnewargs__' not in seen
+        assert '__getnewargs_ex__' not in seen
+
     def test_default_format(self):
         class x(object):
             def __str__(self):

From pypy.commits at gmail.com  Tue Jan 10 05:19:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 02:19:54 -0800 (PST)
Subject: [pypy-commit] pypy default: minimal test for one thing that changed
 in unicodedata 8.0.0
Message-ID: <5874b54a.54161c0a.3aaab.f3fc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89467:5b5373c38aed
Date: 2017-01-10 11:18 +0100
http://bitbucket.org/pypy/pypy/changeset/5b5373c38aed/

Log:	minimal test for one thing that changed in unicodedata 8.0.0

diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py
--- a/rpython/rlib/unicodedata/test/test_unicodedata.py
+++ b/rpython/rlib/unicodedata/test/test_unicodedata.py
@@ -5,7 +5,8 @@
 import py
 
 from rpython.rlib.unicodedata import (
-    unicodedb_3_2_0, unicodedb_5_2_0, unicodedb_6_0_0, unicodedb_6_2_0)
+    unicodedb_3_2_0, unicodedb_5_2_0, unicodedb_6_0_0, unicodedb_6_2_0,
+    unicodedb_8_0_0)
 
 
 class TestUnicodeData(object):
@@ -141,3 +142,9 @@
 
     def test_islower(self):
         assert unicodedb_6_2_0.islower(0x2177)
+
+
+class TestUnicodeData800(object):
+    def test_changed_in_version_8(self):
+        assert unicodedb_6_2_0.toupper_full(0x025C) == [0x025C]
+        assert unicodedb_8_0_0.toupper_full(0x025C) == [0xA7AB]

From pypy.commits at gmail.com  Tue Jan 10 05:19:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 02:19:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Bah,
 the version number of the unicode database we use must be fixed
Message-ID: <5874b54c.8675c20a.e36de.c80c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89468:a3395285a8ad
Date: 2017-01-10 11:19 +0100
http://bitbucket.org/pypy/pypy/changeset/a3395285a8ad/

Log:	Bah, the version number of the unicode database we use must be fixed
	here too. Change the logic so that a single place is enough.

diff --git a/pypy/module/unicodedata/__init__.py b/pypy/module/unicodedata/__init__.py
--- a/pypy/module/unicodedata/__init__.py
+++ b/pypy/module/unicodedata/__init__.py
@@ -3,7 +3,8 @@
 # This is the default unicodedb used in various places:
 # - the unicode type
 # - the regular expression engine
-from rpython.rlib.unicodedata import unicodedb_6_1_0 as unicodedb
+from pypy.module.unicodedata.interp_ucd import ucd as _ucd
+unicodedb = _ucd._unicodedb
 
 # to get information about individual unicode chars look at:
 # http://www.fileformat.info/info/unicode/char/search.htm
diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -77,6 +77,7 @@
 
 class UCD(W_Root):
     def __init__(self, unicodedb):
+        self._unicodedb = unicodedb
         self._lookup = unicodedb.lookup_with_alias
         self._lookup_named_sequence = unicodedb.lookup_named_sequence
         self._name = unicodedb.name
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -234,6 +234,9 @@
         assert ('\u019b\u1d00\u1d86\u0221\u1fb7'.capitalize() ==
                 '\u019b\u1d00\u1d86\u0221\u1fb7')
 
+    def test_changed_in_unicodedata_version_8(self):
+        assert u'\u025C'.upper() == u'\uA7AB'
+
     def test_isprintable(self):
         assert "".isprintable()
         assert " ".isprintable()

From pypy.commits at gmail.com  Tue Jan 10 06:08:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 03:08:36 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Backed out changeset 83e15a9ac426
Message-ID: <5874c0b4.8dae1c0a.5a378.046a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89469:b4a03f3d70b6
Date: 2017-01-10 12:07 +0100
http://bitbucket.org/pypy/pypy/changeset/b4a03f3d70b6/

Log:	Backed out changeset 83e15a9ac426

	It seems that --version was never actually changed to print to
	stderr (it prints to stdout in 3.3, 3.4 and 3.5)

diff --git a/lib-python/3/test/test_ensurepip.py b/lib-python/3/test/test_ensurepip.py
--- a/lib-python/3/test/test_ensurepip.py
+++ b/lib-python/3/test/test_ensurepip.py
@@ -310,7 +310,7 @@
 
     @requires_usable_pip
     def test_bootstrap_version(self):
-        with test.support.captured_stderr() as stdout:
+        with test.support.captured_stdout() as stdout:
             with self.assertRaises(SystemExit):
                 ensurepip._main(["--version"])
         result = stdout.getvalue().strip()
@@ -335,7 +335,7 @@
 class TestUninstallationMainFunction(EnsurepipMixin, unittest.TestCase):
 
     def test_uninstall_version(self):
-        with test.support.captured_stderr() as stdout:
+        with test.support.captured_stdout() as stdout:
             with self.assertRaises(SystemExit):
                 ensurepip._uninstall._main(["--version"])
         result = stdout.getvalue().strip()

From pypy.commits at gmail.com  Tue Jan 10 06:21:02 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 03:21:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Check that the 'opener' of the file
 constructors don't return a negative fd
Message-ID: <5874c39e.810b1c0a.3702f.1a15@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89471:f9c35809153c
Date: 2017-01-10 12:20 +0100
http://bitbucket.org/pypy/pypy/changeset/f9c35809153c/

Log:	Check that the 'opener' of the file constructors don't return a
	negative fd

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -194,6 +194,11 @@
                 w_fd = space.call_function(w_opener, w_name, space.wrap(flags))
                 try:
                     self.fd = space.int_w(w_fd)
+                    if self.fd < 0:
+                        # The opener returned a negative result instead
+                        # of raising an exception
+                        raise oefmt(space.w_ValueError,
+                                    "opener returned %d", self.fd)
                     fd_is_own = True
                 except OperationError as e:
                     if not e.match(space, space.w_TypeError):
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -282,6 +282,12 @@
         if fd1 != fd2:
             raises(OSError, posix.close, fd1)
 
+    def test_opener_negative(self):
+        import _io
+        def opener(*args):
+            return -1
+        raises(ValueError, _io.FileIO, "foo", 'r', opener=opener)
+
 
 def test_flush_at_exit():
     from pypy import conftest

From pypy.commits at gmail.com  Tue Jan 10 06:21:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 03:21:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <5874c39c.86cbc20a.c8d20.dc90@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89470:783976b8ae44
Date: 2017-01-10 12:17 +0100
http://bitbucket.org/pypy/pypy/changeset/783976b8ae44/

Log:	fix test

diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -155,10 +155,17 @@
         f.seek(0)
         m = memoryview(bytearray(b"helloworld"))
         assert f.readinto(m) == 10
+        #
         exc = raises(TypeError, f.readinto, u"hello")
-        assert str(exc.value) == "must be read-write buffer, not str"
+        msg = str(exc.value)
+        print(msg)
+        assert " read-write b" in msg and msg.endswith(", not str")
+        #
         exc = raises(TypeError, f.readinto, memoryview(b"hello"))
-        assert str(exc.value) == "must be read-write buffer, not memoryview"
+        msg = str(exc.value)
+        print(msg)
+        assert " read-write b" in msg and msg.endswith(", not memoryview")
+        #
         f.close()
         assert a == b'a\nb\nc\0\0\0\0\0'
         #

From pypy.commits at gmail.com  Tue Jan 10 06:48:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 03:48:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test if run alone
Message-ID: <5874c9fe.8dae1c0a.5a378.15a0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89472:e10143ffba30
Date: 2017-01-10 12:37 +0100
http://bitbucket.org/pypy/pypy/changeset/e10143ffba30/

Log:	fix test if run alone

diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -151,6 +151,9 @@
         import _io
         a = bytearray(b'x' * 10)
         f = _io.FileIO(self.tmpfile, 'r+')
+        f.seek(5)
+        f.write(b'\x00' * 5)
+        f.seek(0)
         assert f.readinto(a) == 10
         f.seek(0)
         m = memoryview(bytearray(b"helloworld"))

From pypy.commits at gmail.com  Tue Jan 10 06:48:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 03:48:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: readinto1(),
 and fix expected error messages for readinto() test
Message-ID: <5874c9ff.4f831c0a.8bab3.21b8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89473:26d691d3df32
Date: 2017-01-10 12:42 +0100
http://bitbucket.org/pypy/pypy/changeset/26d691d3df32/

Log:	readinto1(), and fix expected error messages for readinto() test

diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -107,18 +107,25 @@
         self._unsupportedoperation(space, "detach")
 
     def readinto_w(self, space, w_buffer):
+        return self._readinto(space, w_buffer, "read")
+
+    def readinto1_w(self, space, w_buffer):
+        return self._readinto(space, w_buffer, "read1")
+
+    def _readinto(self, space, w_buffer, methodname):
         rwbuffer = space.getarg_w('w*', w_buffer)
         length = rwbuffer.getlength()
-        w_data = space.call_method(self, "read", space.wrap(length))
+        w_data = space.call_method(self, methodname, space.wrap(length))
 
         if not space.isinstance_w(w_data, space.w_str):
-            raise oefmt(space.w_TypeError, "read() should return bytes")
+            raise oefmt(space.w_TypeError, "%s() should return bytes",
+                        methodname)
         data = space.bytes_w(w_data)
         if len(data) > length:
             raise oefmt(space.w_ValueError,
-                        "read() returned too much data: "
+                        "%s() returned too much data: "
                         "%d bytes requested, %d returned",
-                        length, len(data))
+                        methodname, length, len(data))
         rwbuffer.setslice(0, data)
         return space.wrap(len(data))
 
@@ -144,6 +151,7 @@
     write = interp2app(W_BufferedIOBase.write_w),
     detach = interp2app(W_BufferedIOBase.detach_w),
     readinto = interp2app(W_BufferedIOBase.readinto_w),
+    readinto1 = interp2app(W_BufferedIOBase.readinto1_w),
 )
 
 class RawBuffer(Buffer):
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -83,9 +83,6 @@
         limit = convert_size(space, w_limit)
         return space.newbytes(self.readline(limit))
 
-    def read1_w(self, space, w_size):
-        return self.read_w(space, w_size)
-
     def readinto_w(self, space, w_buffer):
         self._check_closed(space)
         rwbuffer = space.getarg_w('w*', w_buffer)
@@ -203,9 +200,10 @@
     __init__  = interp2app(W_BytesIO.descr_init),
 
     read = interp2app(W_BytesIO.read_w),
-    read1 = interp2app(W_BytesIO.read1_w),
+    read1 = interp2app(W_BytesIO.read_w),
     readline = interp2app(W_BytesIO.readline_w),
     readinto = interp2app(W_BytesIO.readinto_w),
+    readinto1 = interp2app(W_BytesIO.readinto_w),
     write = interp2app(W_BytesIO.write_w),
     truncate = interp2app(W_BytesIO.truncate_w),
     getbuffer = interp2app(W_BytesIO.getbuffer_w),
diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py
--- a/pypy/module/_io/test/test_bufferedio.py
+++ b/pypy/module/_io/test/test_bufferedio.py
@@ -152,19 +152,28 @@
 
     def test_readinto(self):
         import _io
-        a = bytearray(b'x' * 10)
-        raw = _io.FileIO(self.tmpfile)
-        f = _io.BufferedReader(raw)
-        assert f.readinto(a) == 5
-        f.seek(0)
-        m = memoryview(bytearray(b"hello"))
-        assert f.readinto(m) == 5
-        exc = raises(TypeError, f.readinto, u"hello")
-        assert str(exc.value) == "must be read-write buffer, not str"
-        exc = raises(TypeError, f.readinto, memoryview(b"hello"))
-        assert str(exc.value) == "must be read-write buffer, not memoryview"
-        f.close()
-        assert a == b'a\nb\ncxxxxx'
+        for methodname in ["readinto", "readinto1"]:
+            a = bytearray(b'x' * 10)
+            raw = _io.FileIO(self.tmpfile)
+            f = _io.BufferedReader(raw)
+            readinto = getattr(f, methodname)
+            assert readinto(a) == 5
+            f.seek(0)
+            m = memoryview(bytearray(b"hello"))
+            assert readinto(m) == 5
+            #
+            exc = raises(TypeError, readinto, u"hello")
+            msg = str(exc.value)
+            print(msg)
+            assert " read-write b" in msg and msg.endswith(", not str")
+            #
+            exc = raises(TypeError, readinto, memoryview(b"hello"))
+            msg = str(exc.value)
+            print(msg)
+            assert " read-write b" in msg and msg.endswith(", not memoryview")
+            #
+            f.close()
+            assert a == b'a\nb\ncxxxxx'
 
     def test_readinto_buffer_overflow(self):
         import _io
diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py
--- a/pypy/module/_io/test/test_bytesio.py
+++ b/pypy/module/_io/test/test_bytesio.py
@@ -98,23 +98,31 @@
 
     def test_readinto(self):
         import _io
-
-        b = _io.BytesIO(b"hello")
-        a1 = bytearray(b't')
-        a2 = bytearray(b'testing')
-        assert b.readinto(a1) == 1
-        assert b.readinto(a2) == 4
-        b.seek(0)
-        m = memoryview(bytearray(b"world"))
-        assert b.readinto(m) == 5
-        exc = raises(TypeError, b.readinto, u"hello")
-        assert str(exc.value) == "must be read-write buffer, not str"
-        exc = raises(TypeError, b.readinto, memoryview(b"hello"))
-        assert str(exc.value) == "must be read-write buffer, not memoryview"
-        b.close()
-        assert a1 == b"h"
-        assert a2 == b"elloing"
-        raises(ValueError, b.readinto, bytearray(b"hello"))
+        for methodname in ["readinto", "readinto1"]:
+            b = _io.BytesIO(b"hello")
+            readinto = getattr(b, methodname)
+            a1 = bytearray(b't')
+            a2 = bytearray(b'testing')
+            assert readinto(a1) == 1
+            assert readinto(a2) == 4
+            b.seek(0)
+            m = memoryview(bytearray(b"world"))
+            assert readinto(m) == 5
+            #
+            exc = raises(TypeError, readinto, u"hello")
+            msg = str(exc.value)
+            print(msg)
+            assert " read-write b" in msg and msg.endswith(", not str")
+            #
+            exc = raises(TypeError, readinto, memoryview(b"hello"))
+            msg = str(exc.value)
+            print(msg)
+            assert " read-write b" in msg and msg.endswith(", not memoryview")
+            #
+            b.close()
+            assert a1 == b"h"
+            assert a2 == b"elloing"
+            raises(ValueError, readinto, bytearray(b"hello"))
 
     def test_getbuffer(self):
         import _io

From pypy.commits at gmail.com  Tue Jan 10 11:35:08 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 10 Jan 2017 08:35:08 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Make PyObject_dealloc a
 slot_function
Message-ID: <58750d3c.6a7ac20a.f5ed4.5d06@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89475:4513b1dccd7a
Date: 2017-01-10 14:32 +0000
http://bitbucket.org/pypy/pypy/changeset/4513b1dccd7a/

Log:	Make PyObject_dealloc a slot_function

diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t,
+    PyVarObject, Py_buffer, size_t, slot_function,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -54,7 +54,7 @@
         w_obj = PyObject_InitVar(space, py_objvar, type, itemcount)
     return py_obj
 
- at cpython_api([PyObject], lltype.Void)
+ at slot_function([PyObject], lltype.Void)
 def PyObject_dealloc(space, obj):
     return _dealloc(space, obj)
 
@@ -511,7 +511,7 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Release the buffer view. This should be called when the buffer is 
+    Release the buffer view. This should be called when the buffer is
     no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)

From pypy.commits at gmail.com  Tue Jan 10 11:35:06 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 10 Jan 2017 08:35:06 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: hg merge default
Message-ID: <58750d3a.54161c0a.3aaab.9454@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89474:920775a12e2c
Date: 2017-01-10 13:30 +0000
http://bitbucket.org/pypy/pypy/changeset/920775a12e2c/

Log:	hg merge default

diff too long, truncating to 2000 out of 70624 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -91,3 +91,8 @@
 .. branch: cpyext-cleanup
 
 Refactor cpyext initialisation.
+
+.. branch: cpyext-from2
+
+Fix a test failure introduced by strbuf-as-buffer
+
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -35,8 +35,24 @@
     """
     Fills a newly allocated PyMemoryViewObject with the given W_MemoryView object.
     """
+    assert isinstance(w_obj, W_MemoryView)
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
-    py_obj.c_view.c_obj = rffi.cast(PyObject, 0)
+    view = py_obj.c_view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return
+    fill_Py_buffer(space, w_obj.buf, view)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_userdata)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s),
+                                             track_allocation=False))
+        rffi.setintfield(view, 'c_readonly', 1)
 
 def memory_realize(space, py_obj):
     """
@@ -88,29 +104,13 @@
     return ret
 
 @cpython_api([PyObject], Py_bufferP, error=CANNOT_FAIL)
-def PyMemoryView_GET_BUFFER(space, w_obj):
+def PyMemoryView_GET_BUFFER(space, pyobj):
     """Return a pointer to the buffer-info structure wrapped by the given
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
-    if not isinstance(w_obj, W_MemoryView):
-        return lltype.nullptr(Py_buffer)
-    py_memobj = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_obj)) # no inc_ref
-    view = py_memobj.c_view
-    ndim = w_obj.buf.getndim()
-    if ndim >= Py_MAX_NDIMS:
-        # XXX warn?
-        return view
-    fill_Py_buffer(space, w_obj.buf, view)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-        #view.c_obj = make_ref(space, w_obj) # NO - this creates a ref cycle!
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
-    except ValueError:
-        w_s = w_obj.descr_tobytes(space)
-        view.c_obj = make_ref(space, w_s)
-        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
-        rffi.setintfield(view, 'c_readonly', 1)
-    return view
+    # XXX move to a c-macro
+    py_memobj = rffi.cast(PyMemoryViewObject, pyobj)
+    return py_memobj.c_view
 
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
@@ -202,9 +202,11 @@
         return (_IsCContiguous(view) or _IsFortranContiguous(view))
     return 0
 
- at cpython_api([PyObject], PyObject)
+ at cpython_api([PyObject], PyObject, result_is_ll=True)
 def PyMemoryView_FromObject(space, w_obj):
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    w_memview = space.call_method(space.builtin, "memoryview", w_obj)
+    py_memview = make_ref(space, w_memview, w_obj)
+    return py_memview
 
 @cpython_api([Py_bufferP], PyObject)
 def PyMemoryView_FromBuffer(space, view):
@@ -212,6 +214,7 @@
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
+    assert view.c_obj
     w_obj = from_ref(space, view.c_obj)
     if isinstance(w_obj, W_MemoryView):
         return w_obj
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -4,6 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
+from pypy.module.cpyext.pyobject import from_ref
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
@@ -11,7 +12,7 @@
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
-        w_view = api.PyMemoryView_FromObject(w_hello)
+        w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
         assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
@@ -19,7 +20,7 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_memoryview = from_ref(space, api.PyMemoryView_FromObject(w_buf))
         view = api.PyMemoryView_GET_BUFFER(w_memoryview)
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -56,7 +56,7 @@
         assert u"abc" != memoryview("abc")
 
     def test_pypy_raw_address_base(self):
-        a = memoryview("foobar")._pypy_raw_address()
+        a = memoryview(b"foobar")._pypy_raw_address()
         assert a != 0
-        b = memoryview(bytearray("foobar"))._pypy_raw_address()
+        b = memoryview(bytearray(b"foobar"))._pypy_raw_address()
         assert b != 0
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -210,7 +210,6 @@
     return rlist.ll_pop(rlist.dum_checkidx, l, index)
 _ll_2_list_append = rlist.ll_append
 _ll_2_list_extend = rlist.ll_extend
-_ll_3_list_insert = rlist.ll_insert_nonneg
 _ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly
 _ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop
 _ll_2_list_inplace_mul = rlist.ll_inplace_mul
diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -212,6 +212,8 @@
                 s += lst[0]
                 lst.pop()
                 lst.append(1)
+                lst.insert(0, 5)
+                lst.insert(1, 6)
                 s *= lst.pop()
             return s
         res = self.meta_interp(f, [15], listops=True)
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -51,6 +51,7 @@
     voidpf = rffi_platform.SimpleType('voidpf', rffi.VOIDP)
 
     ZLIB_VERSION = rffi_platform.DefinedConstantString('ZLIB_VERSION')
+    ZLIB_VERNUM = rffi_platform.DefinedConstantInteger('ZLIB_VERNUM')
 
 for _name in constantnames:
     setattr(SimpleCConfig, _name, rffi_platform.ConstantInteger(_name))
@@ -63,6 +64,7 @@
 Bytefp = lltype.Ptr(lltype.Array(Bytef, hints={'nolength': True}))
 
 ZLIB_VERSION = config['ZLIB_VERSION']
+ZLIB_VERNUM = config['ZLIB_VERNUM']
 
 for _name in constantnames:
     globals()[_name] = config[_name]
@@ -261,7 +263,7 @@
 
 def deflateInit(level=Z_DEFAULT_COMPRESSION, method=Z_DEFLATED,
                 wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL,
-                strategy=Z_DEFAULT_STRATEGY):
+                strategy=Z_DEFAULT_STRATEGY, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     compress data.
@@ -270,6 +272,12 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
     if err == Z_OK:
+        if zdict is not None:
+            try:
+                deflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:
@@ -290,7 +298,7 @@
     lltype.free(stream, flavor='raw')
 
 
-def inflateInit(wbits=MAX_WBITS):
+def inflateInit(wbits=MAX_WBITS, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     decompress data.
@@ -299,6 +307,17 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _inflateInit2(stream, wbits)
     if err == Z_OK:
+        if zdict is not None and wbits < 0:
+            try:
+                if ZLIB_VERNUM is None or ZLIB_VERNUM < 0x1221:
+                    raise RZlibError("zlib version %s does not allow raw "
+                                     "inflate with dictionary" %
+                                       ZLIB_VERSION if ZLIB_VERSION is not None
+                                       else "<unknown>")
+                inflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:
diff --git a/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
@@ -0,0 +1,1414 @@
+# CaseFolding-8.0.0.txt
+# Date: 2015-01-13, 18:16:36 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#223;e" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding 
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+# Property: Case_Folding
+
+#  All code points not explicitly listed for Case_Folding
+#  have the value C for the status field, and the code point itself for the mapping field.
+
+# =================================================================
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE
+023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE
+023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR
+023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
+0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP
+0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE
+0244; C; 0289; # LATIN CAPITAL LETTER U BAR
+0245; C; 028C; # LATIN CAPITAL LETTER TURNED V
+0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE
+0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE
+024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
+024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE
+024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0370; C; 0371; # GREEK CAPITAL LETTER HETA
+0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI
+0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
+037F; C; 03F3; # GREEK CAPITAL LETTER YOT
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+03F7; C; 03F8; # GREEK CAPITAL LETTER SHO
+03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL
+03FA; C; 03FB; # GREEK CAPITAL LETTER SAN
+03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
+03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
+03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
+04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK
+04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE
+0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK
+0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA
+0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA
+0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE
+051A; C; 051B; # CYRILLIC CAPITAL LETTER QA
+051C; C; 051D; # CYRILLIC CAPITAL LETTER WE
+051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA
+0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
+0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
+0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER
+0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER
+0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK
+052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE
+052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE
+052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN
+10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN
+10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN
+10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON
+10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN
+10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN
+10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN
+10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN
+10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN
+10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN
+10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS
+10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN
+10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR
+10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON
+10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR
+10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR
+10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE
+10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN
+10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR
+10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN
+10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR
+10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR
+10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN
+10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR
+10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN
+10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN
+10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN
+10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL
+10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL
+10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR
+10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN
+10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN
+10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE
+10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE
+10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE
+10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE
+10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR
+10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE
+10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN
+10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN
+13F8; C; 13F0; # CHEROKEE SMALL LETTER YE
+13F9; C; 13F1; # CHEROKEE SMALL LETTER YI
+13FA; C; 13F2; # CHEROKEE SMALL LETTER YO
+13FB; C; 13F3; # CHEROKEE SMALL LETTER YU
+13FC; C; 13F4; # CHEROKEE SMALL LETTER YV
+13FD; C; 13F5; # CHEROKEE SMALL LETTER MV
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S
+1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL
+1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V
+1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2132; C; 214E; # TURNED CAPITAL F
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N
+24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O
+24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P
+24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q
+24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R
+24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S
+24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T
+24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U
+24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V
+24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W
+24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X
+24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y
+24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z
+2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU
+2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY
+2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE
+2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI
+2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO
+2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU
+2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE
+2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO
+2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA
+2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE
+2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE
+2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I
+2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI
+2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO
+2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE
+2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE
+2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI
+2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU
+2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI
+2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI
+2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO
+2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO
+2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU
+2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU
+2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU
+2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU
+2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE
+2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA
+2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI
+2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI
+2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA
+2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU
+2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI
+2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI
+2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA
+2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU
+2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS
+2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL
+2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO
+2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS
+2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS
+2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS
+2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA
+2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA
+2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC
+2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A
+2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE
+2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR
+2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE
+2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE
+2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL
+2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER
+2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER
+2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER
+2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA
+2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK
+2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A
+2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA
+2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK
+2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H
+2C7E; C; 023F; # LATIN CAPITAL LETTER S WITH SWASH TAIL
+2C7F; C; 0240; # LATIN CAPITAL LETTER Z WITH SWASH TAIL
+2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA
+2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA
+2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA
+2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA
+2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE
+2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU
+2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA
+2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE
+2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE
+2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA
+2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA
+2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA
+2C98; C; 2C99; # COPTIC CAPITAL LETTER MI
+2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI
+2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI
+2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O
+2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI
+2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO
+2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA
+2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU
+2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA
+2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI
+2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI
+2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI
+2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU
+2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF
+2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN
+2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE
+2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA
+2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI
+2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI
+2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU
+2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI
+2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI
+2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI
+2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH
+2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI
+2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI
+2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI
+2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA
+2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA
+2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI
+2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT
+2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA
+2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA
+2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA
+2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA
+2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI
+2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI
+2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU
+2CEB; C; 2CEC; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI
+2CED; C; 2CEE; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA
+2CF2; C; 2CF3; # COPTIC CAPITAL LETTER BOHAIRIC KHEI
+A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA
+A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO
+A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE
+A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA
+A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV
+A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK
+A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA
+A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER
+A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER
+A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT
+A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU
+A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A
+A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS
+A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS
+A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS
+A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN
+A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE
+A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE
+A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL
+A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM
+A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O
+A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O
+A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O
+A680; C; A681; # CYRILLIC CAPITAL LETTER DWE
+A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE
+A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE
+A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE
+A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE
+A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK
+A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE
+A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE
+A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE
+A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE
+A694; C; A695; # CYRILLIC CAPITAL LETTER HWE
+A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE
+A698; C; A699; # CYRILLIC CAPITAL LETTER DOUBLE O
+A69A; C; A69B; # CYRILLIC CAPITAL LETTER CROSSED O
+A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF
+A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN
+A726; C; A727; # LATIN CAPITAL LETTER HENG
+A728; C; A729; # LATIN CAPITAL LETTER TZ
+A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO
+A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO
+A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA
+A732; C; A733; # LATIN CAPITAL LETTER AA
+A734; C; A735; # LATIN CAPITAL LETTER AO
+A736; C; A737; # LATIN CAPITAL LETTER AU
+A738; C; A739; # LATIN CAPITAL LETTER AV
+A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR
+A73C; C; A73D; # LATIN CAPITAL LETTER AY
+A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT
+A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE
+A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE
+A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE
+A746; C; A747; # LATIN CAPITAL LETTER BROKEN L
+A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE
+A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY
+A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP
+A74E; C; A74F; # LATIN CAPITAL LETTER OO
+A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER
+A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH
+A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL
+A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER
+A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE
+A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA
+A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA
+A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE
+A760; C; A761; # LATIN CAPITAL LETTER VY
+A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z
+A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE
+A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER
+A768; C; A769; # LATIN CAPITAL LETTER VEND
+A76A; C; A76B; # LATIN CAPITAL LETTER ET
+A76C; C; A76D; # LATIN CAPITAL LETTER IS
+A76E; C; A76F; # LATIN CAPITAL LETTER CON
+A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D
+A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F
+A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G
+A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G
+A780; C; A781; # LATIN CAPITAL LETTER TURNED L
+A782; C; A783; # LATIN CAPITAL LETTER INSULAR R
+A784; C; A785; # LATIN CAPITAL LETTER INSULAR S
+A786; C; A787; # LATIN CAPITAL LETTER INSULAR T
+A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO
+A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H
+A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER
+A792; C; A793; # LATIN CAPITAL LETTER C WITH BAR
+A796; C; A797; # LATIN CAPITAL LETTER B WITH FLOURISH
+A798; C; A799; # LATIN CAPITAL LETTER F WITH STROKE
+A79A; C; A79B; # LATIN CAPITAL LETTER VOLAPUK AE
+A79C; C; A79D; # LATIN CAPITAL LETTER VOLAPUK OE
+A79E; C; A79F; # LATIN CAPITAL LETTER VOLAPUK UE
+A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE
+A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE
+A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE
+A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE
+A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE
+A7AA; C; 0266; # LATIN CAPITAL LETTER H WITH HOOK
+A7AB; C; 025C; # LATIN CAPITAL LETTER REVERSED OPEN E
+A7AC; C; 0261; # LATIN CAPITAL LETTER SCRIPT G
+A7AD; C; 026C; # LATIN CAPITAL LETTER L WITH BELT
+A7B0; C; 029E; # LATIN CAPITAL LETTER TURNED K
+A7B1; C; 0287; # LATIN CAPITAL LETTER TURNED T
+A7B2; C; 029D; # LATIN CAPITAL LETTER J WITH CROSSED-TAIL
+A7B3; C; AB53; # LATIN CAPITAL LETTER CHI
+A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA
+A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA
+AB70; C; 13A0; # CHEROKEE SMALL LETTER A
+AB71; C; 13A1; # CHEROKEE SMALL LETTER E
+AB72; C; 13A2; # CHEROKEE SMALL LETTER I
+AB73; C; 13A3; # CHEROKEE SMALL LETTER O
+AB74; C; 13A4; # CHEROKEE SMALL LETTER U
+AB75; C; 13A5; # CHEROKEE SMALL LETTER V
+AB76; C; 13A6; # CHEROKEE SMALL LETTER GA
+AB77; C; 13A7; # CHEROKEE SMALL LETTER KA
+AB78; C; 13A8; # CHEROKEE SMALL LETTER GE
+AB79; C; 13A9; # CHEROKEE SMALL LETTER GI
+AB7A; C; 13AA; # CHEROKEE SMALL LETTER GO
+AB7B; C; 13AB; # CHEROKEE SMALL LETTER GU
+AB7C; C; 13AC; # CHEROKEE SMALL LETTER GV
+AB7D; C; 13AD; # CHEROKEE SMALL LETTER HA
+AB7E; C; 13AE; # CHEROKEE SMALL LETTER HE
+AB7F; C; 13AF; # CHEROKEE SMALL LETTER HI
+AB80; C; 13B0; # CHEROKEE SMALL LETTER HO
+AB81; C; 13B1; # CHEROKEE SMALL LETTER HU
+AB82; C; 13B2; # CHEROKEE SMALL LETTER HV
+AB83; C; 13B3; # CHEROKEE SMALL LETTER LA
+AB84; C; 13B4; # CHEROKEE SMALL LETTER LE
+AB85; C; 13B5; # CHEROKEE SMALL LETTER LI
+AB86; C; 13B6; # CHEROKEE SMALL LETTER LO
+AB87; C; 13B7; # CHEROKEE SMALL LETTER LU
+AB88; C; 13B8; # CHEROKEE SMALL LETTER LV
+AB89; C; 13B9; # CHEROKEE SMALL LETTER MA
+AB8A; C; 13BA; # CHEROKEE SMALL LETTER ME
+AB8B; C; 13BB; # CHEROKEE SMALL LETTER MI
+AB8C; C; 13BC; # CHEROKEE SMALL LETTER MO
+AB8D; C; 13BD; # CHEROKEE SMALL LETTER MU
+AB8E; C; 13BE; # CHEROKEE SMALL LETTER NA
+AB8F; C; 13BF; # CHEROKEE SMALL LETTER HNA
+AB90; C; 13C0; # CHEROKEE SMALL LETTER NAH
+AB91; C; 13C1; # CHEROKEE SMALL LETTER NE
+AB92; C; 13C2; # CHEROKEE SMALL LETTER NI
+AB93; C; 13C3; # CHEROKEE SMALL LETTER NO
+AB94; C; 13C4; # CHEROKEE SMALL LETTER NU
+AB95; C; 13C5; # CHEROKEE SMALL LETTER NV
+AB96; C; 13C6; # CHEROKEE SMALL LETTER QUA
+AB97; C; 13C7; # CHEROKEE SMALL LETTER QUE
+AB98; C; 13C8; # CHEROKEE SMALL LETTER QUI
+AB99; C; 13C9; # CHEROKEE SMALL LETTER QUO
+AB9A; C; 13CA; # CHEROKEE SMALL LETTER QUU
+AB9B; C; 13CB; # CHEROKEE SMALL LETTER QUV
+AB9C; C; 13CC; # CHEROKEE SMALL LETTER SA
+AB9D; C; 13CD; # CHEROKEE SMALL LETTER S
+AB9E; C; 13CE; # CHEROKEE SMALL LETTER SE
+AB9F; C; 13CF; # CHEROKEE SMALL LETTER SI
+ABA0; C; 13D0; # CHEROKEE SMALL LETTER SO
+ABA1; C; 13D1; # CHEROKEE SMALL LETTER SU
+ABA2; C; 13D2; # CHEROKEE SMALL LETTER SV
+ABA3; C; 13D3; # CHEROKEE SMALL LETTER DA
+ABA4; C; 13D4; # CHEROKEE SMALL LETTER TA
+ABA5; C; 13D5; # CHEROKEE SMALL LETTER DE
+ABA6; C; 13D6; # CHEROKEE SMALL LETTER TE
+ABA7; C; 13D7; # CHEROKEE SMALL LETTER DI
+ABA8; C; 13D8; # CHEROKEE SMALL LETTER TI
+ABA9; C; 13D9; # CHEROKEE SMALL LETTER DO
+ABAA; C; 13DA; # CHEROKEE SMALL LETTER DU
+ABAB; C; 13DB; # CHEROKEE SMALL LETTER DV
+ABAC; C; 13DC; # CHEROKEE SMALL LETTER DLA
+ABAD; C; 13DD; # CHEROKEE SMALL LETTER TLA
+ABAE; C; 13DE; # CHEROKEE SMALL LETTER TLE
+ABAF; C; 13DF; # CHEROKEE SMALL LETTER TLI
+ABB0; C; 13E0; # CHEROKEE SMALL LETTER TLO
+ABB1; C; 13E1; # CHEROKEE SMALL LETTER TLU
+ABB2; C; 13E2; # CHEROKEE SMALL LETTER TLV
+ABB3; C; 13E3; # CHEROKEE SMALL LETTER TSA
+ABB4; C; 13E4; # CHEROKEE SMALL LETTER TSE
+ABB5; C; 13E5; # CHEROKEE SMALL LETTER TSI
+ABB6; C; 13E6; # CHEROKEE SMALL LETTER TSO
+ABB7; C; 13E7; # CHEROKEE SMALL LETTER TSU
+ABB8; C; 13E8; # CHEROKEE SMALL LETTER TSV
+ABB9; C; 13E9; # CHEROKEE SMALL LETTER WA
+ABBA; C; 13EA; # CHEROKEE SMALL LETTER WE
+ABBB; C; 13EB; # CHEROKEE SMALL LETTER WI
+ABBC; C; 13EC; # CHEROKEE SMALL LETTER WO
+ABBD; C; 13ED; # CHEROKEE SMALL LETTER WU
+ABBE; C; 13EE; # CHEROKEE SMALL LETTER WV
+ABBF; C; 13EF; # CHEROKEE SMALL LETTER YA
+FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF
+FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI
+FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
+FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI
+FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL
+FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T
+FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST
+FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW
+FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH
+FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI
+FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW
+FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH
+FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A
+FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B
+FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C
+FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D
+FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E
+FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F
+FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G
+FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H
+FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I
+FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J
+FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K
+FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L
+FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M
+FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N
+FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O
+FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P
+FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q
+FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R
+FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S
+FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T
+FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U
+FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V
+FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W
+FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X
+FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y
+FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
+10400; C; 10428; # DESERET CAPITAL LETTER LONG I
+10401; C; 10429; # DESERET CAPITAL LETTER LONG E
+10402; C; 1042A; # DESERET CAPITAL LETTER LONG A
+10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH
+10404; C; 1042C; # DESERET CAPITAL LETTER LONG O
+10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO
+10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I
+10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E
+10408; C; 10430; # DESERET CAPITAL LETTER SHORT A
+10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH
+1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O
+1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO
+1040C; C; 10434; # DESERET CAPITAL LETTER AY
+1040D; C; 10435; # DESERET CAPITAL LETTER OW
+1040E; C; 10436; # DESERET CAPITAL LETTER WU
+1040F; C; 10437; # DESERET CAPITAL LETTER YEE
+10410; C; 10438; # DESERET CAPITAL LETTER H
+10411; C; 10439; # DESERET CAPITAL LETTER PEE
+10412; C; 1043A; # DESERET CAPITAL LETTER BEE
+10413; C; 1043B; # DESERET CAPITAL LETTER TEE
+10414; C; 1043C; # DESERET CAPITAL LETTER DEE
+10415; C; 1043D; # DESERET CAPITAL LETTER CHEE
+10416; C; 1043E; # DESERET CAPITAL LETTER JEE
+10417; C; 1043F; # DESERET CAPITAL LETTER KAY
+10418; C; 10440; # DESERET CAPITAL LETTER GAY
+10419; C; 10441; # DESERET CAPITAL LETTER EF
+1041A; C; 10442; # DESERET CAPITAL LETTER VEE
+1041B; C; 10443; # DESERET CAPITAL LETTER ETH
+1041C; C; 10444; # DESERET CAPITAL LETTER THEE
+1041D; C; 10445; # DESERET CAPITAL LETTER ES
+1041E; C; 10446; # DESERET CAPITAL LETTER ZEE
+1041F; C; 10447; # DESERET CAPITAL LETTER ESH
+10420; C; 10448; # DESERET CAPITAL LETTER ZHEE
+10421; C; 10449; # DESERET CAPITAL LETTER ER
+10422; C; 1044A; # DESERET CAPITAL LETTER EL
+10423; C; 1044B; # DESERET CAPITAL LETTER EM
+10424; C; 1044C; # DESERET CAPITAL LETTER EN
+10425; C; 1044D; # DESERET CAPITAL LETTER ENG
+10426; C; 1044E; # DESERET CAPITAL LETTER OI
+10427; C; 1044F; # DESERET CAPITAL LETTER EW
+10C80; C; 10CC0; # OLD HUNGARIAN CAPITAL LETTER A
+10C81; C; 10CC1; # OLD HUNGARIAN CAPITAL LETTER AA
+10C82; C; 10CC2; # OLD HUNGARIAN CAPITAL LETTER EB
+10C83; C; 10CC3; # OLD HUNGARIAN CAPITAL LETTER AMB
+10C84; C; 10CC4; # OLD HUNGARIAN CAPITAL LETTER EC
+10C85; C; 10CC5; # OLD HUNGARIAN CAPITAL LETTER ENC
+10C86; C; 10CC6; # OLD HUNGARIAN CAPITAL LETTER ECS
+10C87; C; 10CC7; # OLD HUNGARIAN CAPITAL LETTER ED
+10C88; C; 10CC8; # OLD HUNGARIAN CAPITAL LETTER AND
+10C89; C; 10CC9; # OLD HUNGARIAN CAPITAL LETTER E
+10C8A; C; 10CCA; # OLD HUNGARIAN CAPITAL LETTER CLOSE E
+10C8B; C; 10CCB; # OLD HUNGARIAN CAPITAL LETTER EE
+10C8C; C; 10CCC; # OLD HUNGARIAN CAPITAL LETTER EF
+10C8D; C; 10CCD; # OLD HUNGARIAN CAPITAL LETTER EG
+10C8E; C; 10CCE; # OLD HUNGARIAN CAPITAL LETTER EGY
+10C8F; C; 10CCF; # OLD HUNGARIAN CAPITAL LETTER EH
+10C90; C; 10CD0; # OLD HUNGARIAN CAPITAL LETTER I
+10C91; C; 10CD1; # OLD HUNGARIAN CAPITAL LETTER II
+10C92; C; 10CD2; # OLD HUNGARIAN CAPITAL LETTER EJ
+10C93; C; 10CD3; # OLD HUNGARIAN CAPITAL LETTER EK
+10C94; C; 10CD4; # OLD HUNGARIAN CAPITAL LETTER AK
+10C95; C; 10CD5; # OLD HUNGARIAN CAPITAL LETTER UNK
+10C96; C; 10CD6; # OLD HUNGARIAN CAPITAL LETTER EL
+10C97; C; 10CD7; # OLD HUNGARIAN CAPITAL LETTER ELY
+10C98; C; 10CD8; # OLD HUNGARIAN CAPITAL LETTER EM
+10C99; C; 10CD9; # OLD HUNGARIAN CAPITAL LETTER EN
+10C9A; C; 10CDA; # OLD HUNGARIAN CAPITAL LETTER ENY
+10C9B; C; 10CDB; # OLD HUNGARIAN CAPITAL LETTER O
+10C9C; C; 10CDC; # OLD HUNGARIAN CAPITAL LETTER OO
+10C9D; C; 10CDD; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE
+10C9E; C; 10CDE; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE
+10C9F; C; 10CDF; # OLD HUNGARIAN CAPITAL LETTER OEE
+10CA0; C; 10CE0; # OLD HUNGARIAN CAPITAL LETTER EP
+10CA1; C; 10CE1; # OLD HUNGARIAN CAPITAL LETTER EMP
+10CA2; C; 10CE2; # OLD HUNGARIAN CAPITAL LETTER ER
+10CA3; C; 10CE3; # OLD HUNGARIAN CAPITAL LETTER SHORT ER
+10CA4; C; 10CE4; # OLD HUNGARIAN CAPITAL LETTER ES
+10CA5; C; 10CE5; # OLD HUNGARIAN CAPITAL LETTER ESZ
+10CA6; C; 10CE6; # OLD HUNGARIAN CAPITAL LETTER ET
+10CA7; C; 10CE7; # OLD HUNGARIAN CAPITAL LETTER ENT
+10CA8; C; 10CE8; # OLD HUNGARIAN CAPITAL LETTER ETY
+10CA9; C; 10CE9; # OLD HUNGARIAN CAPITAL LETTER ECH
+10CAA; C; 10CEA; # OLD HUNGARIAN CAPITAL LETTER U
+10CAB; C; 10CEB; # OLD HUNGARIAN CAPITAL LETTER UU
+10CAC; C; 10CEC; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE
+10CAD; C; 10CED; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE
+10CAE; C; 10CEE; # OLD HUNGARIAN CAPITAL LETTER EV
+10CAF; C; 10CEF; # OLD HUNGARIAN CAPITAL LETTER EZ
+10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS
+10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN
+10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US
+118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA
+118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A
+118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI
+118A3; C; 118C3; # WARANG CITI CAPITAL LETTER YU
+118A4; C; 118C4; # WARANG CITI CAPITAL LETTER YA
+118A5; C; 118C5; # WARANG CITI CAPITAL LETTER YO
+118A6; C; 118C6; # WARANG CITI CAPITAL LETTER II
+118A7; C; 118C7; # WARANG CITI CAPITAL LETTER UU
+118A8; C; 118C8; # WARANG CITI CAPITAL LETTER E
+118A9; C; 118C9; # WARANG CITI CAPITAL LETTER O
+118AA; C; 118CA; # WARANG CITI CAPITAL LETTER ANG
+118AB; C; 118CB; # WARANG CITI CAPITAL LETTER GA
+118AC; C; 118CC; # WARANG CITI CAPITAL LETTER KO
+118AD; C; 118CD; # WARANG CITI CAPITAL LETTER ENY
+118AE; C; 118CE; # WARANG CITI CAPITAL LETTER YUJ
+118AF; C; 118CF; # WARANG CITI CAPITAL LETTER UC
+118B0; C; 118D0; # WARANG CITI CAPITAL LETTER ENN
+118B1; C; 118D1; # WARANG CITI CAPITAL LETTER ODD
+118B2; C; 118D2; # WARANG CITI CAPITAL LETTER TTE
+118B3; C; 118D3; # WARANG CITI CAPITAL LETTER NUNG
+118B4; C; 118D4; # WARANG CITI CAPITAL LETTER DA
+118B5; C; 118D5; # WARANG CITI CAPITAL LETTER AT
+118B6; C; 118D6; # WARANG CITI CAPITAL LETTER AM
+118B7; C; 118D7; # WARANG CITI CAPITAL LETTER BU
+118B8; C; 118D8; # WARANG CITI CAPITAL LETTER PU
+118B9; C; 118D9; # WARANG CITI CAPITAL LETTER HIYO
+118BA; C; 118DA; # WARANG CITI CAPITAL LETTER HOLO
+118BB; C; 118DB; # WARANG CITI CAPITAL LETTER HORR
+118BC; C; 118DC; # WARANG CITI CAPITAL LETTER HAR
+118BD; C; 118DD; # WARANG CITI CAPITAL LETTER SSUU
+118BE; C; 118DE; # WARANG CITI CAPITAL LETTER SII
+118BF; C; 118DF; # WARANG CITI CAPITAL LETTER VIYO
+#
+# EOF
diff --git a/rpython/rlib/unicodedata/CompositionExclusions-8.0.0.txt b/rpython/rlib/unicodedata/CompositionExclusions-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CompositionExclusions-8.0.0.txt
@@ -0,0 +1,206 @@
+# CompositionExclusions-8.0.0.txt
+# Date: 2015-02-19, 00:30:00 GMT [KW, LI]
+#
+# This file lists the characters for the Composition Exclusion Table
+# defined in UAX #15, Unicode Normalization Forms.
+#
+# This file is a normative contributory data file in the
+# Unicode Character Database.
+#
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+#
+# For more information, see
+# http://www.unicode.org/unicode/reports/tr15/#Primary_Exclusion_List_Table
+#
+# For a full derivation of composition exclusions, see the derived property
+# Full_Composition_Exclusion in DerivedNormalizationProps.txt
+#
+
+# ================================================
+# (1) Script Specifics
+#
+# This list of characters cannot be derived from the UnicodeData.txt file.
+# ================================================
+
+0958    #  DEVANAGARI LETTER QA
+0959    #  DEVANAGARI LETTER KHHA
+095A    #  DEVANAGARI LETTER GHHA
+095B    #  DEVANAGARI LETTER ZA
+095C    #  DEVANAGARI LETTER DDDHA
+095D    #  DEVANAGARI LETTER RHA
+095E    #  DEVANAGARI LETTER FA
+095F    #  DEVANAGARI LETTER YYA
+09DC    #  BENGALI LETTER RRA
+09DD    #  BENGALI LETTER RHA
+09DF    #  BENGALI LETTER YYA
+0A33    #  GURMUKHI LETTER LLA
+0A36    #  GURMUKHI LETTER SHA
+0A59    #  GURMUKHI LETTER KHHA
+0A5A    #  GURMUKHI LETTER GHHA
+0A5B    #  GURMUKHI LETTER ZA
+0A5E    #  GURMUKHI LETTER FA
+0B5C    #  ORIYA LETTER RRA
+0B5D    #  ORIYA LETTER RHA
+0F43    #  TIBETAN LETTER GHA
+0F4D    #  TIBETAN LETTER DDHA
+0F52    #  TIBETAN LETTER DHA
+0F57    #  TIBETAN LETTER BHA
+0F5C    #  TIBETAN LETTER DZHA
+0F69    #  TIBETAN LETTER KSSA
+0F76    #  TIBETAN VOWEL SIGN VOCALIC R
+0F78    #  TIBETAN VOWEL SIGN VOCALIC L
+0F93    #  TIBETAN SUBJOINED LETTER GHA
+0F9D    #  TIBETAN SUBJOINED LETTER DDHA
+0FA2    #  TIBETAN SUBJOINED LETTER DHA
+0FA7    #  TIBETAN SUBJOINED LETTER BHA
+0FAC    #  TIBETAN SUBJOINED LETTER DZHA
+0FB9    #  TIBETAN SUBJOINED LETTER KSSA
+FB1D    #  HEBREW LETTER YOD WITH HIRIQ
+FB1F    #  HEBREW LIGATURE YIDDISH YOD YOD PATAH
+FB2A    #  HEBREW LETTER SHIN WITH SHIN DOT
+FB2B    #  HEBREW LETTER SHIN WITH SIN DOT
+FB2C    #  HEBREW LETTER SHIN WITH DAGESH AND SHIN DOT
+FB2D    #  HEBREW LETTER SHIN WITH DAGESH AND SIN DOT
+FB2E    #  HEBREW LETTER ALEF WITH PATAH
+FB2F    #  HEBREW LETTER ALEF WITH QAMATS
+FB30    #  HEBREW LETTER ALEF WITH MAPIQ
+FB31    #  HEBREW LETTER BET WITH DAGESH
+FB32    #  HEBREW LETTER GIMEL WITH DAGESH
+FB33    #  HEBREW LETTER DALET WITH DAGESH
+FB34    #  HEBREW LETTER HE WITH MAPIQ
+FB35    #  HEBREW LETTER VAV WITH DAGESH
+FB36    #  HEBREW LETTER ZAYIN WITH DAGESH
+FB38    #  HEBREW LETTER TET WITH DAGESH
+FB39    #  HEBREW LETTER YOD WITH DAGESH
+FB3A    #  HEBREW LETTER FINAL KAF WITH DAGESH
+FB3B    #  HEBREW LETTER KAF WITH DAGESH
+FB3C    #  HEBREW LETTER LAMED WITH DAGESH
+FB3E    #  HEBREW LETTER MEM WITH DAGESH
+FB40    #  HEBREW LETTER NUN WITH DAGESH
+FB41    #  HEBREW LETTER SAMEKH WITH DAGESH
+FB43    #  HEBREW LETTER FINAL PE WITH DAGESH
+FB44    #  HEBREW LETTER PE WITH DAGESH
+FB46    #  HEBREW LETTER TSADI WITH DAGESH
+FB47    #  HEBREW LETTER QOF WITH DAGESH
+FB48    #  HEBREW LETTER RESH WITH DAGESH
+FB49    #  HEBREW LETTER SHIN WITH DAGESH
+FB4A    #  HEBREW LETTER TAV WITH DAGESH
+FB4B    #  HEBREW LETTER VAV WITH HOLAM
+FB4C    #  HEBREW LETTER BET WITH RAFE
+FB4D    #  HEBREW LETTER KAF WITH RAFE
+FB4E    #  HEBREW LETTER PE WITH RAFE
+
+# Total code points: 67
+
+# ================================================
+# (2) Post Composition Version precomposed characters
+#
+# These characters cannot be derived solely from the UnicodeData.txt file
+# in this version of Unicode.
+#
+# Note that characters added to the standard after the
+# Composition Version and which have canonical decomposition mappings
+# are not automatically added to this list of Post Composition
+# Version precomposed characters.
+# ================================================
+
+2ADC    #  FORKING
+1D15E   #  MUSICAL SYMBOL HALF NOTE
+1D15F   #  MUSICAL SYMBOL QUARTER NOTE
+1D160   #  MUSICAL SYMBOL EIGHTH NOTE
+1D161   #  MUSICAL SYMBOL SIXTEENTH NOTE
+1D162   #  MUSICAL SYMBOL THIRTY-SECOND NOTE
+1D163   #  MUSICAL SYMBOL SIXTY-FOURTH NOTE
+1D164   #  MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE
+1D1BB   #  MUSICAL SYMBOL MINIMA
+1D1BC   #  MUSICAL SYMBOL MINIMA BLACK
+1D1BD   #  MUSICAL SYMBOL SEMIMINIMA WHITE
+1D1BE   #  MUSICAL SYMBOL SEMIMINIMA BLACK
+1D1BF   #  MUSICAL SYMBOL FUSA WHITE
+1D1C0   #  MUSICAL SYMBOL FUSA BLACK
+
+# Total code points: 14
+
+# ================================================
+# (3) Singleton Decompositions
+#
+# These characters can be derived from the UnicodeData.txt file
+# by including all canonically decomposable characters whose
+# canonical decomposition consists of a single character.
+#
+# These characters are simply quoted here for reference.
+# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt
+# ================================================
+
+# 0340..0341       [2] COMBINING GRAVE TONE MARK..COMBINING ACUTE TONE MARK
+# 0343                 COMBINING GREEK KORONIS
+# 0374                 GREEK NUMERAL SIGN
+# 037E                 GREEK QUESTION MARK
+# 0387                 GREEK ANO TELEIA
+# 1F71                 GREEK SMALL LETTER ALPHA WITH OXIA
+# 1F73                 GREEK SMALL LETTER EPSILON WITH OXIA
+# 1F75                 GREEK SMALL LETTER ETA WITH OXIA
+# 1F77                 GREEK SMALL LETTER IOTA WITH OXIA
+# 1F79                 GREEK SMALL LETTER OMICRON WITH OXIA
+# 1F7B                 GREEK SMALL LETTER UPSILON WITH OXIA
+# 1F7D                 GREEK SMALL LETTER OMEGA WITH OXIA
+# 1FBB                 GREEK CAPITAL LETTER ALPHA WITH OXIA
+# 1FBE                 GREEK PROSGEGRAMMENI
+# 1FC9                 GREEK CAPITAL LETTER EPSILON WITH OXIA
+# 1FCB                 GREEK CAPITAL LETTER ETA WITH OXIA
+# 1FD3                 GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+# 1FDB                 GREEK CAPITAL LETTER IOTA WITH OXIA
+# 1FE3                 GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+# 1FEB                 GREEK CAPITAL LETTER UPSILON WITH OXIA
+# 1FEE..1FEF       [2] GREEK DIALYTIKA AND OXIA..GREEK VARIA
+# 1FF9                 GREEK CAPITAL LETTER OMICRON WITH OXIA
+# 1FFB                 GREEK CAPITAL LETTER OMEGA WITH OXIA
+# 1FFD                 GREEK OXIA
+# 2000..2001       [2] EN QUAD..EM QUAD
+# 2126                 OHM SIGN
+# 212A..212B       [2] KELVIN SIGN..ANGSTROM SIGN
+# 2329                 LEFT-POINTING ANGLE BRACKET
+# 232A                 RIGHT-POINTING ANGLE BRACKET
+# F900..FA0D     [270] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA0D
+# FA10                 CJK COMPATIBILITY IDEOGRAPH-FA10
+# FA12                 CJK COMPATIBILITY IDEOGRAPH-FA12
+# FA15..FA1E      [10] CJK COMPATIBILITY IDEOGRAPH-FA15..CJK COMPATIBILITY IDEOGRAPH-FA1E
+# FA20                 CJK COMPATIBILITY IDEOGRAPH-FA20
+# FA22                 CJK COMPATIBILITY IDEOGRAPH-FA22
+# FA25..FA26       [2] CJK COMPATIBILITY IDEOGRAPH-FA25..CJK COMPATIBILITY IDEOGRAPH-FA26
+# FA2A..FA6D      [68] CJK COMPATIBILITY IDEOGRAPH-FA2A..CJK COMPATIBILITY IDEOGRAPH-FA6D
+# FA70..FAD9     [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9
+# 2F800..2FA1D   [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
+
+# Total code points: 1035
+
+# ================================================
+# (4) Non-Starter Decompositions
+#
+# These characters can be derived from the UnicodeData.txt file
+# by including each expanding canonical decomposition
+# (i.e., those which canonically decompose to a sequence
+# of characters instead of a single character), such that:
+#
+# A. The character is not a Starter.
+#
+# OR (inclusive)
+#
+# B. The character's canonical decomposition begins
+# with a character that is not a Starter.
+#
+# Note that a "Starter" is any character with a zero combining class.
+#
+# These characters are simply quoted here for reference.
+# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt
+# ================================================
+
+# 0344                 COMBINING GREEK DIALYTIKA TONOS
+# 0F73                 TIBETAN VOWEL SIGN II
+# 0F75                 TIBETAN VOWEL SIGN UU
+# 0F81                 TIBETAN VOWEL SIGN REVERSED II
+
+# Total code points: 4
+
+# EOF
diff --git a/rpython/rlib/unicodedata/DerivedCoreProperties-8.0.0.txt b/rpython/rlib/unicodedata/DerivedCoreProperties-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/DerivedCoreProperties-8.0.0.txt
@@ -0,0 +1,11029 @@
+# DerivedCoreProperties-8.0.0.txt
+# Date: 2015-03-11, 22:29:21 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+
+# ================================================
+
+# Derived Property: Math
+#  Generated from: Sm + Other_Math
+
+002B          ; Math # Sm       PLUS SIGN
+003C..003E    ; Math # Sm   [3] LESS-THAN SIGN..GREATER-THAN SIGN
+005E          ; Math # Sk       CIRCUMFLEX ACCENT
+007C          ; Math # Sm       VERTICAL LINE
+007E          ; Math # Sm       TILDE
+00AC          ; Math # Sm       NOT SIGN
+00B1          ; Math # Sm       PLUS-MINUS SIGN
+00D7          ; Math # Sm       MULTIPLICATION SIGN
+00F7          ; Math # Sm       DIVISION SIGN
+03D0..03D2    ; Math # L&   [3] GREEK BETA SYMBOL..GREEK UPSILON WITH HOOK SYMBOL
+03D5          ; Math # L&       GREEK PHI SYMBOL
+03F0..03F1    ; Math # L&   [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL
+03F4..03F5    ; Math # L&   [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL
+03F6          ; Math # Sm       GREEK REVERSED LUNATE EPSILON SYMBOL
+0606..0608    ; Math # Sm   [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY
+2016          ; Math # Po       DOUBLE VERTICAL LINE
+2032..2034    ; Math # Po   [3] PRIME..TRIPLE PRIME
+2040          ; Math # Pc       CHARACTER TIE
+2044          ; Math # Sm       FRACTION SLASH
+2052          ; Math # Sm       COMMERCIAL MINUS SIGN
+2061..2064    ; Math # Cf   [4] FUNCTION APPLICATION..INVISIBLE PLUS
+207A..207C    ; Math # Sm   [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN
+207D          ; Math # Ps       SUPERSCRIPT LEFT PARENTHESIS
+207E          ; Math # Pe       SUPERSCRIPT RIGHT PARENTHESIS
+208A..208C    ; Math # Sm   [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN
+208D          ; Math # Ps       SUBSCRIPT LEFT PARENTHESIS
+208E          ; Math # Pe       SUBSCRIPT RIGHT PARENTHESIS
+20D0..20DC    ; Math # Mn  [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE
+20E1          ; Math # Mn       COMBINING LEFT RIGHT ARROW ABOVE
+20E5..20E6    ; Math # Mn   [2] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING DOUBLE VERTICAL STROKE OVERLAY
+20EB..20EF    ; Math # Mn   [5] COMBINING LONG DOUBLE SOLIDUS OVERLAY..COMBINING RIGHT ARROW BELOW
+2102          ; Math # L&       DOUBLE-STRUCK CAPITAL C
+2107          ; Math # L&       EULER CONSTANT
+210A..2113    ; Math # L&  [10] SCRIPT SMALL G..SCRIPT SMALL L
+2115          ; Math # L&       DOUBLE-STRUCK CAPITAL N
+2118          ; Math # Sm       SCRIPT CAPITAL P
+2119..211D    ; Math # L&   [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R
+2124          ; Math # L&       DOUBLE-STRUCK CAPITAL Z
+2128          ; Math # L&       BLACK-LETTER CAPITAL Z
+2129          ; Math # So       TURNED GREEK SMALL LETTER IOTA
+212C..212D    ; Math # L&   [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C
+212F..2131    ; Math # L&   [3] SCRIPT SMALL E..SCRIPT CAPITAL F
+2133..2134    ; Math # L&   [2] SCRIPT CAPITAL M..SCRIPT SMALL O
+2135..2138    ; Math # Lo   [4] ALEF SYMBOL..DALET SYMBOL
+213C..213F    ; Math # L&   [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI
+2140..2144    ; Math # Sm   [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y
+2145..2149    ; Math # L&   [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J
+214B          ; Math # Sm       TURNED AMPERSAND
+2190..2194    ; Math # Sm   [5] LEFTWARDS ARROW..LEFT RIGHT ARROW
+2195..2199    ; Math # So   [5] UP DOWN ARROW..SOUTH WEST ARROW
+219A..219B    ; Math # Sm   [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE
+219C..219F    ; Math # So   [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW
+21A0          ; Math # Sm       RIGHTWARDS TWO HEADED ARROW
+21A1..21A2    ; Math # So   [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL
+21A3          ; Math # Sm       RIGHTWARDS ARROW WITH TAIL
+21A4..21A5    ; Math # So   [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR
+21A6          ; Math # Sm       RIGHTWARDS ARROW FROM BAR
+21A7          ; Math # So       DOWNWARDS ARROW FROM BAR
+21A9..21AD    ; Math # So   [5] LEFTWARDS ARROW WITH HOOK..LEFT RIGHT WAVE ARROW
+21AE          ; Math # Sm       LEFT RIGHT ARROW WITH STROKE
+21B0..21B1    ; Math # So   [2] UPWARDS ARROW WITH TIP LEFTWARDS..UPWARDS ARROW WITH TIP RIGHTWARDS
+21B6..21B7    ; Math # So   [2] ANTICLOCKWISE TOP SEMICIRCLE ARROW..CLOCKWISE TOP SEMICIRCLE ARROW
+21BC..21CD    ; Math # So  [18] LEFTWARDS HARPOON WITH BARB UPWARDS..LEFTWARDS DOUBLE ARROW WITH STROKE
+21CE..21CF    ; Math # Sm   [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE
+21D0..21D1    ; Math # So   [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW
+21D2          ; Math # Sm       RIGHTWARDS DOUBLE ARROW
+21D3          ; Math # So       DOWNWARDS DOUBLE ARROW
+21D4          ; Math # Sm       LEFT RIGHT DOUBLE ARROW
+21D5..21DB    ; Math # So   [7] UP DOWN DOUBLE ARROW..RIGHTWARDS TRIPLE ARROW
+21DD          ; Math # So       RIGHTWARDS SQUIGGLE ARROW
+21E4..21E5    ; Math # So   [2] LEFTWARDS ARROW TO BAR..RIGHTWARDS ARROW TO BAR
+21F4..22FF    ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP
+2308          ; Math # Ps       LEFT CEILING
+2309          ; Math # Pe       RIGHT CEILING
+230A          ; Math # Ps       LEFT FLOOR
+230B          ; Math # Pe       RIGHT FLOOR
+2320..2321    ; Math # Sm   [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL
+237C          ; Math # Sm       RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW
+239B..23B3    ; Math # Sm  [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM
+23B4..23B5    ; Math # So   [2] TOP SQUARE BRACKET..BOTTOM SQUARE BRACKET
+23B7          ; Math # So       RADICAL SYMBOL BOTTOM
+23D0          ; Math # So       VERTICAL LINE EXTENSION
+23DC..23E1    ; Math # Sm   [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET
+23E2          ; Math # So       WHITE TRAPEZIUM
+25A0..25A1    ; Math # So   [2] BLACK SQUARE..WHITE SQUARE
+25AE..25B6    ; Math # So   [9] BLACK VERTICAL RECTANGLE..BLACK RIGHT-POINTING TRIANGLE
+25B7          ; Math # Sm       WHITE RIGHT-POINTING TRIANGLE
+25BC..25C0    ; Math # So   [5] BLACK DOWN-POINTING TRIANGLE..BLACK LEFT-POINTING TRIANGLE
+25C1          ; Math # Sm       WHITE LEFT-POINTING TRIANGLE
+25C6..25C7    ; Math # So   [2] BLACK DIAMOND..WHITE DIAMOND
+25CA..25CB    ; Math # So   [2] LOZENGE..WHITE CIRCLE
+25CF..25D3    ; Math # So   [5] BLACK CIRCLE..CIRCLE WITH UPPER HALF BLACK
+25E2          ; Math # So       BLACK LOWER RIGHT TRIANGLE
+25E4          ; Math # So       BLACK UPPER LEFT TRIANGLE
+25E7..25EC    ; Math # So   [6] SQUARE WITH LEFT HALF BLACK..WHITE UP-POINTING TRIANGLE WITH DOT
+25F8..25FF    ; Math # Sm   [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE
+2605..2606    ; Math # So   [2] BLACK STAR..WHITE STAR
+2640          ; Math # So       FEMALE SIGN
+2642          ; Math # So       MALE SIGN
+2660..2663    ; Math # So   [4] BLACK SPADE SUIT..BLACK CLUB SUIT
+266D..266E    ; Math # So   [2] MUSIC FLAT SIGN..MUSIC NATURAL SIGN
+266F          ; Math # Sm       MUSIC SHARP SIGN
+27C0..27C4    ; Math # Sm   [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET
+27C5          ; Math # Ps       LEFT S-SHAPED BAG DELIMITER
+27C6          ; Math # Pe       RIGHT S-SHAPED BAG DELIMITER
+27C7..27E5    ; Math # Sm  [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK
+27E6          ; Math # Ps       MATHEMATICAL LEFT WHITE SQUARE BRACKET
+27E7          ; Math # Pe       MATHEMATICAL RIGHT WHITE SQUARE BRACKET
+27E8          ; Math # Ps       MATHEMATICAL LEFT ANGLE BRACKET
+27E9          ; Math # Pe       MATHEMATICAL RIGHT ANGLE BRACKET
+27EA          ; Math # Ps       MATHEMATICAL LEFT DOUBLE ANGLE BRACKET
+27EB          ; Math # Pe       MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET
+27EC          ; Math # Ps       MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET
+27ED          ; Math # Pe       MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET
+27EE          ; Math # Ps       MATHEMATICAL LEFT FLATTENED PARENTHESIS
+27EF          ; Math # Pe       MATHEMATICAL RIGHT FLATTENED PARENTHESIS
+27F0..27FF    ; Math # Sm  [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW
+2900..2982    ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON
+2983          ; Math # Ps       LEFT WHITE CURLY BRACKET
+2984          ; Math # Pe       RIGHT WHITE CURLY BRACKET
+2985          ; Math # Ps       LEFT WHITE PARENTHESIS
+2986          ; Math # Pe       RIGHT WHITE PARENTHESIS

From pypy.commits at gmail.com  Tue Jan 10 11:53:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jan 2017 08:53:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Tweak: W_IOBase is the only class in
 CPython 3.5 to have both a
Message-ID: <5875119a.66bec20a.fa38d.6a98@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89476:3e79e13a6a69
Date: 2017-01-10 17:53 +0100
http://bitbucket.org/pypy/pypy/changeset/3e79e13a6a69/

Log:	Tweak: W_IOBase is the only class in CPython 3.5 to have both a
	tp_finalize and be overridable. This gives semantics that are
	markedly different from the old (<= 3.3) ones, and that are closer
	to app-level classes. See comments

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -13,7 +13,7 @@
 
 class TypeDef(object):
     def __init__(self, __name, __base=None, __total_ordering__=None,
-                 __buffer=None, **rawdict):
+                 __buffer=None, __confirm_applevel_del__=False, **rawdict):
         "NOT_RPYTHON: initialization-time only"
         self.name = __name
         if __base is None:
@@ -29,7 +29,8 @@
         self.heaptype = False
         self.hasdict = '__dict__' in rawdict
         # no __del__: use an RPython _finalize_() method and register_finalizer
-        assert '__del__' not in rawdict
+        if not __confirm_applevel_del__:
+            assert '__del__' not in rawdict
         self.weakrefable = '__weakref__' in rawdict
         self.doc = rawdict.get('__doc__', None)
         for base in bases:
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -1010,16 +1010,6 @@
             self.w_writer = None
             raise
 
-    def _finalize_(self):
-        # Don't call the base __del__: do not close the files!
-        # Usually the _finalize_() method is not called at all because
-        # we set 'needs_to_finalize = False' in this class, so
-        # W_IOBase.__init__() won't call register_finalizer().
-        # However, this method might still be called: if the user
-        # makes an app-level subclass and adds a custom __del__.
-        pass
-    needs_to_finalize = False
-
     # forward to reader
     for method in ['read', 'peek', 'read1', 'readinto', 'readable']:
         locals()[method + '_w'] = make_forwarding_method(
@@ -1052,6 +1042,10 @@
         if e:
             raise e
 
+    def needs_finalizer(self):
+        # self.w_writer and self.w_reader have their own finalizer
+        return type(self) is not W_BufferedRWPair
+
     def isatty_w(self, space):
         if space.is_true(space.call_method(self.w_writer, "isatty")):
             return space.w_True
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -166,6 +166,10 @@
     def close_w(self, space):
         self.close()
 
+    def needs_finalizer(self):
+        # self.close() is not necessary when the object goes away
+        return type(self) is not W_BytesIO
+
     def closed_get_w(self, space):
         return space.wrap(self.is_closed())
 
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -60,7 +60,7 @@
         self.__IOBase_closed = False
         if add_to_autoflusher:
             get_autoflusher(space).add(self)
-        if self.needs_to_finalize:
+        if self.needs_finalizer():
             self.register_finalizer(space)
 
     def getdict(self, space):
@@ -75,6 +75,18 @@
         return False
 
     def _finalize_(self):
+        # Note: there is only this empty _finalize_() method here, but
+        # we still need register_finalizer() so that descr_del() is
+        # called.  IMPORTANT: this is not the recommended way to have a
+        # finalizer!  It makes the finalizer appear as __del__() from
+        # app-level, and the user can call __del__() explicitly, or
+        # override it, with or without calling the parent's __del__().
+        # This matches 'tp_finalize' in CPython >= 3.4.  So far (3.5),
+        # this is the only built-in class with a 'tp_finalize' slot that
+        # can be subclassed.
+        pass
+
+    def descr_del(self):
         space = self.space
         w_closed = space.findattr(self, space.wrap('closed'))
         try:
@@ -90,7 +102,6 @@
             # equally as bad, and potentially more frequent (because of
             # shutdown issues).
             pass
-    needs_to_finalize = True
 
     def _CLOSED(self):
         # Use this macro whenever you want to check the internal `closed`
@@ -128,6 +139,11 @@
         finally:
             self.__IOBase_closed = True
 
+    def needs_finalizer(self):
+        # can return False if we know that the precise close() method
+        # of this class will have no effect
+        return True
+
     def _dealloc_warn_w(self, space, w_source):
         """Called when the io is implicitly closed via the deconstructor"""
         pass
@@ -318,6 +334,8 @@
                             doc="True if the file is closed"),
     __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, cls=W_IOBase),
     __weakref__ = make_weakref_descr(W_IOBase),
+    __del__ = interp2app(W_IOBase.descr_del),
+    __confirm_applevel_del__ = True,
 
     readline = interp2app(W_IOBase.readline_w),
     readlines = interp2app(W_IOBase.readlines_w),
diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -248,6 +248,10 @@
     def close_w(self, space):
         self.buf = None
 
+    def needs_finalizer(self):
+        # 'self.buf = None' is not necessary when the object goes away
+        return type(self) is not W_StringIO
+
     def closed_get_w(self, space):
         return space.wrap(self.buf is None)
 
diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py
--- a/pypy/module/_io/test/test_bufferedio.py
+++ b/pypy/module/_io/test/test_bufferedio.py
@@ -337,13 +337,33 @@
         b.flush()
         assert self.readfile() == b'x' * 40
 
-    def test_destructor(self):
+    def test_destructor_1(self):
         import _io
 
         record = []
         class MyIO(_io.BufferedWriter):
             def __del__(self):
                 record.append(1)
+                # doesn't call the inherited __del__, so file not closed
+            def close(self):
+                record.append(2)
+                super(MyIO, self).close()
+            def flush(self):
+                record.append(3)
+                super(MyIO, self).flush()
+        raw = _io.FileIO(self.tmpfile, 'w')
+        MyIO(raw)
+        import gc; gc.collect()
+        assert record == [1]
+
+    def test_destructor_2(self):
+        import _io
+
+        record = []
+        class MyIO(_io.BufferedWriter):
+            def __del__(self):
+                record.append(1)
+                super(MyIO, self).__del__()
             def close(self):
                 record.append(2)
                 super(MyIO, self).close()
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -80,7 +80,7 @@
         bufio.flush()
         assert f.getvalue() == b"ABC"
 
-    def test_destructor(self):
+    def test_destructor_1(self):
         import io
         io.IOBase()
 
@@ -88,6 +88,26 @@
         class MyIO(io.IOBase):
             def __del__(self):
                 record.append(1)
+                # doesn't call the inherited __del__, so file not closed
+            def close(self):
+                record.append(2)
+                super(MyIO, self).close()
+            def flush(self):
+                record.append(3)
+                super(MyIO, self).flush()
+        MyIO()
+        import gc; gc.collect()
+        assert record == [1]
+
+    def test_destructor_2(self):
+        import io
+        io.IOBase()
+
+        record = []
+        class MyIO(io.IOBase):
+            def __del__(self):
+                record.append(1)
+                super(MyIO, self).__del__()
             def close(self):
                 record.append(2)
                 super(MyIO, self).close()

From pypy.commits at gmail.com  Tue Jan 10 13:47:04 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 10 Jan 2017 10:47:04 -0800 (PST)
Subject: [pypy-commit] pypy api_func-refactor: Close branch api_func-refactor
Message-ID: <58752c28.c9f8c20a.ec14.a4cd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: api_func-refactor
Changeset: r89477:b7e9f3538e0e
Date: 2017-01-10 18:46 +0000
http://bitbucket.org/pypy/pypy/changeset/b7e9f3538e0e/

Log:	Close branch api_func-refactor


From pypy.commits at gmail.com  Tue Jan 10 13:47:21 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 10 Jan 2017 10:47:21 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in api_func-refactor (pull
 request #505)
Message-ID: <58752c39.068f1c0a.f38ad.f79d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89478:11109e66a206
Date: 2017-01-10 18:46 +0000
http://bitbucket.org/pypy/pypy/changeset/11109e66a206/

Log:	Merged in api_func-refactor (pull request #505)

	Refactor ApiFunction

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -329,66 +329,19 @@
             wrapper.c_name = cpyext_namespace.uniquename(self.c_name)
         return wrapper
 
-DEFAULT_HEADER = 'pypy_decl.h'
-def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
-                gil=None, result_borrowed=False, result_is_ll=False):
-    """
-    Declares a function to be exported.
-    - `argtypes`, `restype` are lltypes and describe the function signature.
-    - `error` is the value returned when an applevel exception is raised. The
-      special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual
-      exception into a wrapped SystemError.  Unwrapped exceptions also cause a
-      SytemError.
-    - `header` is the header file to export the function in, Set to None to get
-      a C function pointer, but not exported by the API headers.
-    - set `gil` to "acquire", "release" or "around" to acquire the GIL,
-      release the GIL, or both
-    """
-    if isinstance(restype, lltype.Typedef):
-        real_restype = restype.OF
-    else:
-        real_restype = restype
-
-    if error is _NOT_SPECIFIED:
-        if isinstance(real_restype, lltype.Ptr):
-            error = lltype.nullptr(real_restype.TO)
-        elif real_restype is lltype.Void:
-            error = CANNOT_FAIL
-    if type(error) is int:
-        error = rffi.cast(real_restype, error)
-    expect_integer = (isinstance(real_restype, lltype.Primitive) and
-                      rffi.cast(restype, 0) == 0)
-
-    def decorate(func):
-        func._always_inline_ = 'try'
-        func_name = func.func_name
-        if header is not None:
-            c_name = None
-            if func_name in FUNCTIONS_BY_HEADER[header]:
-                raise ValueError("%s already registered" % func_name)
-        else:
-            c_name = func_name
-        api_function = ApiFunction(argtypes, restype, func, error,
-                                   c_name=c_name, gil=gil,
-                                   result_borrowed=result_borrowed,
-                                   result_is_ll=result_is_ll)
-        func.api_func = api_function
-
-        if error is _NOT_SPECIFIED:
-            raise ValueError("function %s has no return value for exceptions"
-                             % func)
-        names = api_function.argnames
-        types_names_enum_ui = unrolling_iterable(enumerate(
-            zip(api_function.argtypes,
-                [tp_name.startswith("w_") for tp_name in names])))
+    def get_unwrapper(self):
+        names = self.argnames
+        argtypesw = zip(self.argtypes,
+                        [_name.startswith("w_") for _name in self.argnames])
+        types_names_enum_ui = unrolling_iterable(enumerate(argtypesw))
 
         @specialize.ll()
         def unwrapper(space, *args):
-            from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj
+            from pypy.module.cpyext.pyobject import is_pyobj
             from pypy.module.cpyext.pyobject import from_ref, as_pyobj
             newargs = ()
             keepalives = ()
-            assert len(args) == len(api_function.argtypes)
+            assert len(args) == len(self.argtypes)
             for i, (ARG, is_wrapped) in types_names_enum_ui:
                 input_arg = args[i]
                 if is_PyObject(ARG) and not is_wrapped:
@@ -413,31 +366,79 @@
                         arg = from_ref(space, input_arg)
                     else:
                         arg = input_arg
-
-                        ## ZZZ: for is_pyobj:
-                        ## try:
-                        ##     arg = from_ref(space,
-                        ##                rffi.cast(PyObject, input_arg))
-                        ## except TypeError, e:
-                        ##     err = oefmt(space.w_TypeError,
-                        ##                 "could not cast arg to PyObject")
-                        ##     if not catch_exception:
-                        ##         raise err
-                        ##     state = space.fromcache(State)
-                        ##     state.set_exception(err)
-                        ##     if is_PyObject(restype):
-                        ##         return None
-                        ##     else:
-                        ##         return api_function.error_value
                 else:
                     # arg is not declared as PyObject, no magic
                     arg = input_arg
                 newargs += (arg, )
             try:
-                return func(space, *newargs)
+                return self.callable(space, *newargs)
             finally:
                 keepalive_until_here(*keepalives)
+        return unwrapper
 
+    def get_c_restype(self, c_writer):
+        return c_writer.gettype(self.restype).replace('@', '').strip()
+
+    def get_c_args(self, c_writer):
+        args = []
+        for i, argtype in enumerate(self.argtypes):
+            if argtype is CONST_STRING:
+                arg = 'const char *@'
+            elif argtype is CONST_STRINGP:
+                arg = 'const char **@'
+            elif argtype is CONST_WSTRING:
+                arg = 'const wchar_t *@'
+            else:
+                arg = c_writer.gettype(argtype)
+            arg = arg.replace('@', 'arg%d' % (i,)).strip()
+            args.append(arg)
+        args = ', '.join(args) or "void"
+        return args
+
+    def get_api_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "PyAPI_FUNC({restype}) {name}({args});".format(**locals())
+
+    def get_ptr_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "{restype} (*{name})({args});".format(**locals())
+
+    def get_ctypes_impl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        callargs = ', '.join('arg%d' % (i,)
+                            for i in range(len(self.argtypes)))
+        if self.restype is lltype.Void:
+            body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+        else:
+            body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+        return '%s %s(%s)\n%s' % (restype, name, args, body)
+
+
+DEFAULT_HEADER = 'pypy_decl.h'
+def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
+                gil=None, result_borrowed=False, result_is_ll=False):
+    """
+    Declares a function to be exported.
+    - `argtypes`, `restype` are lltypes and describe the function signature.
+    - `error` is the value returned when an applevel exception is raised. The
+      special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual
+      exception into a wrapped SystemError.  Unwrapped exceptions also cause a
+      SytemError.
+    - `header` is the header file to export the function in.
+    - set `gil` to "acquire", "release" or "around" to acquire the GIL,
+      release the GIL, or both
+    """
+    assert header is not None
+    def decorate(func):
+        if func.__name__ in FUNCTIONS_BY_HEADER[header]:
+            raise ValueError("%s already registered" % func.__name__)
+        api_function = _create_api_func(
+            func, argtypes, restype, error, gil=gil,
+            result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+        unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
 
@@ -449,25 +450,64 @@
             try:
                 res = unwrapper(space, *args)
             except OperationError as e:
-                if not hasattr(api_function, "error_value"):
+                if not hasattr(unwrapper.api_func, "error_value"):
                     raise
                 state = space.fromcache(State)
                 state.set_exception(e)
                 if is_PyObject(restype):
                     return None
                 else:
-                    return api_function.error_value
+                    return unwrapper.api_func.error_value
             got_integer = isinstance(res, (int, long, float))
+            if isinstance(restype, lltype.Typedef):
+                real_restype = restype.OF
+            else:
+                real_restype = restype
+            expect_integer = (isinstance(real_restype, lltype.Primitive) and
+                            rffi.cast(restype, 0) == 0)
             assert got_integer == expect_integer, (
                 'got %r not integer' % (res,))
             return res
 
         if header is not None:
-            FUNCTIONS_BY_HEADER[header][func_name] = api_function
-        INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
-        return unwrapper  # used in 'normal' RPython code.
+            FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
+            INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
+        return unwrapper
     return decorate
 
+def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
+    def decorate(func):
+        c_name = func.__name__
+        api_function = _create_api_func(func, argtypes, restype, error, c_name)
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
+
+
+def _create_api_func(
+        func, argtypes, restype, error=_NOT_SPECIFIED, c_name=None,
+        gil=None, result_borrowed=False, result_is_ll=False):
+    if isinstance(restype, lltype.Typedef):
+        real_restype = restype.OF
+    else:
+        real_restype = restype
+
+    if error is _NOT_SPECIFIED:
+        if isinstance(real_restype, lltype.Ptr):
+            error = lltype.nullptr(real_restype.TO)
+        elif real_restype is lltype.Void:
+            error = CANNOT_FAIL
+    if type(error) is int:
+        error = rffi.cast(real_restype, error)
+
+    func._always_inline_ = 'try'
+    return ApiFunction(
+        argtypes, restype, func, error, c_name=c_name, gil=gil,
+        result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+
+
 def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
     if level == 1:
@@ -978,23 +1018,6 @@
     for func in BOOTSTRAP_FUNCTIONS:
         func(space)
 
-def c_function_signature(db, func):
-    restype = db.gettype(func.restype).replace('@', '').strip()
-    args = []
-    for i, argtype in enumerate(func.argtypes):
-        if argtype is CONST_STRING:
-            arg = 'const char *@'
-        elif argtype is CONST_STRINGP:
-            arg = 'const char **@'
-        elif argtype is CONST_WSTRING:
-            arg = 'const wchar_t *@'
-        else:
-            arg = db.gettype(argtype)
-        arg = arg.replace('@', 'arg%d' % (i,)).strip()
-        args.append(arg)
-    args = ', '.join(args) or "void"
-    return restype, args
-
 #_____________________________________________________
 # Build the bridge DLL, Allow extension DLLs to call
 # back into Pypy space functions
@@ -1014,15 +1037,8 @@
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            restype, args = c_function_signature(db, func)
-            callargs = ', '.join('arg%d' % (i,)
-                                for i in range(len(func.argtypes)))
-            if func.restype is lltype.Void:
-                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
-            else:
-                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
-            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
-            members.append('%s (*%s)(%s);' % (restype, name, args))
+            functions.append(func.get_ctypes_impl(name, db))
+            members.append(func.get_ptr_decl(name, db))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
     struct_declaration_code = """\
@@ -1217,8 +1233,7 @@
         for name, func in sorted(header_functions.iteritems()):
             _name = mangle_name(prefix, name)
             header.append("#define %s %s" % (name, _name))
-            restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
+            header.append(func.get_api_decl(name, db))
 
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
+    cpython_api, Py_ssize_t, cpython_struct, bootstrap_function, slot_function,
     PyObjectFields, PyObject)
 from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.module.array.interp_array import ArrayBuffer
@@ -72,7 +72,7 @@
                 "Don't know how to realize a buffer")
 
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
     if py_buf.c_b_base:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL)
+    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -25,14 +25,14 @@
 ##
 ## In the PyBytesObject returned, the ob_sval buffer may be modified as
 ## long as the freshly allocated PyBytesObject is not "forced" via a call
-## to any of the more sophisticated C-API functions. 
+## to any of the more sophisticated C-API functions.
 ##
 ## Care has been taken in implementing the functions below, so that
-## if they are called with a non-forced PyBytesObject, they will not 
+## if they are called with a non-forced PyBytesObject, they will not
 ## unintentionally force the creation of a RPython object. As long as only these
 ## are used, the ob_sval buffer is still modifiable:
-## 
-## PyBytes_AsString / PyString_AsString 
+##
+## PyBytes_AsString / PyString_AsString
 ## PyBytes_AS_STRING / PyString_AS_STRING
 ## PyBytes_AsStringAndSize / PyString_AsStringAndSize
 ## PyBytes_Size / PyString_Size
@@ -40,7 +40,7 @@
 ## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string, 
+##   corresponds to the pypy gc-managed string,
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -105,7 +105,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def bytes_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -6,8 +6,8 @@
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
-    bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj, 
+    bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj,
         make_typedescr, track_reference, create_ref, from_ref, decref,
         Py_IncRef)
 from pypy.module.cpyext.object import _dealloc
@@ -36,7 +36,7 @@
     py_dict.c__tmpkeys = lltype.nullptr(PyObject.TO)
     # Problems: if this dict is a typedict, we may have unbound GetSetProperty
     # functions in the dict. The corresponding PyGetSetDescrObject must be
-    # bound to a class, but the actual w_type will be unavailable later on. 
+    # bound to a class, but the actual w_type will be unavailable later on.
     # Solution: use the w_userdata argument when assigning a PyTypeObject's
     # tp_dict slot to pass a w_type in, and force creation of the pair here
     if not space.is_w(w_userdata, space.gettypefor(GetSetProperty)):
@@ -55,7 +55,7 @@
     w_obj = space.newdict()
     track_reference(space, py_obj, w_obj)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def dict_dealloc(space, py_obj):
     py_dict = rffi.cast(PyDictObject, py_obj)
     decref(space, py_dict.c__tmpkeys)
@@ -287,7 +287,7 @@
     if space not in _frozendict_cache:
         _frozendict_cache[space] = _make_frozendict(space)
     return _frozendict_cache[space]
-        
+
 _frozendict_cache = {}
 def _make_frozendict(space):
     return space.appexec([], '''():
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObjectFields, cpython_struct,
-    CANNOT_FAIL)
+    CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, Py_DecRef, make_ref, from_ref, track_reference,
     make_typedescr, get_typedescr)
@@ -39,7 +39,7 @@
     py_frame.c_f_locals = make_ref(space, frame.get_w_locals())
     rffi.setintfield(py_frame, 'c_f_lineno', frame.getorcreatedebug().f_lineno)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def frame_dealloc(space, py_obj):
     py_frame = rffi.cast(PyFrameObject, py_obj)
     py_code = rffi.cast(PyObject, py_frame.c_f_code)
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from rpython.rlib.unroll import unrolling_iterable
@@ -56,7 +57,7 @@
     assert isinstance(w_obj, Function)
     py_func.c_func_name = make_ref(space, space.wrap(w_obj.name))
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def function_dealloc(space, py_obj):
     py_func = rffi.cast(PyFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_func_name)
@@ -75,7 +76,7 @@
     rffi.setintfield(py_code, 'c_co_flags', co_flags)
     rffi.setintfield(py_code, 'c_co_argcount', w_obj.co_argcount)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def code_dealloc(space, py_obj):
     py_code = rffi.cast(PyCodeObject, py_obj)
     Py_DecRef(space, py_code.c_co_name)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,9 +1,9 @@
-from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers,
-                         Py_ssize_tP, PyObjectFields, cpython_struct,
-                         bootstrap_function, Py_bufferP)
-from pypy.module.cpyext.pyobject import (PyObject, make_ref, as_pyobj, incref,
-             decref, from_ref, make_typedescr)
+from pypy.module.cpyext.api import (
+    cpython_api, Py_buffer, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS,
+    build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct,
+    bootstrap_function, Py_bufferP, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -60,7 +60,7 @@
     """
     raise oefmt(space.w_NotImplementedError, "cannot call this yet")
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def memory_dealloc(space, py_obj):
     mem_obj = rffi.cast(PyMemoryViewObject, py_obj)
     if mem_obj.c_view.c_obj:
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,7 +11,7 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr)
+    PyTypeObjectPtr, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -51,7 +51,7 @@
     py_func.c_m_self = make_ref(space, w_obj.w_self)
     py_func.c_m_module = make_ref(space, w_obj.w_module)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def cfunction_dealloc(space, py_obj):
     py_func = rffi.cast(PyCFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_m_self)
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t,
+    PyVarObject, Py_buffer, size_t, slot_function,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -54,7 +54,7 @@
         w_obj = PyObject_InitVar(space, py_objvar, type, itemcount)
     return py_obj
 
- at cpython_api([PyObject], lltype.Void)
+ at slot_function([PyObject], lltype.Void)
 def PyObject_dealloc(space, obj):
     return _dealloc(space, obj)
 
@@ -511,7 +511,7 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Release the buffer view. This should be called when the buffer is 
+    Release the buffer view. This should be called when the buffer is
     no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -87,7 +87,7 @@
     alloc     : allocate and basic initialization of a raw PyObject
     attach    : Function called to tie a raw structure to a pypy object
     realize   : Function called to create a pypy object from a raw struct
-    dealloc   : a cpython_api(header=None), similar to PyObject_dealloc
+    dealloc   : a @slot_function(), similar to PyObject_dealloc
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
@@ -40,7 +41,7 @@
     rffi.setintfield(py_traceback, 'c_tb_lasti', traceback.lasti)
     rffi.setintfield(py_traceback, 'c_tb_lineno',traceback.get_lineno())
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def traceback_dealloc(space, py_obj):
     py_traceback = rffi.cast(PyTracebackObject, py_obj)
     Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next))
diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py
--- a/pypy/module/cpyext/sliceobject.py
+++ b/pypy/module/cpyext/sliceobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields)
+    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, PyObject, make_ref, make_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
@@ -36,7 +36,7 @@
     py_slice.c_stop = make_ref(space, w_obj.w_stop)
     py_slice.c_step = make_ref(space, w_obj.w_step)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def slice_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -6,7 +6,7 @@
 from rpython.rlib.rarithmetic import widen
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
+    slot_function, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
     pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
@@ -64,7 +64,7 @@
 
 @not_rpython
 def llslot(space, func):
-    return llhelper(func.api_func.functype, func.api_func.get_wrapper(space))
+    return func.api_func.get_llhelper(space)
 
 @register_flow_sc(llslot)
 def sc_llslot(ctx, v_space, v_func):
@@ -521,9 +521,6 @@
 def build_slot_tp_function(space, typedef, name):
     w_type = space.gettypeobject(typedef)
 
-    header = pypy_decl
-    if not (name.startswith('Py') or name.startswith('_Py')):
-        header = None
     handled = False
     # unary functions
     for tp_name, attr in [('tp_as_number.c_nb_int', '__int__'),
@@ -545,7 +542,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject], PyObject, header=header)
+            @slot_function([PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self):
                 return space.call_function(slot_fn, w_self)
@@ -571,7 +568,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg):
                 return space.call_function(slot_fn, w_self, w_arg)
@@ -588,7 +585,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, Py_ssize_t], PyObject, header=header)
+            @slot_function([PyObject, Py_ssize_t], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, arg):
                 return space.call_function(slot_fn, w_self, space.wrap(arg))
@@ -602,7 +599,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
@@ -616,8 +613,8 @@
         if setattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real,
-                     error=-1, header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real,
+                     error=-1)
         @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,))
         def slot_tp_setattro(space, w_self, w_name, w_value):
             if w_value is not None:
@@ -631,7 +628,7 @@
         if getattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject], PyObject)
         @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,))
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
@@ -641,7 +638,7 @@
         if call_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_call(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -654,7 +651,7 @@
         if iternext_fn is None:
             return
 
-        @cpython_api([PyObject], PyObject, header=header)
+        @slot_function([PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_iternext(space, w_self):
             try:
@@ -670,8 +667,7 @@
         if init_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1,
-                     header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_init(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -684,7 +680,7 @@
         if new_fn is None:
             return
 
-        @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
+        @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_new(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -695,8 +691,8 @@
         buff_fn = w_type.getdictvalue(space, '__buffer__')
         if buff_fn is None:
             return
-        @cpython_api([PyObject, Py_bufferP, rffi.INT_real],
-                rffi.INT_real, header=None, error=-1)
+        @slot_function([PyObject, Py_bufferP, rffi.INT_real],
+                rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def buff_w(space, w_self, view, flags):
             args = Arguments(space, [space.newint(flags)])
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,10 +89,10 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
-                == ('Py_ssize_t', 'Py_ssize_t arg0'))
-        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
-                == ('Py_ssize_t *', 'Py_ssize_t *arg0'))
+        assert PyPy_TypedefTest1.api_func.get_c_restype(db) == 'Py_ssize_t'
+        assert PyPy_TypedefTest1.api_func.get_c_args(db) == 'Py_ssize_t arg0'
+        assert PyPy_TypedefTest2.api_func.get_c_restype(db) == 'Py_ssize_t *'
+        assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Py_ssize_t *arg0'
 
         PyPy_TypedefTest1(space, 0)
         ppos = lltype.malloc(api.Py_ssize_tP.TO, 1, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test/test_translate.py
--- a/pypy/module/cpyext/test/test_translate.py
+++ b/pypy/module/cpyext/test/test_translate.py
@@ -1,6 +1,6 @@
 from rpython.translator.c.test.test_genc import compile
 import pypy.module.cpyext.api
-from pypy.module.cpyext.api import cpython_api
+from pypy.module.cpyext.api import slot_function
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.objectmodel import specialize
@@ -19,7 +19,7 @@
 
     @specialize.memo()
     def get_tp_function(space, typedef):
-        @cpython_api([], lltype.Signed, error=-1, header=None)
+        @slot_function([], lltype.Signed, error=-1)
         def slot_tp_function(space):
             return typedef.value
 
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -1,11 +1,11 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import fatalerror_notb
-from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL,
-                                    build_type_checkers, PyVarObjectFields,
-                                    cpython_struct, bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef,
-    make_ref, from_ref, decref, incref, pyobj_has_w_obj,
+from pypy.module.cpyext.api import (
+    cpython_api, Py_ssize_t, build_type_checkers,
+    PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, PyObjectP, make_ref, from_ref, decref, incref,
     track_reference, make_typedescr, get_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
@@ -74,7 +74,7 @@
     if py_tup.c_ob_size < length:
         raise oefmt(space.w_ValueError,
             "tuple_attach called on object with ob_size %d but trying to store %d",
-            py_tup.c_ob_size, length) 
+            py_tup.c_ob_size, length)
     i = 0
     try:
         while i < length:
@@ -113,7 +113,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def tuple_dealloc(space, py_obj):
     """Frees allocated PyTupleObject resources.
     """
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -2,21 +2,20 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rstring import rsplit
 from rpython.rtyper.lltypesystem import rffi, lltype
 
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.typedef import (GetSetProperty, TypeDef,
-        interp_attrproperty, interp_attrproperty, interp2app)
+from pypy.interpreter.typedef import (
+    GetSetProperty, TypeDef, interp_attrproperty, interp2app)
 from pypy.module.__builtin__.abstractinst import abstract_issubclass_w
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
+    PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
     Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
@@ -346,7 +345,7 @@
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
 
- at cpython_api([PyObject, PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
 def tp_new_wrapper(space, self, w_args, w_kwds):
     self_pytype = rffi.cast(PyTypeObjectPtr, self)
     tp_new = self_pytype.c_tp_new
@@ -507,7 +506,7 @@
                    realize=type_realize,
                    dealloc=type_dealloc)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def subtype_dealloc(space, obj):
     pto = obj.c_ob_type
     base = pto
@@ -522,15 +521,13 @@
     # hopefully this does not clash with the memory model assumed in
     # extension modules
 
- at cpython_api([PyObject, Py_ssize_tP], lltype.Signed, header=None,
-             error=CANNOT_FAIL)
+ at slot_function([PyObject, Py_ssize_tP], lltype.Signed, error=CANNOT_FAIL)
 def bf_segcount(space, w_obj, ref):
     if ref:
         ref[0] = space.len_w(w_obj)
     return 1
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def bf_getreadbuffer(space, w_buf, segment, ref):
     from rpython.rlib.buffer import StringBuffer
     if segment != 0:
@@ -543,13 +540,11 @@
     ref[0] = address
     return len(buf)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def bf_getcharbuffer(space, w_buf, segment, ref):
     return bf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def bf_getwritebuffer(space, w_buf, segment, ref):
     if segment != 0:
         raise oefmt(space.w_SystemError,
@@ -558,8 +553,7 @@
     ref[0] = buf.get_raw_address()
     return len(buf)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def str_getreadbuffer(space, w_str, segment, ref):
     from pypy.module.cpyext.bytesobject import PyString_AsString
     if segment != 0:
@@ -571,11 +565,10 @@
     Py_DecRef(space, pyref)
     return space.len_w(w_str)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def unicode_getreadbuffer(space, w_str, segment, ref):
     from pypy.module.cpyext.unicodeobject import (
-                PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
+        PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
     if segment != 0:
         raise oefmt(space.w_SystemError,
                     "accessing non-existent unicode segment")
@@ -585,13 +578,11 @@
     Py_DecRef(space, pyref)
     return PyUnicode_GET_DATA_SIZE(space, w_str)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def str_getcharbuffer(space, w_buf, segment, ref):
     return str_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def buf_getreadbuffer(space, pyref, segment, ref):
     from pypy.module.cpyext.bufferobject import PyBufferObject
     if segment != 0:
@@ -601,8 +592,7 @@
     ref[0] = py_buf.c_b_ptr
     return py_buf.c_b_size
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def buf_getcharbuffer(space, w_buf, segment, ref):
     return buf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
@@ -642,7 +632,7 @@
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def type_dealloc(space, obj):
     from pypy.module.cpyext.object import _dealloc
     obj_pto = rffi.cast(PyTypeObjectPtr, obj)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
     bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
-    CONST_WSTRING)
+    CONST_WSTRING, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -84,7 +84,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def unicode_dealloc(space, py_obj):
     py_unicode = rffi.cast(PyUnicodeObject, py_obj)
     Py_DecRef(space, py_unicode.c_defenc)

From pypy.commits at gmail.com  Tue Jan 10 13:49:09 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 10 Jan 2017 10:49:09 -0800 (PST)
Subject: [pypy-commit] pypy default: "document" merged branch
Message-ID: <58752ca5.92ae1c0a.76308.043a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89479:745ebe24fc0b
Date: 2017-01-10 18:48 +0000
http://bitbucket.org/pypy/pypy/changeset/745ebe24fc0b/

Log:	"document" merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -89,6 +89,7 @@
 is readonly) without pinning it.
 
 .. branch: cpyext-cleanup
+.. branch: api_func-refactor
 
 Refactor cpyext initialisation.
 

From pypy.commits at gmail.com  Tue Jan 10 14:08:01 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 10 Jan 2017 11:08:01 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: hg merge default
Message-ID: <58753111.c3e31c0a.d86b5.d718@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89480:0b2f88ee4865
Date: 2017-01-10 19:07 +0000
http://bitbucket.org/pypy/pypy/changeset/0b2f88ee4865/

Log:	hg merge default

diff too long, truncating to 2000 out of 71624 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -89,5 +89,11 @@
 is readonly) without pinning it.
 
 .. branch: cpyext-cleanup
+.. branch: api_func-refactor
 
 Refactor cpyext initialisation.
+
+.. branch: cpyext-from2
+
+Fix a test failure introduced by strbuf-as-buffer
+
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -323,66 +323,19 @@
             wrapper.c_name = cpyext_namespace.uniquename(self.c_name)
         return wrapper
 
-DEFAULT_HEADER = 'pypy_decl.h'
-def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
-                gil=None, result_borrowed=False, result_is_ll=False):
-    """
-    Declares a function to be exported.
-    - `argtypes`, `restype` are lltypes and describe the function signature.
-    - `error` is the value returned when an applevel exception is raised. The
-      special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual
-      exception into a wrapped SystemError.  Unwrapped exceptions also cause a
-      SytemError.
-    - `header` is the header file to export the function in, Set to None to get
-      a C function pointer, but not exported by the API headers.
-    - set `gil` to "acquire", "release" or "around" to acquire the GIL,
-      release the GIL, or both
-    """
-    if isinstance(restype, lltype.Typedef):
-        real_restype = restype.OF
-    else:
-        real_restype = restype
-
-    if error is _NOT_SPECIFIED:
-        if isinstance(real_restype, lltype.Ptr):
-            error = lltype.nullptr(real_restype.TO)
-        elif real_restype is lltype.Void:
-            error = CANNOT_FAIL
-    if type(error) is int:
-        error = rffi.cast(real_restype, error)
-    expect_integer = (isinstance(real_restype, lltype.Primitive) and
-                      rffi.cast(restype, 0) == 0)
-
-    def decorate(func):
-        func._always_inline_ = 'try'
-        func_name = func.func_name
-        if header is not None:
-            c_name = None
-            if func_name in FUNCTIONS_BY_HEADER[header]:
-                raise ValueError("%s already registered" % func_name)
-        else:
-            c_name = func_name
-        api_function = ApiFunction(argtypes, restype, func, error,
-                                   c_name=c_name, gil=gil,
-                                   result_borrowed=result_borrowed,
-                                   result_is_ll=result_is_ll)
-        func.api_func = api_function
-
-        if error is _NOT_SPECIFIED:
-            raise ValueError("function %s has no return value for exceptions"
-                             % func)
-        names = api_function.argnames
-        types_names_enum_ui = unrolling_iterable(enumerate(
-            zip(api_function.argtypes,
-                [tp_name.startswith("w_") for tp_name in names])))
+    def get_unwrapper(self):
+        names = self.argnames
+        argtypesw = zip(self.argtypes,
+                        [_name.startswith("w_") for _name in self.argnames])
+        types_names_enum_ui = unrolling_iterable(enumerate(argtypesw))
 
         @specialize.ll()
         def unwrapper(space, *args):
-            from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj
+            from pypy.module.cpyext.pyobject import is_pyobj
             from pypy.module.cpyext.pyobject import from_ref, as_pyobj
             newargs = ()
             keepalives = ()
-            assert len(args) == len(api_function.argtypes)
+            assert len(args) == len(self.argtypes)
             for i, (ARG, is_wrapped) in types_names_enum_ui:
                 input_arg = args[i]
                 if is_PyObject(ARG) and not is_wrapped:
@@ -407,31 +360,79 @@
                         arg = from_ref(space, input_arg)
                     else:
                         arg = input_arg
-
-                        ## ZZZ: for is_pyobj:
-                        ## try:
-                        ##     arg = from_ref(space,
-                        ##                rffi.cast(PyObject, input_arg))
-                        ## except TypeError, e:
-                        ##     err = oefmt(space.w_TypeError,
-                        ##                 "could not cast arg to PyObject")
-                        ##     if not catch_exception:
-                        ##         raise err
-                        ##     state = space.fromcache(State)
-                        ##     state.set_exception(err)
-                        ##     if is_PyObject(restype):
-                        ##         return None
-                        ##     else:
-                        ##         return api_function.error_value
                 else:
                     # arg is not declared as PyObject, no magic
                     arg = input_arg
                 newargs += (arg, )
             try:
-                return func(space, *newargs)
+                return self.callable(space, *newargs)
             finally:
                 keepalive_until_here(*keepalives)
+        return unwrapper
 
+    def get_c_restype(self, c_writer):
+        return c_writer.gettype(self.restype).replace('@', '').strip()
+
+    def get_c_args(self, c_writer):
+        args = []
+        for i, argtype in enumerate(self.argtypes):
+            if argtype is CONST_STRING:
+                arg = 'const char *@'
+            elif argtype is CONST_STRINGP:
+                arg = 'const char **@'
+            elif argtype is CONST_WSTRING:
+                arg = 'const wchar_t *@'
+            else:
+                arg = c_writer.gettype(argtype)
+            arg = arg.replace('@', 'arg%d' % (i,)).strip()
+            args.append(arg)
+        args = ', '.join(args) or "void"
+        return args
+
+    def get_api_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "PyAPI_FUNC({restype}) {name}({args});".format(**locals())
+
+    def get_ptr_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "{restype} (*{name})({args});".format(**locals())
+
+    def get_ctypes_impl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        callargs = ', '.join('arg%d' % (i,)
+                            for i in range(len(self.argtypes)))
+        if self.restype is lltype.Void:
+            body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+        else:
+            body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+        return '%s %s(%s)\n%s' % (restype, name, args, body)
+
+
+DEFAULT_HEADER = 'pypy_decl.h'
+def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
+                gil=None, result_borrowed=False, result_is_ll=False):
+    """
+    Declares a function to be exported.
+    - `argtypes`, `restype` are lltypes and describe the function signature.
+    - `error` is the value returned when an applevel exception is raised. The
+      special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual
+      exception into a wrapped SystemError.  Unwrapped exceptions also cause a
+      SytemError.
+    - `header` is the header file to export the function in.
+    - set `gil` to "acquire", "release" or "around" to acquire the GIL,
+      release the GIL, or both
+    """
+    assert header is not None
+    def decorate(func):
+        if func.__name__ in FUNCTIONS_BY_HEADER[header]:
+            raise ValueError("%s already registered" % func.__name__)
+        api_function = _create_api_func(
+            func, argtypes, restype, error, gil=gil,
+            result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+        unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
 
@@ -443,25 +444,64 @@
             try:
                 res = unwrapper(space, *args)
             except OperationError as e:
-                if not hasattr(api_function, "error_value"):
+                if not hasattr(unwrapper.api_func, "error_value"):
                     raise
                 state = space.fromcache(State)
                 state.set_exception(e)
                 if is_PyObject(restype):
                     return None
                 else:
-                    return api_function.error_value
+                    return unwrapper.api_func.error_value
             got_integer = isinstance(res, (int, long, float))
+            if isinstance(restype, lltype.Typedef):
+                real_restype = restype.OF
+            else:
+                real_restype = restype
+            expect_integer = (isinstance(real_restype, lltype.Primitive) and
+                            rffi.cast(restype, 0) == 0)
             assert got_integer == expect_integer, (
                 'got %r not integer' % (res,))
             return res
 
         if header is not None:
-            FUNCTIONS_BY_HEADER[header][func_name] = api_function
-        INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
-        return unwrapper  # used in 'normal' RPython code.
+            FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
+            INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
+        return unwrapper
     return decorate
 
+def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
+    def decorate(func):
+        c_name = func.__name__
+        api_function = _create_api_func(func, argtypes, restype, error, c_name)
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
+
+
+def _create_api_func(
+        func, argtypes, restype, error=_NOT_SPECIFIED, c_name=None,
+        gil=None, result_borrowed=False, result_is_ll=False):
+    if isinstance(restype, lltype.Typedef):
+        real_restype = restype.OF
+    else:
+        real_restype = restype
+
+    if error is _NOT_SPECIFIED:
+        if isinstance(real_restype, lltype.Ptr):
+            error = lltype.nullptr(real_restype.TO)
+        elif real_restype is lltype.Void:
+            error = CANNOT_FAIL
+    if type(error) is int:
+        error = rffi.cast(real_restype, error)
+
+    func._always_inline_ = 'try'
+    return ApiFunction(
+        argtypes, restype, func, error, c_name=c_name, gil=gil,
+        result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+
+
 def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
     if level == 1:
@@ -1016,23 +1056,6 @@
     for func in BOOTSTRAP_FUNCTIONS:
         func(space)
 
-def c_function_signature(db, func):
-    restype = db.gettype(func.restype).replace('@', '').strip()
-    args = []
-    for i, argtype in enumerate(func.argtypes):
-        if argtype is CONST_STRING:
-            arg = 'const char *@'
-        elif argtype is CONST_STRINGP:
-            arg = 'const char **@'
-        elif argtype is CONST_WSTRING:
-            arg = 'const wchar_t *@'
-        else:
-            arg = db.gettype(argtype)
-        arg = arg.replace('@', 'arg%d' % (i,)).strip()
-        args.append(arg)
-    args = ', '.join(args) or "void"
-    return restype, args
-
 #_____________________________________________________
 # Build the bridge DLL, Allow extension DLLs to call
 # back into Pypy space functions
@@ -1052,15 +1075,8 @@
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            restype, args = c_function_signature(db, func)
-            callargs = ', '.join('arg%d' % (i,)
-                                for i in range(len(func.argtypes)))
-            if func.restype is lltype.Void:
-                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
-            else:
-                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
-            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
-            members.append('%s (*%s)(%s);' % (restype, name, args))
+            functions.append(func.get_ctypes_impl(name, db))
+            members.append(func.get_ptr_decl(name, db))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
     struct_declaration_code = """\
@@ -1255,8 +1271,7 @@
         for name, func in sorted(header_functions.iteritems()):
             _name = mangle_name(prefix, name)
             header.append("#define %s %s" % (name, _name))
-            restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
+            header.append(func.get_api_decl(name, db))
 
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
+    cpython_api, Py_ssize_t, cpython_struct, bootstrap_function, slot_function,
     PyObjectFields, PyObject)
 from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.module.array.interp_array import ArrayBuffer
@@ -72,7 +72,7 @@
                 "Don't know how to realize a buffer")
 
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
     if py_buf.c_b_base:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL)
+    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -25,14 +25,14 @@
 ##
 ## In the PyBytesObject returned, the ob_sval buffer may be modified as
 ## long as the freshly allocated PyBytesObject is not "forced" via a call
-## to any of the more sophisticated C-API functions. 
+## to any of the more sophisticated C-API functions.
 ##
 ## Care has been taken in implementing the functions below, so that
-## if they are called with a non-forced PyBytesObject, they will not 
+## if they are called with a non-forced PyBytesObject, they will not
 ## unintentionally force the creation of a RPython object. As long as only these
 ## are used, the ob_sval buffer is still modifiable:
-## 
-## PyBytes_AsString / PyString_AsString 
+##
+## PyBytes_AsString / PyString_AsString
 ## PyBytes_AS_STRING / PyString_AS_STRING
 ## PyBytes_AsStringAndSize / PyString_AsStringAndSize
 ## PyBytes_Size / PyString_Size
@@ -40,7 +40,7 @@
 ## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string, 
+##   corresponds to the pypy gc-managed string,
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -105,7 +105,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def bytes_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -6,8 +6,8 @@
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
-    bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj, 
+    bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj,
         make_typedescr, track_reference, create_ref, from_ref, decref,
         Py_IncRef)
 from pypy.module.cpyext.object import _dealloc
@@ -36,7 +36,7 @@
     py_dict.c__tmpkeys = lltype.nullptr(PyObject.TO)
     # Problems: if this dict is a typedict, we may have unbound GetSetProperty
     # functions in the dict. The corresponding PyGetSetDescrObject must be
-    # bound to a class, but the actual w_type will be unavailable later on. 
+    # bound to a class, but the actual w_type will be unavailable later on.
     # Solution: use the w_userdata argument when assigning a PyTypeObject's
     # tp_dict slot to pass a w_type in, and force creation of the pair here
     if not space.is_w(w_userdata, space.gettypefor(GetSetProperty)):
@@ -55,7 +55,7 @@
     w_obj = space.newdict()
     track_reference(space, py_obj, w_obj)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def dict_dealloc(space, py_obj):
     py_dict = rffi.cast(PyDictObject, py_obj)
     decref(space, py_dict.c__tmpkeys)
@@ -287,7 +287,7 @@
     if space not in _frozendict_cache:
         _frozendict_cache[space] = _make_frozendict(space)
     return _frozendict_cache[space]
-        
+
 _frozendict_cache = {}
 def _make_frozendict(space):
     return space.appexec([], '''():
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObjectFields, cpython_struct,
-    CANNOT_FAIL)
+    CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, Py_DecRef, make_ref, from_ref, track_reference,
     make_typedescr, get_typedescr)
@@ -39,7 +39,7 @@
     py_frame.c_f_locals = make_ref(space, frame.get_w_locals())
     rffi.setintfield(py_frame, 'c_f_lineno', frame.getorcreatedebug().f_lineno)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def frame_dealloc(space, py_obj):
     py_frame = rffi.cast(PyFrameObject, py_obj)
     py_code = rffi.cast(PyObject, py_frame.c_f_code)
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from rpython.rlib.unroll import unrolling_iterable
@@ -56,7 +57,7 @@
     assert isinstance(w_obj, Function)
     py_func.c_func_name = make_ref(space, space.wrap(w_obj.name))
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def function_dealloc(space, py_obj):
     py_func = rffi.cast(PyFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_func_name)
@@ -75,7 +76,7 @@
     rffi.setintfield(py_code, 'c_co_flags', co_flags)
     rffi.setintfield(py_code, 'c_co_argcount', w_obj.co_argcount)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def code_dealloc(space, py_obj):
     py_code = rffi.cast(PyCodeObject, py_obj)
     Py_DecRef(space, py_code.c_co_name)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,9 +1,9 @@
-from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers,
-                         Py_ssize_tP, PyObjectFields, cpython_struct,
-                         bootstrap_function, Py_bufferP)
-from pypy.module.cpyext.pyobject import (PyObject, make_ref, as_pyobj, incref,
-             decref, from_ref, make_typedescr)
+from pypy.module.cpyext.api import (
+    cpython_api, Py_buffer, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS,
+    build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct,
+    bootstrap_function, Py_bufferP, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -35,8 +35,24 @@
     """
     Fills a newly allocated PyMemoryViewObject with the given W_MemoryView object.
     """
+    assert isinstance(w_obj, W_MemoryView)
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
-    py_obj.c_view.c_obj = rffi.cast(PyObject, 0)
+    view = py_obj.c_view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return
+    fill_Py_buffer(space, w_obj.buf, view)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_userdata)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s),
+                                             track_allocation=False))
+        rffi.setintfield(view, 'c_readonly', 1)
 
 def memory_realize(space, py_obj):
     """
@@ -44,7 +60,7 @@
     """
     raise oefmt(space.w_NotImplementedError, "cannot call this yet")
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def memory_dealloc(space, py_obj):
     mem_obj = rffi.cast(PyMemoryViewObject, py_obj)
     if mem_obj.c_view.c_obj:
@@ -88,29 +104,13 @@
     return ret
 
 @cpython_api([PyObject], Py_bufferP, error=CANNOT_FAIL)
-def PyMemoryView_GET_BUFFER(space, w_obj):
+def PyMemoryView_GET_BUFFER(space, pyobj):
     """Return a pointer to the buffer-info structure wrapped by the given
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
-    if not isinstance(w_obj, W_MemoryView):
-        return lltype.nullptr(Py_buffer)
-    py_memobj = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_obj)) # no inc_ref
-    view = py_memobj.c_view
-    ndim = w_obj.buf.getndim()
-    if ndim >= Py_MAX_NDIMS:
-        # XXX warn?
-        return view
-    fill_Py_buffer(space, w_obj.buf, view)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-        #view.c_obj = make_ref(space, w_obj) # NO - this creates a ref cycle!
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
-    except ValueError:
-        w_s = w_obj.descr_tobytes(space)
-        view.c_obj = make_ref(space, w_s)
-        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
-        rffi.setintfield(view, 'c_readonly', 1)
-    return view
+    # XXX move to a c-macro
+    py_memobj = rffi.cast(PyMemoryViewObject, pyobj)
+    return py_memobj.c_view
 
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
@@ -202,9 +202,11 @@
         return (_IsCContiguous(view) or _IsFortranContiguous(view))
     return 0
 
- at cpython_api([PyObject], PyObject)
+ at cpython_api([PyObject], PyObject, result_is_ll=True)
 def PyMemoryView_FromObject(space, w_obj):
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    w_memview = space.call_method(space.builtin, "memoryview", w_obj)
+    py_memview = make_ref(space, w_memview, w_obj)
+    return py_memview
 
 @cpython_api([Py_bufferP], PyObject)
 def PyMemoryView_FromBuffer(space, view):
@@ -212,6 +214,7 @@
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
+    assert view.c_obj
     w_obj = from_ref(space, view.c_obj)
     if isinstance(w_obj, W_MemoryView):
         return w_obj
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,7 +11,7 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr)
+    PyTypeObjectPtr, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -51,7 +51,7 @@
     py_func.c_m_self = make_ref(space, w_obj.w_self)
     py_func.c_m_module = make_ref(space, w_obj.w_module)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def cfunction_dealloc(space, py_obj):
     py_func = rffi.cast(PyCFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_m_self)
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t,
+    PyVarObject, Py_buffer, size_t, slot_function,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -54,7 +54,7 @@
         w_obj = PyObject_InitVar(space, py_objvar, type, itemcount)
     return py_obj
 
- at cpython_api([PyObject], lltype.Void)
+ at slot_function([PyObject], lltype.Void)
 def PyObject_dealloc(space, obj):
     return _dealloc(space, obj)
 
@@ -511,7 +511,7 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Release the buffer view. This should be called when the buffer is 
+    Release the buffer view. This should be called when the buffer is
     no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -87,7 +87,7 @@
     alloc     : allocate and basic initialization of a raw PyObject
     attach    : Function called to tie a raw structure to a pypy object
     realize   : Function called to create a pypy object from a raw struct
-    dealloc   : a cpython_api(header=None), similar to PyObject_dealloc
+    dealloc   : a @slot_function(), similar to PyObject_dealloc
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
@@ -40,7 +41,7 @@
     rffi.setintfield(py_traceback, 'c_tb_lasti', traceback.lasti)
     rffi.setintfield(py_traceback, 'c_tb_lineno',traceback.get_lineno())
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def traceback_dealloc(space, py_obj):
     py_traceback = rffi.cast(PyTracebackObject, py_obj)
     Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next))
diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py
--- a/pypy/module/cpyext/sliceobject.py
+++ b/pypy/module/cpyext/sliceobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields)
+    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, PyObject, make_ref, make_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
@@ -36,7 +36,7 @@
     py_slice.c_stop = make_ref(space, w_obj.w_stop)
     py_slice.c_step = make_ref(space, w_obj.w_step)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def slice_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -6,7 +6,7 @@
 from rpython.rlib.rarithmetic import widen
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
+    slot_function, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
     pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
@@ -64,7 +64,7 @@
 
 @not_rpython
 def llslot(space, func):
-    return llhelper(func.api_func.functype, func.api_func.get_wrapper(space))
+    return func.api_func.get_llhelper(space)
 
 @register_flow_sc(llslot)
 def sc_llslot(ctx, v_space, v_func):
@@ -521,9 +521,6 @@
 def build_slot_tp_function(space, typedef, name):
     w_type = space.gettypeobject(typedef)
 
-    header = pypy_decl
-    if not (name.startswith('Py') or name.startswith('_Py')):
-        header = None
     handled = False
     # unary functions
     for tp_name, attr in [('tp_as_number.c_nb_int', '__int__'),
@@ -545,7 +542,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject], PyObject, header=header)
+            @slot_function([PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self):
                 return space.call_function(slot_fn, w_self)
@@ -571,7 +568,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg):
                 return space.call_function(slot_fn, w_self, w_arg)
@@ -588,7 +585,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, Py_ssize_t], PyObject, header=header)
+            @slot_function([PyObject, Py_ssize_t], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, arg):
                 return space.call_function(slot_fn, w_self, space.wrap(arg))
@@ -602,7 +599,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
@@ -616,8 +613,8 @@
         if setattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real,
-                     error=-1, header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real,
+                     error=-1)
         @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,))
         def slot_tp_setattro(space, w_self, w_name, w_value):
             if w_value is not None:
@@ -631,7 +628,7 @@
         if getattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject], PyObject)
         @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,))
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
@@ -641,7 +638,7 @@
         if call_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_call(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -654,7 +651,7 @@
         if iternext_fn is None:
             return
 
-        @cpython_api([PyObject], PyObject, header=header)
+        @slot_function([PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_iternext(space, w_self):
             try:
@@ -670,8 +667,7 @@
         if init_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1,
-                     header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_init(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -684,7 +680,7 @@
         if new_fn is None:
             return
 
-        @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
+        @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_new(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -695,8 +691,8 @@
         buff_fn = w_type.getdictvalue(space, '__buffer__')
         if buff_fn is None:
             return
-        @cpython_api([PyObject, Py_bufferP, rffi.INT_real],
-                rffi.INT_real, header=None, error=-1)
+        @slot_function([PyObject, Py_bufferP, rffi.INT_real],
+                rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def buff_w(space, w_self, view, flags):
             args = Arguments(space, [space.newint(flags)])
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,10 +89,10 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
-                == ('Signed', 'Signed arg0'))
-        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
-                == ('Signed *', 'Signed *arg0'))
+        assert PyPy_TypedefTest1.api_func.get_c_restype(db) == 'Signed'
+        assert PyPy_TypedefTest1.api_func.get_c_args(db) == 'Signed arg0'
+        assert PyPy_TypedefTest2.api_func.get_c_restype(db) == 'Signed *'
+        assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Signed *arg0'
 
         PyPy_TypedefTest1(space, 0)
         ppos = lltype.malloc(api.Py_ssize_tP.TO, 1, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -4,6 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
+from pypy.module.cpyext.pyobject import from_ref
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
@@ -11,7 +12,7 @@
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
-        w_view = api.PyMemoryView_FromObject(w_hello)
+        w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
         assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
@@ -19,7 +20,7 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_memoryview = from_ref(space, api.PyMemoryView_FromObject(w_buf))
         view = api.PyMemoryView_GET_BUFFER(w_memoryview)
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)
diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test/test_translate.py
--- a/pypy/module/cpyext/test/test_translate.py
+++ b/pypy/module/cpyext/test/test_translate.py
@@ -1,6 +1,6 @@
 from rpython.translator.c.test.test_genc import compile
 import pypy.module.cpyext.api
-from pypy.module.cpyext.api import cpython_api
+from pypy.module.cpyext.api import slot_function
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.objectmodel import specialize
@@ -19,7 +19,7 @@
 
     @specialize.memo()
     def get_tp_function(space, typedef):
-        @cpython_api([], lltype.Signed, error=-1, header=None)
+        @slot_function([], lltype.Signed, error=-1)
         def slot_tp_function(space):
             return typedef.value
 
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -1,11 +1,11 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import fatalerror_notb
-from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL,
-                                    build_type_checkers, PyVarObjectFields,
-                                    cpython_struct, bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef,
-    make_ref, from_ref, decref, incref, pyobj_has_w_obj,
+from pypy.module.cpyext.api import (
+    cpython_api, Py_ssize_t, build_type_checkers,
+    PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, PyObjectP, make_ref, from_ref, decref, incref,
     track_reference, make_typedescr, get_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
@@ -74,7 +74,7 @@
     if py_tup.c_ob_size < length:
         raise oefmt(space.w_ValueError,
             "tuple_attach called on object with ob_size %d but trying to store %d",
-            py_tup.c_ob_size, length) 
+            py_tup.c_ob_size, length)
     i = 0
     try:
         while i < length:
@@ -113,7 +113,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def tuple_dealloc(space, py_obj):
     """Frees allocated PyTupleObject resources.
     """
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -2,21 +2,20 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rstring import rsplit
 from rpython.rtyper.lltypesystem import rffi, lltype
 
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.typedef import (GetSetProperty, TypeDef,
-        interp_attrproperty, interp_attrproperty, interp2app)
+from pypy.interpreter.typedef import (
+    GetSetProperty, TypeDef, interp_attrproperty, interp2app)
 from pypy.module.__builtin__.abstractinst import abstract_issubclass_w
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
+    PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
     Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
@@ -346,7 +345,7 @@
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
 
- at cpython_api([PyObject, PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
 def tp_new_wrapper(space, self, w_args, w_kwds):
     self_pytype = rffi.cast(PyTypeObjectPtr, self)
     tp_new = self_pytype.c_tp_new
@@ -507,7 +506,7 @@
                    realize=type_realize,
                    dealloc=type_dealloc)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def subtype_dealloc(space, obj):
     pto = obj.c_ob_type
     base = pto
@@ -522,15 +521,13 @@
     # hopefully this does not clash with the memory model assumed in
     # extension modules
 
- at cpython_api([PyObject, Py_ssize_tP], lltype.Signed, header=None,
-             error=CANNOT_FAIL)
+ at slot_function([PyObject, Py_ssize_tP], lltype.Signed, error=CANNOT_FAIL)
 def bf_segcount(space, w_obj, ref):
     if ref:
         ref[0] = space.len_w(w_obj)
     return 1
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def bf_getreadbuffer(space, w_buf, segment, ref):
     from rpython.rlib.buffer import StringBuffer
     if segment != 0:
@@ -543,13 +540,11 @@
     ref[0] = address
     return len(buf)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def bf_getcharbuffer(space, w_buf, segment, ref):
     return bf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def bf_getwritebuffer(space, w_buf, segment, ref):
     if segment != 0:
         raise oefmt(space.w_SystemError,
@@ -558,8 +553,7 @@
     ref[0] = buf.get_raw_address()
     return len(buf)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def str_getreadbuffer(space, w_str, segment, ref):
     from pypy.module.cpyext.bytesobject import PyString_AsString
     if segment != 0:
@@ -571,11 +565,10 @@
     Py_DecRef(space, pyref)
     return space.len_w(w_str)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def unicode_getreadbuffer(space, w_str, segment, ref):
     from pypy.module.cpyext.unicodeobject import (
-                PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
+        PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
     if segment != 0:
         raise oefmt(space.w_SystemError,
                     "accessing non-existent unicode segment")
@@ -585,13 +578,11 @@
     Py_DecRef(space, pyref)
     return PyUnicode_GET_DATA_SIZE(space, w_str)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def str_getcharbuffer(space, w_buf, segment, ref):
     return str_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def buf_getreadbuffer(space, pyref, segment, ref):
     from pypy.module.cpyext.bufferobject import PyBufferObject
     if segment != 0:
@@ -601,8 +592,7 @@
     ref[0] = py_buf.c_b_ptr
     return py_buf.c_b_size
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def buf_getcharbuffer(space, w_buf, segment, ref):
     return buf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
@@ -642,7 +632,7 @@
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def type_dealloc(space, obj):
     from pypy.module.cpyext.object import _dealloc
     obj_pto = rffi.cast(PyTypeObjectPtr, obj)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
     bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
-    CONST_WSTRING)
+    CONST_WSTRING, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -84,7 +84,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def unicode_dealloc(space, py_obj):
     py_unicode = rffi.cast(PyUnicodeObject, py_obj)
     Py_DecRef(space, py_unicode.c_defenc)
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -548,6 +548,10 @@
 
         sub = self._op_val(space, w_old)
         by = self._op_val(space, w_new)
+        # the following two lines are for being bug-to-bug compatible
+        # with CPython: see issue #2448
+        if count >= 0 and len(input) == 0:
+            return self._empty()
         try:
             res = replace(input, sub, by, count)
         except OverflowError:
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -809,6 +809,16 @@
         s = b"a" * (2**16)
         raises(OverflowError, s.replace, b"", s)
 
+    def test_replace_issue2448(self):
+        # CPython's replace() method has a bug that makes
+        #   ''.replace('', 'x')  gives a different answer than
+        #   ''.replace('', 'x', 1000).  This is the case in all
+        # known versions, at least until 2.7.13.  Some people
+        # call that a feature on the CPython issue report and
+        # the discussion dies out, so it might never be fixed.
+        assert ''.replace('', 'x') == 'x'
+        assert ''.replace('', 'x', 1000) == ''
+
     def test_getslice(self):
         assert "foobar".__getslice__(4, 4321) == "ar"
         s = b"abc"
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -56,7 +56,7 @@
         assert u"abc" != memoryview("abc")
 
     def test_pypy_raw_address_base(self):
-        a = memoryview("foobar")._pypy_raw_address()
+        a = memoryview(b"foobar")._pypy_raw_address()
         assert a != 0
-        b = memoryview(bytearray("foobar"))._pypy_raw_address()
+        b = memoryview(bytearray(b"foobar"))._pypy_raw_address()
         assert b != 0
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -210,7 +210,6 @@
     return rlist.ll_pop(rlist.dum_checkidx, l, index)
 _ll_2_list_append = rlist.ll_append
 _ll_2_list_extend = rlist.ll_extend
-_ll_3_list_insert = rlist.ll_insert_nonneg
 _ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly
 _ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop
 _ll_2_list_inplace_mul = rlist.ll_inplace_mul
diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -212,6 +212,8 @@
                 s += lst[0]
                 lst.pop()
                 lst.append(1)
+                lst.insert(0, 5)
+                lst.insert(1, 6)
                 s *= lst.pop()
             return s
         res = self.meta_interp(f, [15], listops=True)
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -51,6 +51,7 @@
     voidpf = rffi_platform.SimpleType('voidpf', rffi.VOIDP)
 
     ZLIB_VERSION = rffi_platform.DefinedConstantString('ZLIB_VERSION')
+    ZLIB_VERNUM = rffi_platform.DefinedConstantInteger('ZLIB_VERNUM')
 
 for _name in constantnames:
     setattr(SimpleCConfig, _name, rffi_platform.ConstantInteger(_name))
@@ -63,6 +64,7 @@
 Bytefp = lltype.Ptr(lltype.Array(Bytef, hints={'nolength': True}))
 
 ZLIB_VERSION = config['ZLIB_VERSION']
+ZLIB_VERNUM = config['ZLIB_VERNUM']
 
 for _name in constantnames:
     globals()[_name] = config[_name]
@@ -261,7 +263,7 @@
 
 def deflateInit(level=Z_DEFAULT_COMPRESSION, method=Z_DEFLATED,
                 wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL,
-                strategy=Z_DEFAULT_STRATEGY):
+                strategy=Z_DEFAULT_STRATEGY, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     compress data.
@@ -270,6 +272,12 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
     if err == Z_OK:
+        if zdict is not None:
+            try:
+                deflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:
@@ -290,7 +298,7 @@
     lltype.free(stream, flavor='raw')
 
 
-def inflateInit(wbits=MAX_WBITS):
+def inflateInit(wbits=MAX_WBITS, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     decompress data.
@@ -299,6 +307,17 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _inflateInit2(stream, wbits)
     if err == Z_OK:
+        if zdict is not None and wbits < 0:
+            try:
+                if ZLIB_VERNUM is None or ZLIB_VERNUM < 0x1221:
+                    raise RZlibError("zlib version %s does not allow raw "
+                                     "inflate with dictionary" %
+                                       ZLIB_VERSION if ZLIB_VERSION is not None
+                                       else "<unknown>")
+                inflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:
diff --git a/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
@@ -0,0 +1,1414 @@
+# CaseFolding-8.0.0.txt
+# Date: 2015-01-13, 18:16:36 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#223;e" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding 
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+# Property: Case_Folding
+
+#  All code points not explicitly listed for Case_Folding
+#  have the value C for the status field, and the code point itself for the mapping field.
+
+# =================================================================
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE
+023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE
+023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR
+023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
+0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP
+0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE
+0244; C; 0289; # LATIN CAPITAL LETTER U BAR
+0245; C; 028C; # LATIN CAPITAL LETTER TURNED V
+0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE
+0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE
+024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
+024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE
+024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0370; C; 0371; # GREEK CAPITAL LETTER HETA
+0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI
+0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
+037F; C; 03F3; # GREEK CAPITAL LETTER YOT
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+03F7; C; 03F8; # GREEK CAPITAL LETTER SHO
+03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL
+03FA; C; 03FB; # GREEK CAPITAL LETTER SAN
+03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
+03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
+03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
+04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK
+04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE
+0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK
+0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA
+0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA
+0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE
+051A; C; 051B; # CYRILLIC CAPITAL LETTER QA
+051C; C; 051D; # CYRILLIC CAPITAL LETTER WE
+051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA
+0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
+0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
+0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER
+0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER
+0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK
+052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE
+052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE
+052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN
+10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN
+10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN
+10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON
+10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN
+10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN
+10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN
+10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN
+10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN
+10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN
+10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS
+10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN
+10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR
+10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON
+10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR
+10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR
+10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE
+10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN
+10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR
+10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN
+10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR
+10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR
+10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN
+10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR
+10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN
+10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN
+10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN
+10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL
+10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL
+10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR
+10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN
+10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN
+10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE
+10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE
+10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE
+10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE
+10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR
+10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE
+10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN
+10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN
+13F8; C; 13F0; # CHEROKEE SMALL LETTER YE
+13F9; C; 13F1; # CHEROKEE SMALL LETTER YI
+13FA; C; 13F2; # CHEROKEE SMALL LETTER YO
+13FB; C; 13F3; # CHEROKEE SMALL LETTER YU
+13FC; C; 13F4; # CHEROKEE SMALL LETTER YV
+13FD; C; 13F5; # CHEROKEE SMALL LETTER MV
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S
+1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL
+1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V
+1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA

From pypy.commits at gmail.com  Tue Jan 10 14:47:44 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 10 Jan 2017 11:47:44 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Parse function declarations
Message-ID: <58753a60.42061c0a.18fe5.2a8a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89481:5ce867e5a994
Date: 2017-01-10 19:46 +0000
http://bitbucket.org/pypy/pypy/changeset/5ce867e5a994/

Log:	Parse function declarations

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -783,6 +783,16 @@
             result = result.TYPE
         return result
 
+    def parse_func(self, cdecl):
+        cdecl = cdecl.strip()
+        if cdecl[-1] != ';':
+            cdecl += ';'
+        ast, _, _ = self.ctx._parse(cdecl)
+        decl = ast.ext[-1]
+        tp, quals = self.ctx._get_type_and_quals(decl.type, name=decl.name)
+        FUNCP = self.convert_type(tp.as_function_pointer())
+        return decl.name, FUNCP.TO
+
 
 def parse_source(source, includes=None, eci=None, configure_now=False):
     ctx = Parser()
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -163,3 +163,28 @@
     res = parse_source(decl, eci=eci, configure_now=True)
     assert res.gettype('Py_ssize_t') == rffi.SSIZE_T
     assert res.gettype('TestFloatObject *').TO.c_ob_refcnt == rffi.SSIZE_T
+
+def test_parse_funcdecl(tmpdir):
+    decl = """
+    typedef ssize_t Py_ssize_t;
+
+    #define PyObject_HEAD  \
+        Py_ssize_t ob_refcnt;        \
+        Py_ssize_t ob_pypy_link;     \
+
+    typedef struct {
+        PyObject_HEAD
+        double ob_fval;
+    } TestFloatObject;
+
+    typedef TestFloatObject* (*func_t)(int, int);
+    """
+    hdr = tmpdir / 'header.h'
+    hdr.write(decl)
+    eci = ExternalCompilationInfo(
+        include_dirs=[str(tmpdir)], includes=['sys/types.h', 'header.h'])
+    res = parse_source(decl, eci=eci, configure_now=True)
+    name, FUNC = res.parse_func("func_t some_func(TestFloatObject*)")
+    assert name == 'some_func'
+    assert FUNC.RESULT == res.gettype('func_t')
+    assert FUNC.ARGS == (res.gettype('TestFloatObject *'),)

From pypy.commits at gmail.com  Tue Jan 10 16:27:53 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 10 Jan 2017 13:27:53 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-FromBuffer: fix translation (ndim,
 readonly need widen() )
Message-ID: <587551d9.878f1c0a.d2175.0626@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-FromBuffer
Changeset: r89482:629285e34cf7
Date: 2017-01-10 22:51 +0200
http://bitbucket.org/pypy/pypy/changeset/629285e34cf7/

Log:	fix translation (ndim, readonly need widen() )

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -61,19 +61,20 @@
     from pypy.module.cpyext.slotdefs import CPyBuffer, fq
     py_mem = rffi.cast(PyMemoryViewObject, obj)
     view = py_mem.c_view
+    ndim = widen(view.c_ndim)
     shape = None
     if view.c_shape:
-        shape = [view.c_shape[i] for i in range(view.c_ndim)]
+        shape = [view.c_shape[i] for i in range(ndim)]
     strides = None
     if view.c_strides:
-        strides = [view.c_strides[i] for i in range(view.c_ndim)]
+        strides = [view.c_strides[i] for i in range(ndim)]
     format = 'B'
     if view.c_format:
         format = rffi.charp2str(view.c_format)
     buf = CPyBuffer(space, view.c_buf, view.c_len, from_ref(space, view.c_obj),
                     format=format, shape=shape, strides=strides,
-                    ndim=view.c_ndim, itemsize=view.c_itemsize,
-                    readonly=view.c_readonly)
+                    ndim=ndim, itemsize=view.c_itemsize,
+                    readonly=widen(view.c_readonly))
     # Ensure view.c_buf is released upon object finalization
     fq.register_finalizer(buf)
     # Allow subclassing W_MemeoryView

From pypy.commits at gmail.com  Tue Jan 10 17:38:17 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 10 Jan 2017 14:38:17 -0800 (PST)
Subject: [pypy-commit] pypy default: merge cpyext-FromBuffer which fixes ref
 leak in PyMemoryView_FromBuffer
Message-ID: <58756259.ce841c0a.f1f57.63c9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r89484:28d4f9418183
Date: 2017-01-10 23:40 +0200
http://bitbucket.org/pypy/pypy/changeset/28d4f9418183/

Log:	merge cpyext-FromBuffer which fixes ref leak in
	PyMemoryView_FromBuffer

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -124,6 +124,7 @@
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -3,7 +3,8 @@
     build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct,
     bootstrap_function, Py_bufferP, slot_function)
 from pypy.module.cpyext.pyobject import (
-    PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr)
+    PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr,
+    get_typedescr, track_reference)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -28,7 +29,7 @@
                    basestruct=PyMemoryViewObject.TO,
                    attach=memory_attach,
                    dealloc=memory_dealloc,
-                   #realize=memory_realize,
+                   realize=memory_realize,
                   )
 
 def memory_attach(space, py_obj, w_obj, w_userdata=None):
@@ -54,11 +55,35 @@
                                              track_allocation=False))
         rffi.setintfield(view, 'c_readonly', 1)
 
-def memory_realize(space, py_obj):
+def memory_realize(space, obj):
     """
     Creates the memory object in the interpreter
     """
-    raise oefmt(space.w_NotImplementedError, "cannot call this yet")
+    from pypy.module.cpyext.slotdefs import CPyBuffer, fq
+    py_mem = rffi.cast(PyMemoryViewObject, obj)
+    view = py_mem.c_view
+    ndim = widen(view.c_ndim)
+    shape = None
+    if view.c_shape:
+        shape = [view.c_shape[i] for i in range(ndim)]
+    strides = None
+    if view.c_strides:
+        strides = [view.c_strides[i] for i in range(ndim)]
+    format = 'B'
+    if view.c_format:
+        format = rffi.charp2str(view.c_format)
+    buf = CPyBuffer(space, view.c_buf, view.c_len, from_ref(space, view.c_obj),
+                    format=format, shape=shape, strides=strides,
+                    ndim=ndim, itemsize=view.c_itemsize,
+                    readonly=widen(view.c_readonly))
+    # Ensure view.c_buf is released upon object finalization
+    fq.register_finalizer(buf)
+    # Allow subclassing W_MemeoryView
+    w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
+    w_obj = space.allocate_instance(W_MemoryView, w_type)
+    w_obj.__init__(buf)
+    track_reference(space, obj, w_obj)
+    return w_obj
 
 @slot_function([PyObject], lltype.Void)
 def memory_dealloc(space, py_obj):
@@ -208,17 +233,41 @@
     py_memview = make_ref(space, w_memview, w_obj)
     return py_memview
 
- at cpython_api([Py_bufferP], PyObject)
+ at cpython_api([Py_bufferP], PyObject, result_is_ll=True)
 def PyMemoryView_FromBuffer(space, view):
     """Create a memoryview object wrapping the given buffer-info structure view.
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
-    assert view.c_obj
-    w_obj = from_ref(space, view.c_obj)
-    if isinstance(w_obj, W_MemoryView):
-        return w_obj
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    # XXX this should allocate a PyMemoryViewObject and
+    # copy view into obj.c_view, without creating a new view.c_obj
+    typedescr = get_typedescr(W_MemoryView.typedef)
+    py_obj = typedescr.allocate(space, space.w_memoryview)
+    py_mem = rffi.cast(PyMemoryViewObject, py_obj)
+    mview = py_mem.c_view
+    mview.c_buf = view.c_buf
+    mview.c_obj = view.c_obj
+    mview.c_len = view.c_len
+    mview.c_itemsize = view.c_itemsize
+    mview.c_readonly = view.c_readonly
+    mview.c_ndim = view.c_ndim
+    mview.c_format = view.c_format
+    if view.c_strides == rffi.cast(Py_ssize_tP, view.c__strides):
+        py_mem.c_view.c_strides = rffi.cast(Py_ssize_tP, py_mem.c_view.c__strides)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_strides[i] = view.c_strides[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_strides = view.c_strides
+    if view.c_shape == rffi.cast(Py_ssize_tP, view.c__shape):
+        py_mem.c_view.c_shape = rffi.cast(Py_ssize_tP, py_mem.c_view.c__shape)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_shape[i] = view.c_shape[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_shape = view.c_shape
+    # XXX ignore suboffsets?
+    return py_obj
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -2,6 +2,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
     PyVarObject, Py_buffer, size_t, slot_function,
+    PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -486,22 +487,28 @@
     Fills in a buffer-info structure correctly for an exporter that can only
     share a contiguous chunk of memory of "unsigned bytes" of the given
     length. Returns 0 on success and -1 (with raising an error) on error.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
     """
     if flags & PyBUF_WRITABLE and readonly:
         raise oefmt(space.w_ValueError, "Object is not writable")
     view.c_buf = buf
     view.c_len = length
     view.c_obj = obj
-    Py_IncRef(space, obj)
+    if obj:
+        Py_IncRef(space, obj)
     view.c_itemsize = 1
     rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
+    rffi.setintfield(view, 'c_ndim', 1)
     view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
     view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
     view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
 
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -324,7 +324,7 @@
 
     def __init__(self, space, ptr, size, w_obj, format='B', shape=None,
                 strides=None, ndim=1, itemsize=1, readonly=True,
-                releasebuffer=None):
+                releasebufferproc=rffi.cast(rffi.VOIDP, 0)):
         self.space = space
         self.ptr = ptr
         self.size = size
@@ -342,7 +342,7 @@
         self.ndim = ndim
         self.itemsize = itemsize
         self.readonly = readonly
-        self.releasebufferproc = releasebuffer
+        self.releasebufferproc = releasebufferproc
 
     def releasebuffer(self):
         if self.pyobj:
@@ -360,7 +360,10 @@
                 for i in range(self.ndim):
                     pybuf.c_shape[i] = self.shape[i]
                     pybuf.c_strides[i] = self.strides[i]
-                pybuf.c_format = rffi.str2charp(self.format)
+                if self.format:
+                    pybuf.c_format = rffi.str2charp(self.format)
+                else:
+                    pybuf.c_format = rffi.str2charp("B")
                 generic_cpy_call(self.space, func_target, self.pyobj, pybuf)
             self.releasebufferproc = rffi.cast(rffi.VOIDP, 0)
 
@@ -407,9 +410,9 @@
     func_target = rffi.cast(readbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
@@ -417,7 +420,7 @@
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -426,16 +429,16 @@
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
     decref(space, py_obj)
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -443,9 +446,9 @@
     func_target = rffi.cast(getbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(Py_buffer) as pybuf:
         _flags = 0
@@ -471,7 +474,7 @@
                             ndim=ndim, shape=shape, strides=strides,
                             itemsize=pybuf.c_itemsize,
                             readonly=widen(pybuf.c_readonly),
-                            releasebuffer = releasebuffer)
+                            releasebufferproc = rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -30,7 +30,7 @@
         assert view.c_len == 5
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
-        w_mv = api.PyMemoryView_FromBuffer(view)
+        w_mv = from_ref(space, api.PyMemoryView_FromBuffer(view))
         for f in ('format', 'itemsize', 'ndim', 'readonly', 
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)
@@ -44,6 +44,7 @@
                 ("fillinfo", "METH_VARARGS",
                  """
                  Py_buffer buf;
+                 PyObject * ret = NULL;
                  PyObject *str = PyBytes_FromString("hello, world.");
                  if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
                                        0, 0)) {
@@ -55,7 +56,14 @@
                   */
                  Py_DECREF(str);
 
-                 return PyMemoryView_FromBuffer(&buf);
+                 ret = PyMemoryView_FromBuffer(&buf);
+                 if (((PyMemoryViewObject*)ret)->view.obj != buf.obj)
+                 {
+                    PyErr_SetString(PyExc_ValueError, "leaked ref");
+                    Py_DECREF(ret);
+                    return NULL;
+                 }
+                 return ret;
                  """)])
         result = module.fillinfo()
         assert b"hello, world." == result

From pypy.commits at gmail.com  Tue Jan 10 17:38:24 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 10 Jan 2017 14:38:24 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: cpython_api ... header=None =>
 slot_function
Message-ID: <58756260.6170c20a.fe744.f2b7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89487:748bf3d6f4c5
Date: 2017-01-11 00:37 +0200
http://bitbucket.org/pypy/pypy/changeset/748bf3d6f4c5/

Log:	cpython_api ... header=None => slot_function

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -562,7 +562,7 @@
             slot_fn = w_type.getdictvalue(space, attr)
             if slot_fn is None:
                 return
-            @cpython_api([PyObject], lltype.Signed, header=header, error=-1)
+            @slot_function([PyObject], lltype.Signed, header=header, error=-1)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_obj):
                 return space.int_w(space.call_function(slot_fn, w_obj))
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -12,23 +12,23 @@
 
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.module.cpyext.api import cpython_api, PyObject, Py_ssize_t
+from pypy.module.cpyext.api import slot_function, PyObject, Py_ssize_t
 from pypy.module.cpyext.api import PyTypeObjectPtr
 from rpython.rtyper.lltypesystem import rffi, lltype
 
- at cpython_api([PyObject], Py_ssize_t, error=-1, header=None)
+ at slot_function([PyObject], Py_ssize_t, error=-1)
 def slot_sq_length(space, w_obj):
     return space.int_w(space.len(w_obj))
 
- at cpython_api([PyObject], lltype.Signed, header=None, error=-1)
+ at slot_function([PyObject], lltype.Signed, error=-1)
 def slot_tp_hash(space, w_obj):
     return space.int_w(space.hash(w_obj))
 
- at cpython_api([PyObject, Py_ssize_t], PyObject, header=None)
+ at slot_function([PyObject, Py_ssize_t], PyObject)
 def slot_sq_item(space, w_obj, index):
     return space.getitem(w_obj, space.wrap(index))
 
- at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
 def slot_tp_new(space, w_type, w_args, w_kwds):
     # XXX problem - we need to find the actual __new__ function to call.
     #     but we have no 'self' argument. Theoretically, self will be
@@ -51,61 +51,61 @@
 
 # unary functions
 
- at cpython_api([PyObject], PyObject, header=None)
+ at slot_function([PyObject], PyObject)
 def slot_tp_str(space, w_obj):
     return space.str(w_obj)
 
- at cpython_api([PyObject], PyObject, header=None)
+ at slot_function([PyObject], PyObject)
 def slot_tp_repr(space, w_obj):
     return space.repr(w_obj)
 
 #binary functions
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_nb_add(space, w_obj1, w_obj2):
     return space.add(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_nb_subtract(space, w_obj1, w_obj2):
     return space.sub(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_nb_multiply(space, w_obj1, w_obj2):
     return space.mul(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_nb_divide(space, w_obj1, w_obj2):
     return space.div(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_nb_inplace_add(space, w_obj1, w_obj2):
     return space.add(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_nb_inplace_subtract(space, w_obj1, w_obj2):
     return space.sub(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_nb_inplace_multiply(space, w_obj1, w_obj2):
     return space.mul(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_nb_inplace_divide(space, w_obj1, w_obj2):
     return space.div(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_sq_concat(space, w_obj1, w_obj2):
     return space.add(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_sq_inplace_concat(space, w_obj1, w_obj2):
     return space.add(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_mp_subscript(space, w_obj1, w_obj2):
     return space.getitem(w_obj1, w_obj2)
 
- at cpython_api([PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject], PyObject)
 def slot_tp_getattr(space, w_obj1, w_obj2):
     return space.getattr(w_obj1, w_obj2)
 

From pypy.commits at gmail.com  Tue Jan 10 17:38:15 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 10 Jan 2017 14:38:15 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-FromBuffer: close branch to be merged
Message-ID: <58756257.d5091c0a.e35e5.5bbe@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-FromBuffer
Changeset: r89483:483f0ddb128f
Date: 2017-01-10 23:37 +0200
http://bitbucket.org/pypy/pypy/changeset/483f0ddb128f/

Log:	close branch to be merged


From pypy.commits at gmail.com  Tue Jan 10 17:38:19 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 10 Jan 2017 14:38:19 -0800 (PST)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <5875625b.2105c30a.e40d0.e926@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r89485:9bfc055b22fc
Date: 2017-01-10 23:44 +0200
http://bitbucket.org/pypy/pypy/changeset/9bfc055b22fc/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -97,3 +97,7 @@
 
 Fix a test failure introduced by strbuf-as-buffer
 
+.. branch: cpyext-FromBuffer
+
+Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to
+the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak.

From pypy.commits at gmail.com  Tue Jan 10 17:38:22 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 10 Jan 2017 14:38:22 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <5875625e.4306c20a.ae7d6.ee06@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89486:99f094da36b2
Date: 2017-01-10 23:45 +0200
http://bitbucket.org/pypy/pypy/changeset/99f094da36b2/

Log:	merge default into branch

diff too long, truncating to 2000 out of 71830 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -89,5 +89,15 @@
 is readonly) without pinning it.
 
 .. branch: cpyext-cleanup
+.. branch: api_func-refactor
 
 Refactor cpyext initialisation.
+
+.. branch: cpyext-from2
+
+Fix a test failure introduced by strbuf-as-buffer
+
+.. branch: cpyext-FromBuffer
+
+Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to
+the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -125,6 +125,7 @@
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -330,66 +331,19 @@
             wrapper.c_name = cpyext_namespace.uniquename(self.c_name)
         return wrapper
 
-DEFAULT_HEADER = 'pypy_decl.h'
-def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
-                gil=None, result_borrowed=False, result_is_ll=False):
-    """
-    Declares a function to be exported.
-    - `argtypes`, `restype` are lltypes and describe the function signature.
-    - `error` is the value returned when an applevel exception is raised. The
-      special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual
-      exception into a wrapped SystemError.  Unwrapped exceptions also cause a
-      SytemError.
-    - `header` is the header file to export the function in, Set to None to get
-      a C function pointer, but not exported by the API headers.
-    - set `gil` to "acquire", "release" or "around" to acquire the GIL,
-      release the GIL, or both
-    """
-    if isinstance(restype, lltype.Typedef):
-        real_restype = restype.OF
-    else:
-        real_restype = restype
-
-    if error is _NOT_SPECIFIED:
-        if isinstance(real_restype, lltype.Ptr):
-            error = lltype.nullptr(real_restype.TO)
-        elif real_restype is lltype.Void:
-            error = CANNOT_FAIL
-    if type(error) is int:
-        error = rffi.cast(real_restype, error)
-    expect_integer = (isinstance(real_restype, lltype.Primitive) and
-                      rffi.cast(restype, 0) == 0)
-
-    def decorate(func):
-        func._always_inline_ = 'try'
-        func_name = func.func_name
-        if header is not None:
-            c_name = None
-            if func_name in FUNCTIONS_BY_HEADER[header]:
-                raise ValueError("%s already registered" % func_name)
-        else:
-            c_name = func_name
-        api_function = ApiFunction(argtypes, restype, func, error,
-                                   c_name=c_name, gil=gil,
-                                   result_borrowed=result_borrowed,
-                                   result_is_ll=result_is_ll)
-        func.api_func = api_function
-
-        if error is _NOT_SPECIFIED:
-            raise ValueError("function %s has no return value for exceptions"
-                             % func)
-        names = api_function.argnames
-        types_names_enum_ui = unrolling_iterable(enumerate(
-            zip(api_function.argtypes,
-                [tp_name.startswith("w_") for tp_name in names])))
+    def get_unwrapper(self):
+        names = self.argnames
+        argtypesw = zip(self.argtypes,
+                        [_name.startswith("w_") for _name in self.argnames])
+        types_names_enum_ui = unrolling_iterable(enumerate(argtypesw))
 
         @specialize.ll()
         def unwrapper(space, *args):
-            from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj
+            from pypy.module.cpyext.pyobject import is_pyobj
             from pypy.module.cpyext.pyobject import from_ref, as_pyobj
             newargs = ()
             keepalives = ()
-            assert len(args) == len(api_function.argtypes)
+            assert len(args) == len(self.argtypes)
             for i, (ARG, is_wrapped) in types_names_enum_ui:
                 input_arg = args[i]
                 if is_PyObject(ARG) and not is_wrapped:
@@ -414,31 +368,79 @@
                         arg = from_ref(space, input_arg)
                     else:
                         arg = input_arg
-
-                        ## ZZZ: for is_pyobj:
-                        ## try:
-                        ##     arg = from_ref(space,
-                        ##                rffi.cast(PyObject, input_arg))
-                        ## except TypeError, e:
-                        ##     err = oefmt(space.w_TypeError,
-                        ##                 "could not cast arg to PyObject")
-                        ##     if not catch_exception:
-                        ##         raise err
-                        ##     state = space.fromcache(State)
-                        ##     state.set_exception(err)
-                        ##     if is_PyObject(restype):
-                        ##         return None
-                        ##     else:
-                        ##         return api_function.error_value
                 else:
                     # arg is not declared as PyObject, no magic
                     arg = input_arg
                 newargs += (arg, )
             try:
-                return func(space, *newargs)
+                return self.callable(space, *newargs)
             finally:
                 keepalive_until_here(*keepalives)
+        return unwrapper
 
+    def get_c_restype(self, c_writer):
+        return c_writer.gettype(self.restype).replace('@', '').strip()
+
+    def get_c_args(self, c_writer):
+        args = []
+        for i, argtype in enumerate(self.argtypes):
+            if argtype is CONST_STRING:
+                arg = 'const char *@'
+            elif argtype is CONST_STRINGP:
+                arg = 'const char **@'
+            elif argtype is CONST_WSTRING:
+                arg = 'const wchar_t *@'
+            else:
+                arg = c_writer.gettype(argtype)
+            arg = arg.replace('@', 'arg%d' % (i,)).strip()
+            args.append(arg)
+        args = ', '.join(args) or "void"
+        return args
+
+    def get_api_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "PyAPI_FUNC({restype}) {name}({args});".format(**locals())
+
+    def get_ptr_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "{restype} (*{name})({args});".format(**locals())
+
+    def get_ctypes_impl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        callargs = ', '.join('arg%d' % (i,)
+                            for i in range(len(self.argtypes)))
+        if self.restype is lltype.Void:
+            body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+        else:
+            body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+        return '%s %s(%s)\n%s' % (restype, name, args, body)
+
+
+DEFAULT_HEADER = 'pypy_decl.h'
+def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
+                gil=None, result_borrowed=False, result_is_ll=False):
+    """
+    Declares a function to be exported.
+    - `argtypes`, `restype` are lltypes and describe the function signature.
+    - `error` is the value returned when an applevel exception is raised. The
+      special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual
+      exception into a wrapped SystemError.  Unwrapped exceptions also cause a
+      SytemError.
+    - `header` is the header file to export the function in.
+    - set `gil` to "acquire", "release" or "around" to acquire the GIL,
+      release the GIL, or both
+    """
+    assert header is not None
+    def decorate(func):
+        if func.__name__ in FUNCTIONS_BY_HEADER[header]:
+            raise ValueError("%s already registered" % func.__name__)
+        api_function = _create_api_func(
+            func, argtypes, restype, error, gil=gil,
+            result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+        unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
 
@@ -450,25 +452,64 @@
             try:
                 res = unwrapper(space, *args)
             except OperationError as e:
-                if not hasattr(api_function, "error_value"):
+                if not hasattr(unwrapper.api_func, "error_value"):
                     raise
                 state = space.fromcache(State)
                 state.set_exception(e)
                 if is_PyObject(restype):
                     return None
                 else:
-                    return api_function.error_value
+                    return unwrapper.api_func.error_value
             got_integer = isinstance(res, (int, long, float))
+            if isinstance(restype, lltype.Typedef):
+                real_restype = restype.OF
+            else:
+                real_restype = restype
+            expect_integer = (isinstance(real_restype, lltype.Primitive) and
+                            rffi.cast(restype, 0) == 0)
             assert got_integer == expect_integer, (
                 'got %r not integer' % (res,))
             return res
 
         if header is not None:
-            FUNCTIONS_BY_HEADER[header][func_name] = api_function
-        INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
-        return unwrapper  # used in 'normal' RPython code.
+            FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
+            INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
+        return unwrapper
     return decorate
 
+def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
+    def decorate(func):
+        c_name = func.__name__
+        api_function = _create_api_func(func, argtypes, restype, error, c_name)
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
+
+
+def _create_api_func(
+        func, argtypes, restype, error=_NOT_SPECIFIED, c_name=None,
+        gil=None, result_borrowed=False, result_is_ll=False):
+    if isinstance(restype, lltype.Typedef):
+        real_restype = restype.OF
+    else:
+        real_restype = restype
+
+    if error is _NOT_SPECIFIED:
+        if isinstance(real_restype, lltype.Ptr):
+            error = lltype.nullptr(real_restype.TO)
+        elif real_restype is lltype.Void:
+            error = CANNOT_FAIL
+    if type(error) is int:
+        error = rffi.cast(real_restype, error)
+
+    func._always_inline_ = 'try'
+    return ApiFunction(
+        argtypes, restype, func, error, c_name=c_name, gil=gil,
+        result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+
+
 def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
     if level == 1:
@@ -980,23 +1021,6 @@
     for func in BOOTSTRAP_FUNCTIONS:
         func(space)
 
-def c_function_signature(db, func):
-    restype = db.gettype(func.restype).replace('@', '').strip()
-    args = []
-    for i, argtype in enumerate(func.argtypes):
-        if argtype is CONST_STRING:
-            arg = 'const char *@'
-        elif argtype is CONST_STRINGP:
-            arg = 'const char **@'
-        elif argtype is CONST_WSTRING:
-            arg = 'const wchar_t *@'
-        else:
-            arg = db.gettype(argtype)
-        arg = arg.replace('@', 'arg%d' % (i,)).strip()
-        args.append(arg)
-    args = ', '.join(args) or "void"
-    return restype, args
-
 #_____________________________________________________
 # Build the bridge DLL, Allow extension DLLs to call
 # back into Pypy space functions
@@ -1016,15 +1040,8 @@
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            restype, args = c_function_signature(db, func)
-            callargs = ', '.join('arg%d' % (i,)
-                                for i in range(len(func.argtypes)))
-            if func.restype is lltype.Void:
-                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
-            else:
-                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
-            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
-            members.append('%s (*%s)(%s);' % (restype, name, args))
+            functions.append(func.get_ctypes_impl(name, db))
+            members.append(func.get_ptr_decl(name, db))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
     struct_declaration_code = """\
@@ -1241,8 +1258,7 @@
         for name, func in sorted(header_functions.iteritems()):
             _name = mangle_name(prefix, name)
             header.append("#define %s %s" % (name, _name))
-            restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
+            header.append(func.get_api_decl(name, db))
 
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
+    cpython_api, Py_ssize_t, cpython_struct, bootstrap_function, slot_function,
     PyObjectFields, PyObject)
 from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.module.array.interp_array import ArrayBuffer
@@ -72,7 +72,7 @@
                 "Don't know how to realize a buffer")
 
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
     if py_buf.c_b_base:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL)
+    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -25,14 +25,14 @@
 ##
 ## In the PyBytesObject returned, the ob_sval buffer may be modified as
 ## long as the freshly allocated PyBytesObject is not "forced" via a call
-## to any of the more sophisticated C-API functions. 
+## to any of the more sophisticated C-API functions.
 ##
 ## Care has been taken in implementing the functions below, so that
-## if they are called with a non-forced PyBytesObject, they will not 
+## if they are called with a non-forced PyBytesObject, they will not
 ## unintentionally force the creation of a RPython object. As long as only these
 ## are used, the ob_sval buffer is still modifiable:
-## 
-## PyBytes_AsString / PyString_AsString 
+##
+## PyBytes_AsString / PyString_AsString
 ## PyBytes_AS_STRING / PyString_AS_STRING
 ## PyBytes_AsStringAndSize / PyString_AsStringAndSize
 ## PyBytes_Size / PyString_Size
@@ -40,7 +40,7 @@
 ## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string, 
+##   corresponds to the pypy gc-managed string,
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -111,7 +111,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def bytes_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -6,8 +6,8 @@
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
-    bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj, 
+    bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj,
         make_typedescr, track_reference, create_ref, from_ref, decref,
         Py_IncRef)
 from pypy.module.cpyext.object import _dealloc
@@ -36,7 +36,7 @@
     py_dict.c__tmpkeys = lltype.nullptr(PyObject.TO)
     # Problems: if this dict is a typedict, we may have unbound GetSetProperty
     # functions in the dict. The corresponding PyGetSetDescrObject must be
-    # bound to a class, but the actual w_type will be unavailable later on. 
+    # bound to a class, but the actual w_type will be unavailable later on.
     # Solution: use the w_userdata argument when assigning a PyTypeObject's
     # tp_dict slot to pass a w_type in, and force creation of the pair here
     if not space.is_w(w_userdata, space.gettypefor(GetSetProperty)):
@@ -55,7 +55,7 @@
     w_obj = space.newdict()
     track_reference(space, py_obj, w_obj)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def dict_dealloc(space, py_obj):
     py_dict = rffi.cast(PyDictObject, py_obj)
     decref(space, py_dict.c__tmpkeys)
@@ -287,7 +287,7 @@
     if space not in _frozendict_cache:
         _frozendict_cache[space] = _make_frozendict(space)
     return _frozendict_cache[space]
-        
+
 _frozendict_cache = {}
 def _make_frozendict(space):
     return space.appexec([], '''():
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObjectFields, cpython_struct,
-    CANNOT_FAIL)
+    CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, Py_DecRef, make_ref, from_ref, track_reference,
     make_typedescr, get_typedescr)
@@ -39,7 +39,7 @@
     py_frame.c_f_locals = make_ref(space, frame.get_w_locals())
     rffi.setintfield(py_frame, 'c_f_lineno', frame.getorcreatedebug().f_lineno)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def frame_dealloc(space, py_obj):
     py_frame = rffi.cast(PyFrameObject, py_obj)
     py_code = rffi.cast(PyObject, py_frame.c_f_code)
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from rpython.rlib.unroll import unrolling_iterable
@@ -56,7 +57,7 @@
     assert isinstance(w_obj, Function)
     py_func.c_func_name = make_ref(space, space.wrap(w_obj.name))
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def function_dealloc(space, py_obj):
     py_func = rffi.cast(PyFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_func_name)
@@ -75,7 +76,7 @@
     rffi.setintfield(py_code, 'c_co_flags', co_flags)
     rffi.setintfield(py_code, 'c_co_argcount', w_obj.co_argcount)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def code_dealloc(space, py_obj):
     py_code = rffi.cast(PyCodeObject, py_obj)
     Py_DecRef(space, py_code.c_co_name)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,9 +1,10 @@
-from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers,
-                         Py_ssize_tP, PyObjectFields, cpython_struct,
-                         bootstrap_function, Py_bufferP)
-from pypy.module.cpyext.pyobject import (PyObject, make_ref, as_pyobj, incref,
-             decref, from_ref, make_typedescr)
+from pypy.module.cpyext.api import (
+    cpython_api, Py_buffer, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS,
+    build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct,
+    bootstrap_function, Py_bufferP, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr,
+    get_typedescr, track_reference)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -28,23 +29,63 @@
                    basestruct=PyMemoryViewObject.TO,
                    attach=memory_attach,
                    dealloc=memory_dealloc,
-                   #realize=memory_realize,
+                   realize=memory_realize,
                   )
 
 def memory_attach(space, py_obj, w_obj, w_userdata=None):
     """
     Fills a newly allocated PyMemoryViewObject with the given W_MemoryView object.
     """
+    assert isinstance(w_obj, W_MemoryView)
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
-    py_obj.c_view.c_obj = rffi.cast(PyObject, 0)
+    view = py_obj.c_view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return
+    fill_Py_buffer(space, w_obj.buf, view)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_userdata)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s),
+                                             track_allocation=False))
+        rffi.setintfield(view, 'c_readonly', 1)
 
-def memory_realize(space, py_obj):
+def memory_realize(space, obj):
     """
     Creates the memory object in the interpreter
     """
-    raise oefmt(space.w_NotImplementedError, "cannot call this yet")
+    from pypy.module.cpyext.slotdefs import CPyBuffer, fq
+    py_mem = rffi.cast(PyMemoryViewObject, obj)
+    view = py_mem.c_view
+    ndim = widen(view.c_ndim)
+    shape = None
+    if view.c_shape:
+        shape = [view.c_shape[i] for i in range(ndim)]
+    strides = None
+    if view.c_strides:
+        strides = [view.c_strides[i] for i in range(ndim)]
+    format = 'B'
+    if view.c_format:
+        format = rffi.charp2str(view.c_format)
+    buf = CPyBuffer(space, view.c_buf, view.c_len, from_ref(space, view.c_obj),
+                    format=format, shape=shape, strides=strides,
+                    ndim=ndim, itemsize=view.c_itemsize,
+                    readonly=widen(view.c_readonly))
+    # Ensure view.c_buf is released upon object finalization
+    fq.register_finalizer(buf)
+    # Allow subclassing W_MemeoryView
+    w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
+    w_obj = space.allocate_instance(W_MemoryView, w_type)
+    w_obj.__init__(buf)
+    track_reference(space, obj, w_obj)
+    return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def memory_dealloc(space, py_obj):
     mem_obj = rffi.cast(PyMemoryViewObject, py_obj)
     if mem_obj.c_view.c_obj:
@@ -88,29 +129,13 @@
     return ret
 
 @cpython_api([PyObject], Py_bufferP, error=CANNOT_FAIL)
-def PyMemoryView_GET_BUFFER(space, w_obj):
+def PyMemoryView_GET_BUFFER(space, pyobj):
     """Return a pointer to the buffer-info structure wrapped by the given
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
-    if not isinstance(w_obj, W_MemoryView):
-        return lltype.nullptr(Py_buffer)
-    py_memobj = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_obj)) # no inc_ref
-    view = py_memobj.c_view
-    ndim = w_obj.buf.getndim()
-    if ndim >= Py_MAX_NDIMS:
-        # XXX warn?
-        return view
-    fill_Py_buffer(space, w_obj.buf, view)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-        #view.c_obj = make_ref(space, w_obj) # NO - this creates a ref cycle!
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
-    except ValueError:
-        w_s = w_obj.descr_tobytes(space)
-        view.c_obj = make_ref(space, w_s)
-        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
-        rffi.setintfield(view, 'c_readonly', 1)
-    return view
+    # XXX move to a c-macro
+    py_memobj = rffi.cast(PyMemoryViewObject, pyobj)
+    return py_memobj.c_view
 
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
@@ -202,20 +227,47 @@
         return (_IsCContiguous(view) or _IsFortranContiguous(view))
     return 0
 
- at cpython_api([PyObject], PyObject)
+ at cpython_api([PyObject], PyObject, result_is_ll=True)
 def PyMemoryView_FromObject(space, w_obj):
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    w_memview = space.call_method(space.builtin, "memoryview", w_obj)
+    py_memview = make_ref(space, w_memview, w_obj)
+    return py_memview
 
- at cpython_api([Py_bufferP], PyObject)
+ at cpython_api([Py_bufferP], PyObject, result_is_ll=True)
 def PyMemoryView_FromBuffer(space, view):
     """Create a memoryview object wrapping the given buffer-info structure view.
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
-    w_obj = from_ref(space, view.c_obj)
-    if isinstance(w_obj, W_MemoryView):
-        return w_obj
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    # XXX this should allocate a PyMemoryViewObject and
+    # copy view into obj.c_view, without creating a new view.c_obj
+    typedescr = get_typedescr(W_MemoryView.typedef)
+    py_obj = typedescr.allocate(space, space.w_memoryview)
+    py_mem = rffi.cast(PyMemoryViewObject, py_obj)
+    mview = py_mem.c_view
+    mview.c_buf = view.c_buf
+    mview.c_obj = view.c_obj
+    mview.c_len = view.c_len
+    mview.c_itemsize = view.c_itemsize
+    mview.c_readonly = view.c_readonly
+    mview.c_ndim = view.c_ndim
+    mview.c_format = view.c_format
+    if view.c_strides == rffi.cast(Py_ssize_tP, view.c__strides):
+        py_mem.c_view.c_strides = rffi.cast(Py_ssize_tP, py_mem.c_view.c__strides)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_strides[i] = view.c_strides[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_strides = view.c_strides
+    if view.c_shape == rffi.cast(Py_ssize_tP, view.c__shape):
+        py_mem.c_view.c_shape = rffi.cast(Py_ssize_tP, py_mem.c_view.c__shape)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_shape[i] = view.c_shape[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_shape = view.c_shape
+    # XXX ignore suboffsets?
+    return py_obj
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,7 +11,7 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr)
+    PyTypeObjectPtr, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -51,7 +51,7 @@
     py_func.c_m_self = make_ref(space, w_obj.w_self)
     py_func.c_m_module = make_ref(space, w_obj.w_module)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def cfunction_dealloc(space, py_obj):
     py_func = rffi.cast(PyCFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_m_self)
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t,
+    PyVarObject, Py_buffer, size_t, slot_function,
+    PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -54,7 +55,7 @@
         w_obj = PyObject_InitVar(space, py_objvar, type, itemcount)
     return py_obj
 
- at cpython_api([PyObject], lltype.Void)
+ at slot_function([PyObject], lltype.Void)
 def PyObject_dealloc(space, obj):
     return _dealloc(space, obj)
 
@@ -486,22 +487,28 @@
     Fills in a buffer-info structure correctly for an exporter that can only
     share a contiguous chunk of memory of "unsigned bytes" of the given
     length. Returns 0 on success and -1 (with raising an error) on error.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
     """
     if flags & PyBUF_WRITABLE and readonly:
         raise oefmt(space.w_ValueError, "Object is not writable")
     view.c_buf = buf
     view.c_len = length
     view.c_obj = obj
-    Py_IncRef(space, obj)
+    if obj:
+        Py_IncRef(space, obj)
     view.c_itemsize = 1
     rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
+    rffi.setintfield(view, 'c_ndim', 1)
     view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
     view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
     view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
 
@@ -511,7 +518,7 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Release the buffer view. This should be called when the buffer is 
+    Release the buffer view. This should be called when the buffer is
     no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -87,7 +87,7 @@
     alloc     : allocate and basic initialization of a raw PyObject
     attach    : Function called to tie a raw structure to a pypy object
     realize   : Function called to create a pypy object from a raw struct
-    dealloc   : a cpython_api(header=None), similar to PyObject_dealloc
+    dealloc   : a @slot_function(), similar to PyObject_dealloc
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
@@ -40,7 +41,7 @@
     rffi.setintfield(py_traceback, 'c_tb_lasti', traceback.lasti)
     rffi.setintfield(py_traceback, 'c_tb_lineno',traceback.get_lineno())
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def traceback_dealloc(space, py_obj):
     py_traceback = rffi.cast(PyTracebackObject, py_obj)
     Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next))
diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py
--- a/pypy/module/cpyext/sliceobject.py
+++ b/pypy/module/cpyext/sliceobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields)
+    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, PyObject, make_ref, make_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
@@ -36,7 +36,7 @@
     py_slice.c_stop = make_ref(space, w_obj.w_stop)
     py_slice.c_step = make_ref(space, w_obj.w_step)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def slice_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -6,7 +6,7 @@
 from rpython.rlib.rarithmetic import widen
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
+    slot_function, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
     pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
@@ -65,7 +65,7 @@
 
 @not_rpython
 def llslot(space, func):
-    return llhelper(func.api_func.functype, func.api_func.get_wrapper(space))
+    return func.api_func.get_llhelper(space)
 
 @register_flow_sc(llslot)
 def sc_llslot(ctx, v_space, v_func):
@@ -325,7 +325,7 @@
 
     def __init__(self, space, ptr, size, w_obj, format='B', shape=None,
                 strides=None, ndim=1, itemsize=1, readonly=True,
-                releasebuffer=None):
+                releasebufferproc=rffi.cast(rffi.VOIDP, 0)):
         self.space = space
         self.ptr = ptr
         self.size = size
@@ -343,7 +343,7 @@
         self.ndim = ndim
         self.itemsize = itemsize
         self.readonly = readonly
-        self.releasebufferproc = releasebuffer
+        self.releasebufferproc = releasebufferproc
 
     def releasebuffer(self):
         if self.pyobj:
@@ -361,7 +361,10 @@
                 for i in range(self.ndim):
                     pybuf.c_shape[i] = self.shape[i]
                     pybuf.c_strides[i] = self.strides[i]
-                pybuf.c_format = rffi.str2charp(self.format)
+                if self.format:
+                    pybuf.c_format = rffi.str2charp(self.format)
+                else:
+                    pybuf.c_format = rffi.str2charp("B")
                 generic_cpy_call(self.space, func_target, self.pyobj, pybuf)
             self.releasebufferproc = rffi.cast(rffi.VOIDP, 0)
 
@@ -408,9 +411,9 @@
     func_target = rffi.cast(readbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
@@ -418,7 +421,7 @@
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -427,16 +430,16 @@
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
     decref(space, py_obj)
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -444,9 +447,9 @@
     func_target = rffi.cast(getbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(Py_buffer) as pybuf:
         _flags = 0
@@ -472,7 +475,7 @@
                             ndim=ndim, shape=shape, strides=strides,
                             itemsize=pybuf.c_itemsize,
                             readonly=widen(pybuf.c_readonly),
-                            releasebuffer = releasebuffer)
+                            releasebufferproc = rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -526,9 +529,6 @@
 def build_slot_tp_function(space, typedef, name):
     w_type = space.gettypeobject(typedef)
 
-    header = pypy_decl
-    if not (name.startswith('Py') or name.startswith('_Py')):
-        header = None
     handled = False
     # unary functions
     for tp_name, attr in [('tp_as_number.c_nb_int', '__int__'),
@@ -550,7 +550,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject], PyObject, header=header)
+            @slot_function([PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self):
                 return space.call_function(slot_fn, w_self)
@@ -589,7 +589,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg):
                 return space.call_function(slot_fn, w_self, w_arg)
@@ -606,7 +606,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, Py_ssize_t], PyObject, header=header)
+            @slot_function([PyObject, Py_ssize_t], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, arg):
                 return space.call_function(slot_fn, w_self, space.wrap(arg))
@@ -620,7 +620,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
@@ -634,8 +634,8 @@
         if setattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real,
-                     error=-1, header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real,
+                     error=-1)
         @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,))
         def slot_tp_setattro(space, w_self, w_name, w_value):
             if w_value is not None:
@@ -649,7 +649,7 @@
         if getattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject], PyObject)
         @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,))
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
@@ -659,7 +659,7 @@
         if call_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_call(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -672,7 +672,7 @@
         if iternext_fn is None:
             return
 
-        @cpython_api([PyObject], PyObject, header=header)
+        @slot_function([PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_iternext(space, w_self):
             try:
@@ -688,8 +688,7 @@
         if init_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1,
-                     header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_init(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -702,7 +701,7 @@
         if new_fn is None:
             return
 
-        @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
+        @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_new(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -713,8 +712,8 @@
         buff_fn = w_type.getdictvalue(space, '__buffer__')
         if buff_fn is None:
             return
-        @cpython_api([PyObject, Py_bufferP, rffi.INT_real],
-                rffi.INT_real, header=None, error=-1)
+        @slot_function([PyObject, Py_bufferP, rffi.INT_real],
+                rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def buff_w(space, w_self, view, flags):
             args = Arguments(space, [space.newint(flags)])
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,10 +89,10 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
-                == ('Py_ssize_t', 'Py_ssize_t arg0'))
-        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
-                == ('Py_ssize_t *', 'Py_ssize_t *arg0'))
+        assert PyPy_TypedefTest1.api_func.get_c_restype(db) == 'Py_ssize_t'
+        assert PyPy_TypedefTest1.api_func.get_c_args(db) == 'Py_ssize_t arg0'
+        assert PyPy_TypedefTest2.api_func.get_c_restype(db) == 'Py_ssize_t *'
+        assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Py_ssize_t *arg0'
 
         PyPy_TypedefTest1(space, 0)
         ppos = lltype.malloc(api.Py_ssize_tP.TO, 1, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -4,6 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
+from pypy.module.cpyext.pyobject import from_ref
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
@@ -11,7 +12,7 @@
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
-        w_view = api.PyMemoryView_FromObject(w_hello)
+        w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
         assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
@@ -19,7 +20,7 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_memoryview = from_ref(space, api.PyMemoryView_FromObject(w_buf))
         view = api.PyMemoryView_GET_BUFFER(w_memoryview)
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)
@@ -29,7 +30,7 @@
         assert view.c_len == 5
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
-        w_mv = api.PyMemoryView_FromBuffer(view)
+        w_mv = from_ref(space, api.PyMemoryView_FromBuffer(view))
         for f in ('format', 'itemsize', 'ndim', 'readonly', 
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)
@@ -43,6 +44,7 @@
                 ("fillinfo", "METH_VARARGS",
                  """
                  Py_buffer buf;
+                 PyObject * ret = NULL;
                  PyObject *str = PyBytes_FromString("hello, world.");
                  if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
                                        0, 0)) {
@@ -54,7 +56,14 @@
                   */
                  Py_DECREF(str);
 
-                 return PyMemoryView_FromBuffer(&buf);
+                 ret = PyMemoryView_FromBuffer(&buf);
+                 if (((PyMemoryViewObject*)ret)->view.obj != buf.obj)
+                 {
+                    PyErr_SetString(PyExc_ValueError, "leaked ref");
+                    Py_DECREF(ret);
+                    return NULL;
+                 }
+                 return ret;
                  """)])
         result = module.fillinfo()
         assert b"hello, world." == result
diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test/test_translate.py
--- a/pypy/module/cpyext/test/test_translate.py
+++ b/pypy/module/cpyext/test/test_translate.py
@@ -1,6 +1,6 @@
 from rpython.translator.c.test.test_genc import compile
 import pypy.module.cpyext.api
-from pypy.module.cpyext.api import cpython_api
+from pypy.module.cpyext.api import slot_function
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.objectmodel import specialize
@@ -19,7 +19,7 @@
 
     @specialize.memo()
     def get_tp_function(space, typedef):
-        @cpython_api([], lltype.Signed, error=-1, header=None)
+        @slot_function([], lltype.Signed, error=-1)
         def slot_tp_function(space):
             return typedef.value
 
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -1,11 +1,11 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import fatalerror_notb
-from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL,
-                                    build_type_checkers, PyVarObjectFields,
-                                    cpython_struct, bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef,
-    make_ref, from_ref, decref, incref, pyobj_has_w_obj,
+from pypy.module.cpyext.api import (
+    cpython_api, Py_ssize_t, build_type_checkers,
+    PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, PyObjectP, make_ref, from_ref, decref, incref,
     track_reference, make_typedescr, get_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
@@ -74,7 +74,7 @@
     if py_tup.c_ob_size < length:
         raise oefmt(space.w_ValueError,
             "tuple_attach called on object with ob_size %d but trying to store %d",
-            py_tup.c_ob_size, length) 
+            py_tup.c_ob_size, length)
     i = 0
     try:
         while i < length:
@@ -113,7 +113,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def tuple_dealloc(space, py_obj):
     """Frees allocated PyTupleObject resources.
     """
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -2,21 +2,20 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib.rstring import rsplit
 from rpython.rtyper.lltypesystem import rffi, lltype
 
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.typedef import (GetSetProperty, TypeDef,
-        interp_attrproperty, interp_attrproperty, interp2app)
+from pypy.interpreter.typedef import (
+    GetSetProperty, TypeDef, interp_attrproperty, interp2app)
 from pypy.module.__builtin__.abstractinst import abstract_issubclass_w
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
+    PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
     Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
@@ -373,7 +372,7 @@
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
 
- at cpython_api([PyObject, PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
 def tp_new_wrapper(space, self, w_args, w_kwds):
     self_pytype = rffi.cast(PyTypeObjectPtr, self)
     tp_new = self_pytype.c_tp_new
@@ -527,7 +526,7 @@
                    realize=type_realize,
                    dealloc=type_dealloc)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def subtype_dealloc(space, obj):
     pto = obj.c_ob_type
     base = pto
@@ -558,15 +557,13 @@
     # hopefully this does not clash with the memory model assumed in
     # extension modules
 
- at cpython_api([PyObject, Py_ssize_tP], lltype.Signed, header=None,
-             error=CANNOT_FAIL)
+ at slot_function([PyObject, Py_ssize_tP], lltype.Signed, error=CANNOT_FAIL)
 def bf_segcount(space, w_obj, ref):
     if ref:
         ref[0] = space.len_w(w_obj)
     return 1
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def bf_getreadbuffer(space, w_buf, segment, ref):
     from rpython.rlib.buffer import StringBuffer
     if segment != 0:
@@ -579,13 +576,11 @@
     ref[0] = address
     return len(buf)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def bf_getcharbuffer(space, w_buf, segment, ref):
     return bf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def bf_getwritebuffer(space, w_buf, segment, ref):
     if segment != 0:
         raise oefmt(space.w_SystemError,
@@ -594,8 +589,7 @@
     ref[0] = buf.get_raw_address()
     return len(buf)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def str_getreadbuffer(space, w_str, segment, ref):
     from pypy.module.cpyext.bytesobject import PyString_AsString
     if segment != 0:
@@ -607,11 +601,10 @@
     Py_DecRef(space, pyref)
     return space.len_w(w_str)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def unicode_getreadbuffer(space, w_str, segment, ref):
     from pypy.module.cpyext.unicodeobject import (
-                PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
+        PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
     if segment != 0:
         raise oefmt(space.w_SystemError,
                     "accessing non-existent unicode segment")
@@ -621,13 +614,11 @@
     Py_DecRef(space, pyref)
     return PyUnicode_GET_DATA_SIZE(space, w_str)
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def str_getcharbuffer(space, w_buf, segment, ref):
     return str_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
- at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, error=-1)
 def buf_getreadbuffer(space, pyref, segment, ref):
     from pypy.module.cpyext.bufferobject import PyBufferObject
     if segment != 0:
@@ -637,8 +628,7 @@
     ref[0] = py_buf.c_b_ptr
     return py_buf.c_b_size
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
-             header=None, error=-1)
+ at slot_function([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, error=-1)
 def buf_getcharbuffer(space, w_buf, segment, ref):
     return buf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
 
@@ -678,7 +668,7 @@
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def type_dealloc(space, obj):
     from pypy.module.cpyext.object import _dealloc
     obj_pto = rffi.cast(PyTypeObjectPtr, obj)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
     bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
-    CONST_WSTRING)
+    CONST_WSTRING, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -85,7 +85,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def unicode_dealloc(space, py_obj):
     py_unicode = rffi.cast(PyUnicodeObject, py_obj)
     Py_DecRef(space, py_unicode.c_defenc)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -56,7 +56,7 @@
         assert u"abc" != memoryview("abc")
 
     def test_pypy_raw_address_base(self):
-        a = memoryview("foobar")._pypy_raw_address()
+        a = memoryview(b"foobar")._pypy_raw_address()
         assert a != 0
-        b = memoryview(bytearray("foobar"))._pypy_raw_address()
+        b = memoryview(bytearray(b"foobar"))._pypy_raw_address()
         assert b != 0
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -210,7 +210,6 @@
     return rlist.ll_pop(rlist.dum_checkidx, l, index)
 _ll_2_list_append = rlist.ll_append
 _ll_2_list_extend = rlist.ll_extend
-_ll_3_list_insert = rlist.ll_insert_nonneg
 _ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly
 _ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop
 _ll_2_list_inplace_mul = rlist.ll_inplace_mul
diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -212,6 +212,8 @@
                 s += lst[0]
                 lst.pop()
                 lst.append(1)
+                lst.insert(0, 5)
+                lst.insert(1, 6)
                 s *= lst.pop()
             return s
         res = self.meta_interp(f, [15], listops=True)
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -51,6 +51,7 @@
     voidpf = rffi_platform.SimpleType('voidpf', rffi.VOIDP)
 
     ZLIB_VERSION = rffi_platform.DefinedConstantString('ZLIB_VERSION')
+    ZLIB_VERNUM = rffi_platform.DefinedConstantInteger('ZLIB_VERNUM')
 
 for _name in constantnames:
     setattr(SimpleCConfig, _name, rffi_platform.ConstantInteger(_name))
@@ -63,6 +64,7 @@
 Bytefp = lltype.Ptr(lltype.Array(Bytef, hints={'nolength': True}))
 
 ZLIB_VERSION = config['ZLIB_VERSION']
+ZLIB_VERNUM = config['ZLIB_VERNUM']
 
 for _name in constantnames:
     globals()[_name] = config[_name]
@@ -261,7 +263,7 @@
 
 def deflateInit(level=Z_DEFAULT_COMPRESSION, method=Z_DEFLATED,
                 wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL,
-                strategy=Z_DEFAULT_STRATEGY):
+                strategy=Z_DEFAULT_STRATEGY, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     compress data.
@@ -270,6 +272,12 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
     if err == Z_OK:
+        if zdict is not None:
+            try:
+                deflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:
@@ -290,7 +298,7 @@
     lltype.free(stream, flavor='raw')
 
 
-def inflateInit(wbits=MAX_WBITS):
+def inflateInit(wbits=MAX_WBITS, zdict=None):
     """
     Allocate and return an opaque 'stream' object that can be used to
     decompress data.
@@ -299,6 +307,17 @@
     rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _inflateInit2(stream, wbits)
     if err == Z_OK:
+        if zdict is not None and wbits < 0:
+            try:
+                if ZLIB_VERNUM is None or ZLIB_VERNUM < 0x1221:
+                    raise RZlibError("zlib version %s does not allow raw "
+                                     "inflate with dictionary" %
+                                       ZLIB_VERSION if ZLIB_VERSION is not None
+                                       else "<unknown>")
+                inflateSetDictionary(stream, zdict)
+            except:
+                lltype.free(stream, flavor='raw')
+                raise
         return stream
     else:
         try:
diff --git a/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-8.0.0.txt
@@ -0,0 +1,1414 @@
+# CaseFolding-8.0.0.txt
+# Date: 2015-01-13, 18:16:36 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#223;e" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding 
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+# Property: Case_Folding
+
+#  All code points not explicitly listed for Case_Folding
+#  have the value C for the status field, and the code point itself for the mapping field.
+
+# =================================================================
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE
+023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE
+023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR
+023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
+0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP
+0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE
+0244; C; 0289; # LATIN CAPITAL LETTER U BAR
+0245; C; 028C; # LATIN CAPITAL LETTER TURNED V
+0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE
+0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE
+024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
+024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE
+024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0370; C; 0371; # GREEK CAPITAL LETTER HETA
+0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI
+0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
+037F; C; 03F3; # GREEK CAPITAL LETTER YOT
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+03F7; C; 03F8; # GREEK CAPITAL LETTER SHO
+03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL
+03FA; C; 03FB; # GREEK CAPITAL LETTER SAN
+03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
+03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
+03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
+04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK
+04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE
+0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK
+0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA
+0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA
+0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE
+051A; C; 051B; # CYRILLIC CAPITAL LETTER QA
+051C; C; 051D; # CYRILLIC CAPITAL LETTER WE
+051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA
+0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
+0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
+0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER
+0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER
+0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK
+052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE
+052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE
+052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN
+10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN
+10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN
+10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON
+10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN
+10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN
+10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN
+10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN
+10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN
+10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN

From pypy.commits at gmail.com  Tue Jan 10 22:17:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 10 Jan 2017 19:17:25 -0800 (PST)
Subject: [pypy-commit] pypy default: Cleanup; kill _create_api_func()
Message-ID: <5875a3c5.849c1c0a.189ed.802d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89488:c501282f4438
Date: 2017-01-11 03:15 +0000
http://bitbucket.org/pypy/pypy/changeset/c501282f4438/

Log:	Cleanup; kill _create_api_func()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -249,14 +249,13 @@
 cpyext_namespace = NameManager('cpyext_')
 
 class ApiFunction(object):
-    def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED,
+    def __init__(self, argtypes, restype, callable, error=CANNOT_FAIL,
                  c_name=None, gil=None, result_borrowed=False, result_is_ll=False):
         self.argtypes = argtypes
         self.restype = restype
         self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
         self.callable = callable
-        if error is not _NOT_SPECIFIED:
-            self.error_value = error
+        self.error_value = error
         self.c_name = c_name
 
         # extract the signature from the (CPython-level) code object
@@ -298,7 +297,7 @@
 
         argtypesw = zip(self.argtypes,
                         [_name.startswith("w_") for _name in self.argnames])
-        error_value = getattr(self, "error_value", CANNOT_FAIL)
+        error_value = self.error_value
         if (isinstance(self.restype, lltype.Ptr)
                 and error_value is not CANNOT_FAIL):
             assert lltype.typeOf(error_value) == self.restype
@@ -436,12 +435,12 @@
     def decorate(func):
         if func.__name__ in FUNCTIONS_BY_HEADER[header]:
             raise ValueError("%s already registered" % func.__name__)
-        api_function = _create_api_func(
-            func, argtypes, restype, error, gil=gil,
+        func._always_inline_ = 'try'
+        api_function = ApiFunction(
+            argtypes, restype, func,
+            error=_compute_error(error, restype), gil=gil,
             result_borrowed=result_borrowed, result_is_ll=result_is_ll)
-        unwrapper = api_function.get_unwrapper()
-        unwrapper.func = func
-        unwrapper.api_func = api_function
+        FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
 
         # ZZZ is this whole logic really needed???  It seems to be only
         # for RPython code calling PyXxx() functions directly.  I would
@@ -469,32 +468,33 @@
             assert got_integer == expect_integer, (
                 'got %r not integer' % (res,))
             return res
+        INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
 
-        if header is not None:
-            FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
-            INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
-        return unwrapper
-    return decorate
-
-def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
-    def decorate(func):
-        c_name = func.__name__
-        api_function = _create_api_func(func, argtypes, restype, error, c_name)
         unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
         return unwrapper
     return decorate
 
+def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
+    def decorate(func):
+        func._always_inline_ = 'try'
+        api_function = ApiFunction(
+            argtypes, restype, func,
+            error=_compute_error(error, restype),
+            c_name=func.__name__)
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
 
-def _create_api_func(
-        func, argtypes, restype, error=_NOT_SPECIFIED, c_name=None,
-        gil=None, result_borrowed=False, result_is_ll=False):
+def _compute_error(error, restype):
+    """Convert error specification to actual error value of type restype."""
     if isinstance(restype, lltype.Typedef):
         real_restype = restype.OF
     else:
         real_restype = restype
-
     if error is _NOT_SPECIFIED:
         if isinstance(real_restype, lltype.Ptr):
             error = lltype.nullptr(real_restype.TO)
@@ -502,11 +502,7 @@
             error = CANNOT_FAIL
     if type(error) is int:
         error = rffi.cast(real_restype, error)
-
-    func._always_inline_ = 'try'
-    return ApiFunction(
-        argtypes, restype, func, error, c_name=c_name, gil=gil,
-        result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+    return error
 
 
 def cpython_struct(name, fields, forward=None, level=1):

From pypy.commits at gmail.com  Wed Jan 11 06:25:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 11 Jan 2017 03:25:55 -0800 (PST)
Subject: [pypy-commit] pypy default: Disable all tests when the '-std=c++11'
 option is not understood by the compiler
Message-ID: <58761643.8a281c0a.2faca.0ac0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89489:48bdd88d30d9
Date: 2017-01-11 12:25 +0100
http://bitbucket.org/pypy/pypy/changeset/48bdd88d30d9/

Log:	Disable all tests when the '-std=c++11' option is not understood by
	the compiler

diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -23,6 +23,10 @@
 def pytest_ignore_collect(path, config):
     if py.path.local.sysfind('genreflex') is None and config.option.runappdirect:
         return True          # "can't run dummy tests in -A"
+    if disabled:
+        return True
+
+disabled = None
 
 def pytest_configure(config):
     if py.path.local.sysfind('genreflex') is None:
@@ -37,7 +41,7 @@
             # build dummy backend (which has reflex info and calls hard-wired)
             import os
             from rpython.translator.tool.cbuild import ExternalCompilationInfo
-            from rpython.translator.platform import platform
+            from rpython.translator.platform import platform, CompilationError
             from rpython.translator import cdir
 
             from rpython.rtyper.lltypesystem import rffi
@@ -55,9 +59,16 @@
                 use_cpp_linker=True,
             )
 
-            soname = platform.compile(
-                [], eci,
-                outputfilename='libcppyy_dummy_backend',
-                standalone=False)
+            try:
+                soname = platform.compile(
+                    [], eci,
+                    outputfilename='libcppyy_dummy_backend',
+                    standalone=False)
+            except CompilationError as e:
+                if '-std=c++11' in str(e):
+                    global disabled
+                    disabled = str(e)
+                    return
+                raise
 
             lcapi.reflection_library = str(soname)

From pypy.commits at gmail.com  Wed Jan 11 06:52:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 11 Jan 2017 03:52:56 -0800 (PST)
Subject: [pypy-commit] pypy default: Test with /dev instead of /tmp
Message-ID: <58761c98.876ec20a.f3d5a.e795@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89490:6622cf362ad3
Date: 2017-01-11 12:52 +0100
http://bitbucket.org/pypy/pypy/changeset/6622cf362ad3/

Log:	Test with /dev instead of /tmp

diff --git a/rpython/rlib/test/test_rposix_stat.py b/rpython/rlib/test/test_rposix_stat.py
--- a/rpython/rlib/test/test_rposix_stat.py
+++ b/rpython/rlib/test/test_rposix_stat.py
@@ -29,7 +29,10 @@
             check(os.environ['TEMP'])
         else:
             check('/')
-            check('/tmp')
+            check('/dev')
+            # don't test with /tmp, because if another process is also
+            # creating files in /tmp it is more likely that the mtime
+            # we get during successive calls was actually changed
         check(sys.executable)
 
     def test_fstat(self):

From pypy.commits at gmail.com  Wed Jan 11 07:17:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 11 Jan 2017 04:17:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Test the hash of RPython,
 which is the same as the PyPy 2.7 hash
Message-ID: <58762252.ce841c0a.f1f57.62ea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89491:2242a56ca9bb
Date: 2017-01-11 13:16 +0100
http://bitbucket.org/pypy/pypy/changeset/2242a56ca9bb/

Log:	Test the hash of RPython, which is the same as the PyPy 2.7 hash

diff --git a/lib-python/3/test/test_hash.py b/lib-python/3/test/test_hash.py
--- a/lib-python/3/test/test_hash.py
+++ b/lib-python/3/test/test_hash.py
@@ -43,7 +43,7 @@
 
 def skip_unless_internalhash(test):
     """Skip decorator for tests that depend on SipHash24 or FNV"""
-    ok = sys.hash_info.algorithm in {"fnv", "siphash24"}
+    ok = sys.hash_info.algorithm in {"fnv", "siphash24", "rpython"}
     msg = "Requires SipHash24 or FNV"
     return test if ok else unittest.skip(msg)(test)
 
@@ -175,7 +175,6 @@
     def get_hash_command(self, repr_):
         return 'print(hash(eval(%a)))' % repr_
 
-    @impl_detail("PyPy does not support hash randomization", pypy=False)
     def get_hash(self, repr_, seed=None):
         env = os.environ.copy()
         env['__cleanenv'] = True  # signal to assert_python not to do a copy
@@ -190,6 +189,7 @@
         stdout = out[1].strip()
         return int(stdout)
 
+    @impl_detail("PyPy does not support hash randomization", pypy=False)
     def test_randomized_hash(self):
         # two runs should return different hashes
         run1 = self.get_hash(self.repr_, seed='random')
@@ -241,7 +241,26 @@
             # seed 42, '&#228;&#250;&#8721;&#8455;'
             [-283066365, -4576729883824601543, -271871407,
              -3927695501187247084],
-        ]
+        ],
+        'rpython': [
+            # This is for PyPy.  NOTE: PyPy does not support hash
+            # randomization for now, so the results don't depend on the seed.
+            # seed 0, 'abc'
+            [-1600925533, 1453079729188098211, -1600925533,
+             1453079729188098211],
+            # seed 42, 'abc'
+            [-1600925533, 1453079729188098211, -1600925533,
+             1453079729188098211],
+            # seed 42, 'abcdefghijk'
+            [112677275, -7109391839480295013, 112677275,
+             -7109391839480295013],
+            # seed 0, '&#228;&#250;&#8721;&#8455;'
+            [-272186246, 6456588004676256890, -272186246,
+             6456588004676256890],
+            # seed 42, '&#228;&#250;&#8721;&#8455;'
+            [-272186246, 6456588004676256890, -272186246,
+             6456588004676256890],
+        ],
     }
 
     def get_expected_hash(self, position, length):
@@ -261,7 +280,8 @@
         known_hash_of_obj = self.get_expected_hash(0, 3)
 
         # Randomization is enabled by default:
-        self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
+        if check_impl_detail(pypy=False):
+            self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
 
         # It can also be disabled by setting the seed to 0:
         self.assertEqual(self.get_hash(self.repr_, seed=0), known_hash_of_obj)
@@ -287,6 +307,7 @@
     repr_long = repr('abcdefghijk')
     repr_ucs2 = repr('&#228;&#250;&#8721;&#8455;')
 
+    @impl_detail("hash('') == -2 on PyPy", pypy=False)
     @skip_unless_internalhash
     def test_empty_string(self):
         self.assertEqual(hash(""), 0)

From pypy.commits at gmail.com  Wed Jan 11 07:19:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 11 Jan 2017 04:19:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: weakref support for mmap objects
Message-ID: <587622e8.c4251c0a.8fe0a.25a8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89492:ac904ea5b8b7
Date: 2017-01-11 13:19 +0100
http://bitbucket.org/pypy/pypy/changeset/ac904ea5b8b7/

Log:	weakref support for mmap objects

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from rpython.rlib import rmmap, rarithmetic, objectmodel
 from rpython.rlib.buffer import Buffer
@@ -282,6 +282,7 @@
     __setitem__ = interp2app(W_MMap.descr_setitem),
     __enter__ = interp2app(W_MMap.descr_enter),
     __exit__ = interp2app(W_MMap.descr_exit),
+    __weakref__ = make_weakref_descr(W_MMap),
 
     closed = GetSetProperty(W_MMap.closed_get),
 )
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -3,7 +3,6 @@
 """
 
 import time
-import weakref
 from rpython.rlib import rthread
 from pypy.module.thread.error import wrap_thread_error
 from pypy.interpreter.baseobjspace import W_Root

From pypy.commits at gmail.com  Wed Jan 11 11:06:03 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 11 Jan 2017 08:06:03 -0800 (PST)
Subject: [pypy-commit] pypy default: Cleanup: avoid confusion between the
 api object and the api module, don't use api.PyXXX magic
Message-ID: <587657eb.2403c20a.5c297.4ba9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89493:83b73296760e
Date: 2017-01-11 16:05 +0000
http://bitbucket.org/pypy/pypy/changeset/83b73296760e/

Log:	Cleanup: avoid confusion between the api object and the api module,
	don't use api.PyXXX magic

diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -2,20 +2,14 @@
 from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
-from pypy.module.cpyext import api
+from pypy.module.cpyext.api import (
+    slot_function, cpython_api, copy_header_files, INTERPLEVEL_API,
+    Py_ssize_t, Py_ssize_tP, PyObject)
 from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest
-PyObject = api.PyObject
 from pypy.interpreter.error import OperationError
 from rpython.rlib import rawrefcount
 import os
 
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetWrapped(space, w_arg):
-    assert isinstance(w_arg, W_Root)
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetReference(space, arg):
-    assert lltype.typeOf(arg) ==  PyObject
-
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
         space = cls.space
@@ -35,7 +29,7 @@
             def __getattr__(self, name):
                 return getattr(cls.space, name)
         cls.api = CAPI()
-        CAPI.__dict__.update(api.INTERPLEVEL_API)
+        CAPI.__dict__.update(INTERPLEVEL_API)
 
         print 'DONT_FREE_ANY_MORE'
         rawrefcount._dont_free_any_more()
@@ -71,20 +65,28 @@
         if self.check_and_print_leaks():
             assert False, "Test leaks or loses object(s)."
 
- at api.cpython_api([api.Py_ssize_t], api.Py_ssize_t, error=-1)
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetWrapped(space, w_arg):
+    assert isinstance(w_arg, W_Root)
+
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetReference(space, arg):
+    assert lltype.typeOf(arg) ==  PyObject
+
+ at cpython_api([Py_ssize_t], Py_ssize_t, error=-1)
 def PyPy_TypedefTest1(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_t
+    assert lltype.typeOf(arg) == Py_ssize_t
     return 0
 
- at api.cpython_api([api.Py_ssize_tP], api.Py_ssize_tP)
+ at cpython_api([Py_ssize_tP], Py_ssize_tP)
 def PyPy_TypedefTest2(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_tP
+    assert lltype.typeOf(arg) == Py_ssize_tP
     return None
 
 class TestConversion(BaseApiTest):
-    def test_conversions(self, space, api):
-        api.PyPy_GetWrapped(space.w_None)
-        api.PyPy_GetReference(space.w_None)
+    def test_conversions(self, space):
+        PyPy_GetWrapped(space, space.w_None)
+        PyPy_GetReference(space, space.w_None)
 
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
@@ -95,7 +97,7 @@
         assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Py_ssize_t *arg0'
 
         PyPy_TypedefTest1(space, 0)
-        ppos = lltype.malloc(api.Py_ssize_tP.TO, 1, flavor='raw')
+        ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
         ppos[0] = 0
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
@@ -103,7 +105,7 @@
 @pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
-    api.copy_header_files(tmpdir, True)
+    copy_header_files(tmpdir, True)
     def check(name):
         f = tmpdir.join(name)
         assert f.check(file=True)

From pypy.commits at gmail.com  Wed Jan 11 11:41:04 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 11 Jan 2017 08:41:04 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove some uses of api.PyXXX magic
Message-ID: <58766020.973f1c0a.d9c5a.98ab@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89494:e3b03044a75c
Date: 2017-01-11 16:40 +0000
http://bitbucket.org/pypy/pypy/changeset/e3b03044a75c/

Log:	Remove some uses of api.PyXXX magic

diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py
--- a/pypy/module/cpyext/test/test_boolobject.py
+++ b/pypy/module/cpyext/test/test_boolobject.py
@@ -1,20 +1,22 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong
+from pypy.module.cpyext.floatobject import PyFloat_FromDouble
 
 class TestBoolObject(BaseApiTest):
-    def test_fromlong(self, space, api):
+    def test_fromlong(self, space):
         for i in range(-3, 3):
-            obj = api.PyBool_FromLong(i)
+            obj = PyBool_FromLong(space, i)
             if i:
                 assert obj is space.w_True
             else:
                 assert obj is space.w_False
 
-    def test_check(self, space, api):
-        assert api.PyBool_Check(space.w_True)
-        assert api.PyBool_Check(space.w_False)
-        assert not api.PyBool_Check(space.w_None)
-        assert not api.PyBool_Check(api.PyFloat_FromDouble(1.0))
+    def test_check(self, space):
+        assert PyBool_Check(space, space.w_True)
+        assert PyBool_Check(space, space.w_False)
+        assert not PyBool_Check(space, space.w_None)
+        assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0))
 
 class AppTestBoolMacros(AppTestCpythonExtensionBase):
     def test_macros(self):
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -2,13 +2,14 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject
+from pypy.module.cpyext.bytesobject import (
+    new_empty_str, PyBytesObject, _PyString_Resize, PyString_Concat,
+    PyString_ConcatAndDel, PyString_Format, PyString_InternFromString,
+    PyString_AsEncodedObject, PyString_AsDecodedObject, _PyString_Eq,
+    _PyString_Join)
 from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref
-from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
 
-import py
-import sys
 
 class AppTestBytesObject(AppTestCpythonExtensionBase):
     def test_bytesobject(self):
@@ -441,21 +442,21 @@
         assert a == 'abc'
 
 class TestBytes(BaseApiTest):
-    def test_bytes_resize(self, space, api):
+    def test_bytes_resize(self, space):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         py_str.c_ob_sval[0] = 'a'
         py_str.c_ob_sval[1] = 'b'
         py_str.c_ob_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
-        api._PyString_Resize(ar, 3)
+        _PyString_Resize(space, ar, 3)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 3
         assert py_str.c_ob_sval[1] == 'b'
         assert py_str.c_ob_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
-        api._PyString_Resize(ar, 10)
+        _PyString_Resize(space, ar, 10)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 10
         assert py_str.c_ob_sval[1] == 'b'
@@ -463,7 +464,7 @@
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
-    def test_string_buffer(self, space, api):
+    def test_string_buffer(self, space):
         py_str = new_empty_str(space, 10)
         c_buf = py_str.c_ob_type.c_tp_as_buffer
         assert c_buf
@@ -486,36 +487,36 @@
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref
         prev_refcnt = ref.c_ob_refcnt
-        api.PyString_Concat(ptr, space.wrap('def'))
+        PyString_Concat(space, ptr, space.wrap('def'))
         assert ref.c_ob_refcnt == prev_refcnt - 1
         assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
         api.PyString_Concat(ptr, space.w_None)
         assert not ptr[0]
         api.PyErr_Clear()
         ptr[0] = lltype.nullptr(PyObject.TO)
-        api.PyString_Concat(ptr, space.wrap('def')) # should not crash
+        PyString_Concat(space, ptr, space.wrap('def')) # should not crash
         lltype.free(ptr, flavor='raw')
 
-    def test_ConcatAndDel(self, space, api):
+    def test_ConcatAndDel(self, space):
         ref1 = make_ref(space, space.wrap('abc'))
         ref2 = make_ref(space, space.wrap('def'))
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref1
         prev_refcnf = ref2.c_ob_refcnt
-        api.PyString_ConcatAndDel(ptr, ref2)
+        PyString_ConcatAndDel(space, ptr, ref2)
         assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
         assert ref2.c_ob_refcnt == prev_refcnf - 1
         Py_DecRef(space, ptr[0])
         ptr[0] = lltype.nullptr(PyObject.TO)
         ref2 = make_ref(space, space.wrap('foo'))
         prev_refcnf = ref2.c_ob_refcnt
-        api.PyString_ConcatAndDel(ptr, ref2) # should not crash
+        PyString_ConcatAndDel(space, ptr, ref2) # should not crash
         assert ref2.c_ob_refcnt == prev_refcnf - 1
         lltype.free(ptr, flavor='raw')
 
-    def test_format(self, space, api):
+    def test_format(self, space):
         assert "1 2" == space.unwrap(
-            api.PyString_Format(space.wrap('%s %d'), space.wrap((1, 2))))
+            PyString_Format(space, space.wrap('%s %d'), space.wrap((1, 2))))
 
     def test_asbuffer(self, space, api):
         bufp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
@@ -530,12 +531,12 @@
         assert rffi.charp2str(bufp[0]) == 'text'
         lltype.free(bufp, flavor='raw')
         lltype.free(lenp, flavor='raw')
-        api.Py_DecRef(ref)
+        Py_DecRef(space, ref)
 
-    def test_intern(self, space, api):
+    def test_intern(self, space):
         buf = rffi.str2charp("test")
-        w_s1 = api.PyString_InternFromString(buf)
-        w_s2 = api.PyString_InternFromString(buf)
+        w_s1 = PyString_InternFromString(space, buf)
+        w_s2 = PyString_InternFromString(space, buf)
         rffi.free_charp(buf)
         assert w_s1 is w_s2
 
@@ -545,11 +546,10 @@
         errors = rffi.str2charp("strict")
 
         encoding = rffi.str2charp("hex")
-        res = api.PyString_AsEncodedObject(
-            ptr, encoding, errors)
+        res = PyString_AsEncodedObject(space, ptr, encoding, errors)
         assert space.unwrap(res) == "616263"
 
-        res = api.PyString_AsEncodedObject(
+        res = PyString_AsEncodedObject(space,
             ptr, encoding, lltype.nullptr(rffi.CCHARP.TO))
         assert space.unwrap(res) == "616263"
         rffi.free_charp(encoding)
@@ -561,28 +561,30 @@
 
         rffi.free_charp(errors)
 
-        res = api.PyString_AsEncodedObject(
-            ptr, lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO))
+        res = PyString_AsEncodedObject(
+            space, ptr, lltype.nullptr(rffi.CCHARP.TO),
+            lltype.nullptr(rffi.CCHARP.TO))
         assert space.unwrap(res) == "abc"
 
         self.raises(space, api, TypeError, api.PyString_AsEncodedObject,
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
-    def test_AsDecodedObject(self, space, api):
+    def test_AsDecodedObject(self, space):
         w_str = space.wrap('caf\xe9')
         encoding = rffi.str2charp("latin-1")
-        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        w_res = PyString_AsDecodedObject(space, w_str, encoding, None)
         rffi.free_charp(encoding)
         assert space.unwrap(w_res) == u"caf\xe9"
 
-    def test_eq(self, space, api):
-        assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
-        assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
+    def test_eq(self, space):
+        assert 1 == _PyString_Eq(
+            space, space.wrap("hello"), space.wrap("hello"))
+        assert 0 == _PyString_Eq(
+            space, space.wrap("hello"), space.wrap("world"))
 
-    def test_join(self, space, api):
+    def test_join(self, space):
         w_sep = space.wrap('<sep>')
         w_seq = space.wrap(['a', 'b'])
-        w_joined = api._PyString_Join(w_sep, w_seq)
+        w_joined = _PyString_Join(space, w_sep, w_seq)
         assert space.unwrap(w_joined) == 'a<sep>b'
-

From pypy.commits at gmail.com  Wed Jan 11 11:53:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 11 Jan 2017 08:53:44 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2464: give an __objclass__ to
 the __dict__ and __weakref__
Message-ID: <58766318.820bc30a.678ae.6c88@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89495:308258847565
Date: 2017-01-11 16:26 +0100
http://bitbucket.org/pypy/pypy/changeset/308258847565/

Log:	Issue #2464: give an __objclass__ to the __dict__ and __weakref__
	descriptors that are added to user-defined classes

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -221,10 +221,6 @@
     exec source.compile() in miniglobals
     return miniglobals['descr_typecheck_%s' % func.__name__]
 
-def unknown_objclass_getter(space):
-    # NB. this is an AttributeError to make inspect.py happy
-    raise oefmt(space.w_AttributeError, "generic property has no __objclass__")
-
 @specialize.arg(0)
 def make_objclass_getter(tag, func, cls):
     if func and hasattr(func, 'im_func'):
@@ -235,7 +231,7 @@
 @specialize.memo()
 def _make_objclass_getter(cls):
     if not cls:
-        return unknown_objclass_getter, cls
+        return None, cls
     miniglobals = {}
     if isinstance(cls, str):
         assert cls.startswith('<'), "pythontype typecheck should begin with <"
@@ -257,7 +253,7 @@
 
     @specialize.arg(7)
     def __init__(self, fget, fset=None, fdel=None, doc=None,
-                 cls=None, use_closure=False, tag=None):
+                 cls=None, use_closure=False, tag=None, w_type=None):
         objclass_getter, cls = make_objclass_getter(tag, fget, cls)
         fget = make_descr_typecheck_wrapper((tag, 0), fget,
                                             cls=cls, use_closure=use_closure)
@@ -272,6 +268,7 @@
         self.reqcls = cls
         self.name = '<generic property>'
         self.objclass_getter = objclass_getter
+        self.w_type = w_type
         self.use_closure = use_closure
 
     @unwrap_spec(w_cls = WrappedDefault(None))
@@ -322,7 +319,14 @@
                                                space.wrap(self.name)]))
 
     def descr_get_objclass(space, property):
-        return property.objclass_getter(space)
+        if property.w_type is not None:
+            return property.w_type
+        if property.objclass_getter is not None:
+            return property.objclass_getter(space)
+        # NB. this is an AttributeError to make inspect.py happy
+        raise oefmt(space.w_AttributeError,
+                    "generic property has no __objclass__")
+
 
 def interp_attrproperty(name, cls, doc=None):
     "NOT_RPYTHON: initialization-time only"
@@ -466,9 +470,13 @@
         return space.w_None
     return lifeline.get_any_weakref(space)
 
-dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
-                            doc="dictionary for instance variables (if defined)")
-dict_descr.name = '__dict__'
+def make_dict_descr_for_type(w_type):
+    descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
+                           w_type=w_type,
+                           doc="dictionary for instance variables")
+    descr.name = '__dict__'
+    return descr
+dict_descr = make_dict_descr_for_type(None)
 
 
 def generic_ne(space, w_obj1, w_obj2):
@@ -498,9 +506,12 @@
     w_docstring = code.getdocstring(space)
     return space.newtuple([w_docstring])
 
-weakref_descr = GetSetProperty(descr_get_weakref,
-                    doc="list of weak references to the object (if defined)")
-weakref_descr.name = '__weakref__'
+def make_weakref_descr_for_type(w_type):
+    descr = GetSetProperty(descr_get_weakref, w_type=w_type,
+                           doc="list of weak references to the object")
+    descr.name = '__weakref__'
+    return descr
+weakref_descr = make_weakref_descr_for_type(None)
 
 def make_weakref_descr(cls):
     """Make instances of the W_Root subclass 'cls' weakrefable.
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -4,7 +4,8 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import Function, StaticMethod
 from pypy.interpreter.typedef import (
-    weakref_descr, GetSetProperty, dict_descr, Member, TypeDef)
+    make_weakref_descr_for_type, GetSetProperty, weakref_descr,
+    make_dict_descr_for_type, Member, TypeDef)
 from pypy.interpreter.astcompiler.misc import mangle
 from pypy.module.__builtin__ import abstractinst
 
@@ -1117,12 +1118,14 @@
 
 def create_dict_slot(w_self):
     if not w_self.hasdict:
+        dict_descr = make_dict_descr_for_type(w_self)
         w_self.dict_w.setdefault('__dict__',
                                  w_self.space.wrap(dict_descr))
         w_self.hasdict = True
 
 def create_weakref_slot(w_self):
     if not w_self.weakrefable:
+        weakref_descr = make_weakref_descr_for_type(w_self)
         w_self.dict_w.setdefault('__weakref__',
                                  w_self.space.wrap(weakref_descr))
         w_self.weakrefable = True

From pypy.commits at gmail.com  Wed Jan 11 11:53:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 11 Jan 2017 08:53:46 -0800 (PST)
Subject: [pypy-commit] pypy default: Change the approach to fix translation,
 add a test
Message-ID: <5876631a.8c7e1c0a.ceabd.ce88@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89496:8ad2a82f6189
Date: 2017-01-11 17:04 +0000
http://bitbucket.org/pypy/pypy/changeset/8ad2a82f6189/

Log:	Change the approach to fix translation, add a test

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -8,6 +8,7 @@
 
 from rpython.rlib.jit import promote
 from rpython.rlib.objectmodel import compute_identity_hash, specialize
+from rpython.rlib.objectmodel import instantiate
 from rpython.tool.sourcetools import compile2, func_with_new_name
 
 
@@ -250,10 +251,12 @@
 
 class GetSetProperty(W_Root):
     _immutable_fields_ = ["fget", "fset", "fdel"]
+    name = '<generic property>'
+    w_objclass = None
 
     @specialize.arg(7)
     def __init__(self, fget, fset=None, fdel=None, doc=None,
-                 cls=None, use_closure=False, tag=None, w_type=None):
+                 cls=None, use_closure=False, tag=None):
         objclass_getter, cls = make_objclass_getter(tag, fget, cls)
         fget = make_descr_typecheck_wrapper((tag, 0), fget,
                                             cls=cls, use_closure=use_closure)
@@ -261,16 +264,25 @@
                                             cls=cls, use_closure=use_closure)
         fdel = make_descr_typecheck_wrapper((tag, 2), fdel,
                                             cls=cls, use_closure=use_closure)
+        self._init(fget, fset, fdel, doc, cls, objclass_getter, use_closure)
+
+    def _init(self, fget, fset, fdel, doc, cls, objclass_getter, use_closure):
         self.fget = fget
         self.fset = fset
         self.fdel = fdel
         self.doc = doc
         self.reqcls = cls
-        self.name = '<generic property>'
         self.objclass_getter = objclass_getter
-        self.w_type = w_type
         self.use_closure = use_closure
 
+    def copy_for_type(self, w_objclass):
+        new = instantiate(GetSetProperty)
+        new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
+                  None, self.use_closure)
+        new.name = self.name
+        new.w_objclass = w_objclass
+        return new
+
     @unwrap_spec(w_cls = WrappedDefault(None))
     def descr_property_get(self, space, w_obj, w_cls=None):
         """property.__get__(obj[, type]) -> value
@@ -319,8 +331,8 @@
                                                space.wrap(self.name)]))
 
     def descr_get_objclass(space, property):
-        if property.w_type is not None:
-            return property.w_type
+        if property.w_objclass is not None:
+            return property.w_objclass
         if property.objclass_getter is not None:
             return property.objclass_getter(space)
         # NB. this is an AttributeError to make inspect.py happy
@@ -470,13 +482,9 @@
         return space.w_None
     return lifeline.get_any_weakref(space)
 
-def make_dict_descr_for_type(w_type):
-    descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
-                           w_type=w_type,
-                           doc="dictionary for instance variables")
-    descr.name = '__dict__'
-    return descr
-dict_descr = make_dict_descr_for_type(None)
+dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
+                            doc="dictionary for instance variables")
+dict_descr.name = '__dict__'
 
 
 def generic_ne(space, w_obj1, w_obj2):
@@ -506,12 +514,9 @@
     w_docstring = code.getdocstring(space)
     return space.newtuple([w_docstring])
 
-def make_weakref_descr_for_type(w_type):
-    descr = GetSetProperty(descr_get_weakref, w_type=w_type,
-                           doc="list of weak references to the object")
-    descr.name = '__weakref__'
-    return descr
-weakref_descr = make_weakref_descr_for_type(None)
+weakref_descr = GetSetProperty(descr_get_weakref,
+                               doc="list of weak references to the object")
+weakref_descr.name = '__weakref__'
 
 def make_weakref_descr(cls):
     """Make instances of the W_Root subclass 'cls' weakrefable.
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1323,3 +1323,9 @@
         assert not self.compares_by_identity(X)
         del X.__eq__
         assert self.compares_by_identity(X)
+
+    def test_descriptor_objclass(self):
+        class X(object):
+            pass
+        assert X.__dict__['__dict__'].__objclass__ is X
+        assert X.__dict__['__weakref__'].__objclass__ is X
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -4,8 +4,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import Function, StaticMethod
 from pypy.interpreter.typedef import (
-    make_weakref_descr_for_type, GetSetProperty, weakref_descr,
-    make_dict_descr_for_type, Member, TypeDef)
+    weakref_descr, GetSetProperty, dict_descr, Member, TypeDef)
 from pypy.interpreter.astcompiler.misc import mangle
 from pypy.module.__builtin__ import abstractinst
 
@@ -1118,16 +1117,16 @@
 
 def create_dict_slot(w_self):
     if not w_self.hasdict:
-        dict_descr = make_dict_descr_for_type(w_self)
+        descr = dict_descr.copy_for_type(w_self)
         w_self.dict_w.setdefault('__dict__',
-                                 w_self.space.wrap(dict_descr))
+                                 w_self.space.wrap(descr))
         w_self.hasdict = True
 
 def create_weakref_slot(w_self):
     if not w_self.weakrefable:
-        weakref_descr = make_weakref_descr_for_type(w_self)
+        descr = weakref_descr.copy_for_type(w_self)
         w_self.dict_w.setdefault('__weakref__',
-                                 w_self.space.wrap(weakref_descr))
+                                 w_self.space.wrap(descr))
         w_self.weakrefable = True
 
 def valid_slot_name(slot_name):

From pypy.commits at gmail.com  Wed Jan 11 11:53:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 11 Jan 2017 08:53:48 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <5876631c.42061c0a.18fe5.dee5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89497:b0e1e683bc0e
Date: 2017-01-11 17:04 +0000
http://bitbucket.org/pypy/pypy/changeset/b0e1e683bc0e/

Log:	merge heads

diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -2,20 +2,14 @@
 from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
-from pypy.module.cpyext import api
+from pypy.module.cpyext.api import (
+    slot_function, cpython_api, copy_header_files, INTERPLEVEL_API,
+    Py_ssize_t, Py_ssize_tP, PyObject)
 from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest
-PyObject = api.PyObject
 from pypy.interpreter.error import OperationError
 from rpython.rlib import rawrefcount
 import os
 
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetWrapped(space, w_arg):
-    assert isinstance(w_arg, W_Root)
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetReference(space, arg):
-    assert lltype.typeOf(arg) ==  PyObject
-
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
         space = cls.space
@@ -35,7 +29,7 @@
             def __getattr__(self, name):
                 return getattr(cls.space, name)
         cls.api = CAPI()
-        CAPI.__dict__.update(api.INTERPLEVEL_API)
+        CAPI.__dict__.update(INTERPLEVEL_API)
 
         print 'DONT_FREE_ANY_MORE'
         rawrefcount._dont_free_any_more()
@@ -71,20 +65,28 @@
         if self.check_and_print_leaks():
             assert False, "Test leaks or loses object(s)."
 
- at api.cpython_api([api.Py_ssize_t], api.Py_ssize_t, error=-1)
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetWrapped(space, w_arg):
+    assert isinstance(w_arg, W_Root)
+
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetReference(space, arg):
+    assert lltype.typeOf(arg) ==  PyObject
+
+ at cpython_api([Py_ssize_t], Py_ssize_t, error=-1)
 def PyPy_TypedefTest1(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_t
+    assert lltype.typeOf(arg) == Py_ssize_t
     return 0
 
- at api.cpython_api([api.Py_ssize_tP], api.Py_ssize_tP)
+ at cpython_api([Py_ssize_tP], Py_ssize_tP)
 def PyPy_TypedefTest2(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_tP
+    assert lltype.typeOf(arg) == Py_ssize_tP
     return None
 
 class TestConversion(BaseApiTest):
-    def test_conversions(self, space, api):
-        api.PyPy_GetWrapped(space.w_None)
-        api.PyPy_GetReference(space.w_None)
+    def test_conversions(self, space):
+        PyPy_GetWrapped(space, space.w_None)
+        PyPy_GetReference(space, space.w_None)
 
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
@@ -95,7 +97,7 @@
         assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Py_ssize_t *arg0'
 
         PyPy_TypedefTest1(space, 0)
-        ppos = lltype.malloc(api.Py_ssize_tP.TO, 1, flavor='raw')
+        ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
         ppos[0] = 0
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
@@ -103,7 +105,7 @@
 @pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
-    api.copy_header_files(tmpdir, True)
+    copy_header_files(tmpdir, True)
     def check(name):
         f = tmpdir.join(name)
         assert f.check(file=True)
diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py
--- a/pypy/module/cpyext/test/test_boolobject.py
+++ b/pypy/module/cpyext/test/test_boolobject.py
@@ -1,20 +1,22 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong
+from pypy.module.cpyext.floatobject import PyFloat_FromDouble
 
 class TestBoolObject(BaseApiTest):
-    def test_fromlong(self, space, api):
+    def test_fromlong(self, space):
         for i in range(-3, 3):
-            obj = api.PyBool_FromLong(i)
+            obj = PyBool_FromLong(space, i)
             if i:
                 assert obj is space.w_True
             else:
                 assert obj is space.w_False
 
-    def test_check(self, space, api):
-        assert api.PyBool_Check(space.w_True)
-        assert api.PyBool_Check(space.w_False)
-        assert not api.PyBool_Check(space.w_None)
-        assert not api.PyBool_Check(api.PyFloat_FromDouble(1.0))
+    def test_check(self, space):
+        assert PyBool_Check(space, space.w_True)
+        assert PyBool_Check(space, space.w_False)
+        assert not PyBool_Check(space, space.w_None)
+        assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0))
 
 class AppTestBoolMacros(AppTestCpythonExtensionBase):
     def test_macros(self):
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -2,13 +2,14 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject
+from pypy.module.cpyext.bytesobject import (
+    new_empty_str, PyBytesObject, _PyString_Resize, PyString_Concat,
+    PyString_ConcatAndDel, PyString_Format, PyString_InternFromString,
+    PyString_AsEncodedObject, PyString_AsDecodedObject, _PyString_Eq,
+    _PyString_Join)
 from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref
-from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
 
-import py
-import sys
 
 class AppTestBytesObject(AppTestCpythonExtensionBase):
     def test_bytesobject(self):
@@ -441,21 +442,21 @@
         assert a == 'abc'
 
 class TestBytes(BaseApiTest):
-    def test_bytes_resize(self, space, api):
+    def test_bytes_resize(self, space):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         py_str.c_ob_sval[0] = 'a'
         py_str.c_ob_sval[1] = 'b'
         py_str.c_ob_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
-        api._PyString_Resize(ar, 3)
+        _PyString_Resize(space, ar, 3)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 3
         assert py_str.c_ob_sval[1] == 'b'
         assert py_str.c_ob_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
-        api._PyString_Resize(ar, 10)
+        _PyString_Resize(space, ar, 10)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 10
         assert py_str.c_ob_sval[1] == 'b'
@@ -463,7 +464,7 @@
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
-    def test_string_buffer(self, space, api):
+    def test_string_buffer(self, space):
         py_str = new_empty_str(space, 10)
         c_buf = py_str.c_ob_type.c_tp_as_buffer
         assert c_buf
@@ -486,36 +487,36 @@
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref
         prev_refcnt = ref.c_ob_refcnt
-        api.PyString_Concat(ptr, space.wrap('def'))
+        PyString_Concat(space, ptr, space.wrap('def'))
         assert ref.c_ob_refcnt == prev_refcnt - 1
         assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
         api.PyString_Concat(ptr, space.w_None)
         assert not ptr[0]
         api.PyErr_Clear()
         ptr[0] = lltype.nullptr(PyObject.TO)
-        api.PyString_Concat(ptr, space.wrap('def')) # should not crash
+        PyString_Concat(space, ptr, space.wrap('def')) # should not crash
         lltype.free(ptr, flavor='raw')
 
-    def test_ConcatAndDel(self, space, api):
+    def test_ConcatAndDel(self, space):
         ref1 = make_ref(space, space.wrap('abc'))
         ref2 = make_ref(space, space.wrap('def'))
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref1
         prev_refcnf = ref2.c_ob_refcnt
-        api.PyString_ConcatAndDel(ptr, ref2)
+        PyString_ConcatAndDel(space, ptr, ref2)
         assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
         assert ref2.c_ob_refcnt == prev_refcnf - 1
         Py_DecRef(space, ptr[0])
         ptr[0] = lltype.nullptr(PyObject.TO)
         ref2 = make_ref(space, space.wrap('foo'))
         prev_refcnf = ref2.c_ob_refcnt
-        api.PyString_ConcatAndDel(ptr, ref2) # should not crash
+        PyString_ConcatAndDel(space, ptr, ref2) # should not crash
         assert ref2.c_ob_refcnt == prev_refcnf - 1
         lltype.free(ptr, flavor='raw')
 
-    def test_format(self, space, api):
+    def test_format(self, space):
         assert "1 2" == space.unwrap(
-            api.PyString_Format(space.wrap('%s %d'), space.wrap((1, 2))))
+            PyString_Format(space, space.wrap('%s %d'), space.wrap((1, 2))))
 
     def test_asbuffer(self, space, api):
         bufp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
@@ -530,12 +531,12 @@
         assert rffi.charp2str(bufp[0]) == 'text'
         lltype.free(bufp, flavor='raw')
         lltype.free(lenp, flavor='raw')
-        api.Py_DecRef(ref)
+        Py_DecRef(space, ref)
 
-    def test_intern(self, space, api):
+    def test_intern(self, space):
         buf = rffi.str2charp("test")
-        w_s1 = api.PyString_InternFromString(buf)
-        w_s2 = api.PyString_InternFromString(buf)
+        w_s1 = PyString_InternFromString(space, buf)
+        w_s2 = PyString_InternFromString(space, buf)
         rffi.free_charp(buf)
         assert w_s1 is w_s2
 
@@ -545,11 +546,10 @@
         errors = rffi.str2charp("strict")
 
         encoding = rffi.str2charp("hex")
-        res = api.PyString_AsEncodedObject(
-            ptr, encoding, errors)
+        res = PyString_AsEncodedObject(space, ptr, encoding, errors)
         assert space.unwrap(res) == "616263"
 
-        res = api.PyString_AsEncodedObject(
+        res = PyString_AsEncodedObject(space,
             ptr, encoding, lltype.nullptr(rffi.CCHARP.TO))
         assert space.unwrap(res) == "616263"
         rffi.free_charp(encoding)
@@ -561,28 +561,30 @@
 
         rffi.free_charp(errors)
 
-        res = api.PyString_AsEncodedObject(
-            ptr, lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO))
+        res = PyString_AsEncodedObject(
+            space, ptr, lltype.nullptr(rffi.CCHARP.TO),
+            lltype.nullptr(rffi.CCHARP.TO))
         assert space.unwrap(res) == "abc"
 
         self.raises(space, api, TypeError, api.PyString_AsEncodedObject,
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
-    def test_AsDecodedObject(self, space, api):
+    def test_AsDecodedObject(self, space):
         w_str = space.wrap('caf\xe9')
         encoding = rffi.str2charp("latin-1")
-        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        w_res = PyString_AsDecodedObject(space, w_str, encoding, None)
         rffi.free_charp(encoding)
         assert space.unwrap(w_res) == u"caf\xe9"
 
-    def test_eq(self, space, api):
-        assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
-        assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
+    def test_eq(self, space):
+        assert 1 == _PyString_Eq(
+            space, space.wrap("hello"), space.wrap("hello"))
+        assert 0 == _PyString_Eq(
+            space, space.wrap("hello"), space.wrap("world"))
 
-    def test_join(self, space, api):
+    def test_join(self, space):
         w_sep = space.wrap('<sep>')
         w_seq = space.wrap(['a', 'b'])
-        w_joined = api._PyString_Join(w_sep, w_seq)
+        w_joined = _PyString_Join(space, w_sep, w_seq)
         assert space.unwrap(w_joined) == 'a<sep>b'
-

From pypy.commits at gmail.com  Wed Jan 11 12:05:10 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 11 Jan 2017 09:05:10 -0800 (PST)
Subject: [pypy-commit] pypy default: Add replacement for BaseApiTest.raises()
Message-ID: <587665c6.581d1c0a.2903b.ec31@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89498:3e0005c54841
Date: 2017-01-11 17:04 +0000
http://bitbucket.org/pypy/pypy/changeset/3e0005c54841/

Log:	Add replacement for BaseApiTest.raises()

diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,4 +1,5 @@
 import py, pytest
+import contextlib
 from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
@@ -10,6 +11,13 @@
 from rpython.rlib import rawrefcount
 import os
 
+ at contextlib.contextmanager
+def raises_w(space, expected_exc):
+    with pytest.raises(OperationError) as excinfo:
+        yield
+    operror = excinfo.value
+    assert operror.w_type is getattr(space, 'w_' + expected_exc.__name__)
+
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
         space = cls.space
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -1,6 +1,8 @@
 # encoding: utf-8
+import pytest
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.interpreter.error import OperationError
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.bytesobject import (
     new_empty_str, PyBytesObject, _PyString_Resize, PyString_Concat,
@@ -9,6 +11,7 @@
     _PyString_Join)
 from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref
+from pypy.module.cpyext.object import PyObject_AsCharBuffer
 
 
 class AppTestBytesObject(AppTestCpythonExtensionBase):
@@ -482,7 +485,7 @@
         lltype.free(ref, flavor='raw')
         Py_DecRef(space, py_obj)
 
-    def test_Concat(self, space, api):
+    def test_Concat(self, space):
         ref = make_ref(space, space.wrap('abc'))
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref
@@ -490,9 +493,9 @@
         PyString_Concat(space, ptr, space.wrap('def'))
         assert ref.c_ob_refcnt == prev_refcnt - 1
         assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
-        api.PyString_Concat(ptr, space.w_None)
+        with pytest.raises(OperationError):
+            PyString_Concat(space, ptr, space.w_None)
         assert not ptr[0]
-        api.PyErr_Clear()
         ptr[0] = lltype.nullptr(PyObject.TO)
         PyString_Concat(space, ptr, space.wrap('def')) # should not crash
         lltype.free(ptr, flavor='raw')
@@ -518,14 +521,14 @@
         assert "1 2" == space.unwrap(
             PyString_Format(space, space.wrap('%s %d'), space.wrap((1, 2))))
 
-    def test_asbuffer(self, space, api):
+    def test_asbuffer(self, space):
         bufp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
 
         w_text = space.wrap("text")
         ref = make_ref(space, w_text)
         prev_refcnt = ref.c_ob_refcnt
-        assert api.PyObject_AsCharBuffer(ref, bufp, lenp) == 0
+        assert PyObject_AsCharBuffer(space, ref, bufp, lenp) == 0
         assert ref.c_ob_refcnt == prev_refcnt
         assert lenp[0] == 4
         assert rffi.charp2str(bufp[0]) == 'text'
@@ -540,7 +543,7 @@
         rffi.free_charp(buf)
         assert w_s1 is w_s2
 
-    def test_AsEncodedObject(self, space, api):
+    def test_AsEncodedObject(self, space):
         ptr = space.wrap('abc')
 
         errors = rffi.str2charp("strict")
@@ -555,20 +558,17 @@
         rffi.free_charp(encoding)
 
         encoding = rffi.str2charp("unknown_encoding")
-        self.raises(space, api, LookupError, api.PyString_AsEncodedObject,
-                    ptr, encoding, errors)
+        with raises_w(space, LookupError):
+            PyString_AsEncodedObject(space, ptr, encoding, errors)
         rffi.free_charp(encoding)
 
         rffi.free_charp(errors)
 
-        res = PyString_AsEncodedObject(
-            space, ptr, lltype.nullptr(rffi.CCHARP.TO),
-            lltype.nullptr(rffi.CCHARP.TO))
+        NULL = lltype.nullptr(rffi.CCHARP.TO)
+        res = PyString_AsEncodedObject(space, ptr, NULL, NULL)
         assert space.unwrap(res) == "abc"
-
-        self.raises(space, api, TypeError, api.PyString_AsEncodedObject,
-            space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
-        )
+        with raises_w(space, TypeError):
+            PyString_AsEncodedObject(space, space.wrap(2), NULL, NULL)
 
     def test_AsDecodedObject(self, space):
         w_str = space.wrap('caf\xe9')

From pypy.commits at gmail.com  Wed Jan 11 12:41:58 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 11 Jan 2017 09:41:58 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove more uses of api.XXX magic
Message-ID: <58766e66.2717c30a.b5839.a905@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89499:77a7751e0e13
Date: 2017-01-11 17:41 +0000
http://bitbucket.org/pypy/pypy/changeset/77a7751e0e13/

Log:	Remove more uses of api.XXX magic

diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -118,12 +118,16 @@
                     """ % (type_name,)))
         except OperationError:
             return 0
+    return check, check_exact
 
-make_check_function("PyDateTime_Check", "datetime")
-make_check_function("PyDate_Check", "date")
-make_check_function("PyTime_Check", "time")
-make_check_function("PyDelta_Check", "timedelta")
-make_check_function("PyTZInfo_Check", "tzinfo")
+PyDateTime_Check, PyDateTime_CheckExact = make_check_function(
+    "PyDateTime_Check", "datetime")
+PyDate_Check, PyDate_CheckExact = make_check_function("PyDate_Check", "date")
+PyTime_Check, PyTime_CheckExact = make_check_function("PyTime_Check", "time")
+PyDelta_Check, PyDelta_CheckExact = make_check_function(
+    "PyDelta_Check", "timedelta")
+PyTZInfo_Check, PyTZInfo_CheckExact = make_check_function(
+    "PyTZInfo_Check", "tzinfo")
 
 # Constructors. They are better used as macros.
 
diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py
--- a/pypy/module/cpyext/test/test_classobject.py
+++ b/pypy/module/cpyext/test/test_classobject.py
@@ -1,9 +1,13 @@
+from pypy.interpreter.function import Function
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.interpreter.function import Function, Method
+from pypy.module.cpyext.classobject import (
+    PyClass_Check, PyClass_New, PyInstance_Check, PyInstance_New,
+    PyInstance_NewRaw, _PyInstance_Lookup)
+from pypy.module.cpyext.object import PyObject_GetAttr
 
 class TestClassObject(BaseApiTest):
-    def test_newinstance(self, space, api):
+    def test_newinstance(self, space):
         w_class = space.appexec([], """():
             class C:
                 x = None
@@ -14,23 +18,23 @@
             return C
         """)
 
-        assert api.PyClass_Check(w_class)
+        assert PyClass_Check(space, w_class)
 
-        w_instance = api.PyInstance_NewRaw(w_class, None)
-        assert api.PyInstance_Check(w_instance)
+        w_instance = PyInstance_NewRaw(space, w_class, None)
+        assert PyInstance_Check(space, w_instance)
         assert space.getattr(w_instance, space.wrap('x')) is space.w_None
 
-        w_instance = api.PyInstance_NewRaw(w_class, space.wrap(dict(a=3)))
+        w_instance = PyInstance_NewRaw(space, w_class, space.wrap(dict(a=3)))
         assert space.getattr(w_instance, space.wrap('x')) is space.w_None
         assert space.unwrap(space.getattr(w_instance, space.wrap('a'))) == 3
 
-        w_instance = api.PyInstance_New(w_class,
+        w_instance = PyInstance_New(space, w_class,
                                         space.wrap((3,)), space.wrap(dict(y=2)))
         assert space.unwrap(space.getattr(w_instance, space.wrap('x'))) == 1
         assert space.unwrap(space.getattr(w_instance, space.wrap('y'))) == 2
         assert space.unwrap(space.getattr(w_instance, space.wrap('args'))) == (3,)
 
-    def test_lookup(self, space, api):
+    def test_lookup(self, space):
         w_instance = space.appexec([], """():
             class C:
                 def __init__(self):
@@ -39,25 +43,26 @@
             return C()
         """)
 
-        assert api.PyInstance_Check(w_instance)
-        assert api.PyObject_GetAttr(w_instance, space.wrap('x')) is space.w_None
-        assert api._PyInstance_Lookup(w_instance, space.wrap('x')) is space.w_None
-        assert api._PyInstance_Lookup(w_instance, space.wrap('y')) is None
-        assert not api.PyErr_Occurred()
+        assert PyInstance_Check(space, w_instance)
+        assert PyObject_GetAttr(space, w_instance, space.wrap('x')) is space.w_None
+        assert _PyInstance_Lookup(space, w_instance, space.wrap('x')) is space.w_None
+        assert _PyInstance_Lookup(space, w_instance, space.wrap('y')) is None
 
         # getattr returns a bound method
-        assert not isinstance(api.PyObject_GetAttr(w_instance, space.wrap('f')), Function)
+        assert not isinstance(
+            PyObject_GetAttr(space, w_instance, space.wrap('f')), Function)
         # _PyInstance_Lookup returns the raw descriptor
-        assert isinstance(api._PyInstance_Lookup(w_instance, space.wrap('f')), Function)
+        assert isinstance(
+            _PyInstance_Lookup(space, w_instance, space.wrap('f')), Function)
 
-    def test_pyclass_new(self, space, api):
+    def test_pyclass_new(self, space):
         w_bases = space.newtuple([])
         w_dict = space.newdict()
         w_name = space.wrap("C")
-        w_class = api.PyClass_New(w_bases, w_dict, w_name)
+        w_class = PyClass_New(space, w_bases, w_dict, w_name)
         assert not space.isinstance_w(w_class, space.w_type)
         w_instance = space.call_function(w_class)
-        assert api.PyInstance_Check(w_instance)
+        assert PyInstance_Check(space, w_instance)
         assert space.is_true(space.call_method(space.builtin, "isinstance",
                                                w_instance, w_class))
 
@@ -69,5 +74,6 @@
                  Py_INCREF(&PyClass_Type);
                  return (PyObject*)&PyClass_Type;
              """)])
-        class C: pass
+        class C:
+            pass
         assert module.get_classtype() is type(C)
diff --git a/pypy/module/cpyext/test/test_codecs.py b/pypy/module/cpyext/test/test_codecs.py
--- a/pypy/module/cpyext/test/test_codecs.py
+++ b/pypy/module/cpyext/test/test_codecs.py
@@ -1,14 +1,15 @@
 # encoding: iso-8859-15
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.codecs import (
+    PyCodec_IncrementalEncoder, PyCodec_IncrementalDecoder)
 
 class TestCodecs(BaseApiTest):
-    def test_incremental(self, space, api):
+    def test_incremental(self, space):
         utf8 = rffi.str2charp('utf-8')
-        w_encoder = api.PyCodec_IncrementalEncoder(utf8, None)
+        w_encoder = PyCodec_IncrementalEncoder(space, utf8, None)
         w_encoded = space.call_method(w_encoder, 'encode', space.wrap(u'sp&#228;m'))
-        w_decoder = api.PyCodec_IncrementalDecoder(utf8, None)
+        w_decoder = PyCodec_IncrementalDecoder(space, utf8, None)
         w_decoded = space.call_method(w_decoder, 'decode', w_encoded)
         assert space.unwrap(w_decoded) == u'sp&#228;m'
         rffi.free_charp(utf8)
-
diff --git a/pypy/module/cpyext/test/test_complexobject.py b/pypy/module/cpyext/test/test_complexobject.py
--- a/pypy/module/cpyext/test/test_complexobject.py
+++ b/pypy/module/cpyext/test/test_complexobject.py
@@ -1,23 +1,23 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.complexobject import (
+    PyComplex_FromDoubles, PyComplex_RealAsDouble, PyComplex_ImagAsDouble)
 
 class TestComplexObject(BaseApiTest):
-    def test_complexobject(self, space, api):
-        w_value = api.PyComplex_FromDoubles(1.2, 3.4)
+    def test_complexobject(self, space):
+        w_value = PyComplex_FromDoubles(space, 1.2, 3.4)
         assert space.unwrap(w_value) == 1.2+3.4j
-        assert api.PyComplex_RealAsDouble(w_value) == 1.2
-        assert api.PyComplex_ImagAsDouble(w_value) == 3.4
+        assert PyComplex_RealAsDouble(space, w_value) == 1.2
+        assert PyComplex_ImagAsDouble(space, w_value) == 3.4
 
-        assert api.PyComplex_RealAsDouble(space.wrap(42)) == 42
-        assert api.PyComplex_RealAsDouble(space.wrap(1.5)) == 1.5
-        assert api.PyComplex_ImagAsDouble(space.wrap(1.5)) == 0.0
+        assert PyComplex_RealAsDouble(space, space.wrap(42)) == 42
+        assert PyComplex_RealAsDouble(space, space.wrap(1.5)) == 1.5
+        assert PyComplex_ImagAsDouble(space, space.wrap(1.5)) == 0.0
 
         # cpython accepts anything for PyComplex_ImagAsDouble
-        assert api.PyComplex_ImagAsDouble(space.w_None) == 0.0
-        assert not api.PyErr_Occurred()
-        assert api.PyComplex_RealAsDouble(space.w_None) == -1.0
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
+        assert PyComplex_ImagAsDouble(space, space.w_None) == 0.0
+        with raises_w(space, TypeError):
+            PyComplex_RealAsDouble(space, space.w_None)
 
 class AppTestCComplex(AppTestCpythonExtensionBase):
     def test_AsCComplex(self):
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -1,92 +1,96 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.cdatetime import *
+from pypy.module.cpyext.cdatetime import (
+    _PyDateTime_Import, _PyDateTime_FromDateAndTime, _PyDate_FromDate,
+    _PyTime_FromTime, _PyDelta_FromDelta)
 import datetime
 
 class TestDatetime(BaseApiTest):
-    def test_date(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_date = api._PyDate_FromDate(2010, 06, 03, date_api.c_DateType)
+    def test_date(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_date = _PyDate_FromDate(space, 2010, 06, 03, date_api.c_DateType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03'
 
-        assert api.PyDate_Check(w_date)
-        assert api.PyDate_CheckExact(w_date)
+        assert PyDate_Check(space, w_date)
+        assert PyDate_CheckExact(space, w_date)
 
-        assert api.PyDateTime_GET_YEAR(w_date) == 2010
-        assert api.PyDateTime_GET_MONTH(w_date) == 6
-        assert api.PyDateTime_GET_DAY(w_date) == 3
+        assert PyDateTime_GET_YEAR(space, w_date) == 2010
+        assert PyDateTime_GET_MONTH(space, w_date) == 6
+        assert PyDateTime_GET_DAY(space, w_date) == 3
 
-    def test_time(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_time = api._PyTime_FromTime(23, 15, 40, 123456,
-                                      space.w_None, date_api.c_TimeType)
+    def test_time(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_time = _PyTime_FromTime(
+            space, 23, 15, 40, 123456, space.w_None, date_api.c_TimeType)
         assert space.unwrap(space.str(w_time)) == '23:15:40.123456'
 
-        assert api.PyTime_Check(w_time)
-        assert api.PyTime_CheckExact(w_time)
+        assert PyTime_Check(space, w_time)
+        assert PyTime_CheckExact(space, w_time)
 
-        assert api.PyDateTime_TIME_GET_HOUR(w_time) == 23
-        assert api.PyDateTime_TIME_GET_MINUTE(w_time) == 15
-        assert api.PyDateTime_TIME_GET_SECOND(w_time) == 40
-        assert api.PyDateTime_TIME_GET_MICROSECOND(w_time) == 123456
+        assert PyDateTime_TIME_GET_HOUR(space, w_time) == 23
+        assert PyDateTime_TIME_GET_MINUTE(space, w_time) == 15
+        assert PyDateTime_TIME_GET_SECOND(space, w_time) == 40
+        assert PyDateTime_TIME_GET_MICROSECOND(space, w_time) == 123456
 
-    def test_datetime(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_date = api._PyDateTime_FromDateAndTime(
-            2010, 06, 03, 23, 15, 40, 123456,
-            space.w_None, date_api.c_DateTimeType)
+    def test_datetime(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_date = _PyDateTime_FromDateAndTime(
+            space, 2010, 06, 03, 23, 15, 40, 123456, space.w_None,
+            date_api.c_DateTimeType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03 23:15:40.123456'
 
-        assert api.PyDateTime_Check(w_date)
-        assert api.PyDateTime_CheckExact(w_date)
-        assert api.PyDate_Check(w_date)
-        assert not api.PyDate_CheckExact(w_date)
+        assert PyDateTime_Check(space, w_date)
+        assert PyDateTime_CheckExact(space, w_date)
+        assert PyDate_Check(space, w_date)
+        assert not PyDate_CheckExact(space, w_date)
 
-        assert api.PyDateTime_GET_YEAR(w_date) == 2010
-        assert api.PyDateTime_GET_MONTH(w_date) == 6
-        assert api.PyDateTime_GET_DAY(w_date) == 3
-        assert api.PyDateTime_DATE_GET_HOUR(w_date) == 23
-        assert api.PyDateTime_DATE_GET_MINUTE(w_date) == 15
-        assert api.PyDateTime_DATE_GET_SECOND(w_date) == 40
-        assert api.PyDateTime_DATE_GET_MICROSECOND(w_date) == 123456
+        assert PyDateTime_GET_YEAR(space, w_date) == 2010
+        assert PyDateTime_GET_MONTH(space, w_date) == 6
+        assert PyDateTime_GET_DAY(space, w_date) == 3
+        assert PyDateTime_DATE_GET_HOUR(space, w_date) == 23
+        assert PyDateTime_DATE_GET_MINUTE(space, w_date) == 15
+        assert PyDateTime_DATE_GET_SECOND(space, w_date) == 40
+        assert PyDateTime_DATE_GET_MICROSECOND(space, w_date) == 123456
 
-    def test_delta(self, space, api):
-        date_api = api._PyDateTime_Import()
+    def test_delta(self, space):
+        date_api = _PyDateTime_Import(space)
         w_delta = space.appexec(
             [space.wrap(3), space.wrap(15)], """(days, seconds):
             from datetime import timedelta
             return timedelta(days, seconds)
         """)
-        assert api.PyDelta_Check(w_delta)
-        assert api.PyDelta_CheckExact(w_delta)
+        assert PyDelta_Check(space, w_delta)
+        assert PyDelta_CheckExact(space, w_delta)
 
-        w_delta = api._PyDelta_FromDelta(10, 20, 30, True, date_api.c_DeltaType)
-        assert api.PyDelta_Check(w_delta)
-        assert api.PyDelta_CheckExact(w_delta)
+        w_delta = _PyDelta_FromDelta(space, 10, 20, 30, True, date_api.c_DeltaType)
+        assert PyDelta_Check(space, w_delta)
+        assert PyDelta_CheckExact(space, w_delta)
 
-        assert api.PyDateTime_DELTA_GET_DAYS(w_delta) == 10
-        assert api.PyDateTime_DELTA_GET_SECONDS(w_delta) == 20
-        assert api.PyDateTime_DELTA_GET_MICROSECONDS(w_delta) == 30
+        assert PyDateTime_DELTA_GET_DAYS(space, w_delta) == 10
+        assert PyDateTime_DELTA_GET_SECONDS(space, w_delta) == 20
+        assert PyDateTime_DELTA_GET_MICROSECONDS(space, w_delta) == 30
 
-    def test_fromtimestamp(self, space, api):
+    def test_fromtimestamp(self, space):
         w_args = space.wrap((0,))
-        w_date = api.PyDate_FromTimestamp(w_args)
+        w_date = PyDate_FromTimestamp(space, w_args)
         date = datetime.date.fromtimestamp(0)
         assert space.unwrap(space.str(w_date)) == str(date)
 
         w_args = space.wrap((0,))
-        w_date = api.PyDateTime_FromTimestamp(w_args)
+        w_date = PyDateTime_FromTimestamp(space, w_args)
         date = datetime.datetime.fromtimestamp(0)
         assert space.unwrap(space.str(w_date)) == str(date)
 
-    def test_tzinfo(self, space, api):
+    def test_tzinfo(self, space):
         w_tzinfo = space.appexec(
             [], """():
             from datetime import tzinfo
             return tzinfo()
         """)
-        assert api.PyTZInfo_Check(w_tzinfo)
-        assert api.PyTZInfo_CheckExact(w_tzinfo)
-        assert not api.PyTZInfo_Check(space.w_None)
+        assert PyTZInfo_Check(space, w_tzinfo)
+        assert PyTZInfo_CheckExact(space, w_tzinfo)
+        assert not PyTZInfo_Check(space, space.w_None)
 
 class AppTestDatetime(AppTestCpythonExtensionBase):
     def test_CAPI(self):

From pypy.commits at gmail.com  Wed Jan 11 13:43:57 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 11 Jan 2017 10:43:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58767ced.8c7e1c0a.ceabd.035f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89500:d33af2cf0f9f
Date: 2017-01-11 18:43 +0000
http://bitbucket.org/pypy/pypy/changeset/d33af2cf0f9f/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -98,6 +98,7 @@
 is readonly) without pinning it.
 
 .. branch: cpyext-cleanup
+.. branch: api_func-refactor
 
 Refactor cpyext initialisation.
 
@@ -105,3 +106,7 @@
 
 Fix a test failure introduced by strbuf-as-buffer
 
+.. branch: cpyext-FromBuffer
+
+Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to
+the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak.
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -23,6 +23,10 @@
 def pytest_ignore_collect(path, config):
     if py.path.local.sysfind('genreflex') is None and config.option.runappdirect:
         return True          # "can't run dummy tests in -A"
+    if disabled:
+        return True
+
+disabled = None
 
 def pytest_configure(config):
     if py.path.local.sysfind('genreflex') is None:
@@ -37,7 +41,7 @@
             # build dummy backend (which has reflex info and calls hard-wired)
             import os
             from rpython.translator.tool.cbuild import ExternalCompilationInfo
-            from rpython.translator.platform import platform
+            from rpython.translator.platform import platform, CompilationError
             from rpython.translator import cdir
 
             from rpython.rtyper.lltypesystem import rffi
@@ -55,9 +59,16 @@
                 use_cpp_linker=True,
             )
 
-            soname = platform.compile(
-                [], eci,
-                outputfilename='libcppyy_dummy_backend',
-                standalone=False)
+            try:
+                soname = platform.compile(
+                    [], eci,
+                    outputfilename='libcppyy_dummy_backend',
+                    standalone=False)
+            except CompilationError as e:
+                if '-std=c++11' in str(e):
+                    global disabled
+                    disabled = str(e)
+                    return
+                raise
 
             lcapi.reflection_library = str(soname)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -124,6 +124,7 @@
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_MAX_NDIMS
 Py_CLEANUP_SUPPORTED
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -248,14 +249,13 @@
 cpyext_namespace = NameManager('cpyext_')
 
 class ApiFunction(object):
-    def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED,
+    def __init__(self, argtypes, restype, callable, error=CANNOT_FAIL,
                  c_name=None, gil=None, result_borrowed=False, result_is_ll=False):
         self.argtypes = argtypes
         self.restype = restype
         self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
         self.callable = callable
-        if error is not _NOT_SPECIFIED:
-            self.error_value = error
+        self.error_value = error
         self.c_name = c_name
 
         # extract the signature from the (CPython-level) code object
@@ -300,7 +300,7 @@
 
         argtypesw = zip(self.argtypes,
                         [_name.startswith("w_") for _name in self.argnames])
-        error_value = getattr(self, "error_value", CANNOT_FAIL)
+        error_value = self.error_value
         if (isinstance(self.restype, lltype.Ptr)
                 and error_value is not CANNOT_FAIL):
             assert lltype.typeOf(error_value) == self.restype
@@ -332,66 +332,19 @@
             wrapper.c_name = cpyext_namespace.uniquename(self.c_name)
         return wrapper
 
-DEFAULT_HEADER = 'pypy_decl.h'
-def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
-                gil=None, result_borrowed=False, result_is_ll=False):
-    """
-    Declares a function to be exported.
-    - `argtypes`, `restype` are lltypes and describe the function signature.
-    - `error` is the value returned when an applevel exception is raised. The
-      special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual
-      exception into a wrapped SystemError.  Unwrapped exceptions also cause a
-      SytemError.
-    - `header` is the header file to export the function in, Set to None to get
-      a C function pointer, but not exported by the API headers.
-    - set `gil` to "acquire", "release" or "around" to acquire the GIL,
-      release the GIL, or both
-    """
-    if isinstance(restype, lltype.Typedef):
-        real_restype = restype.OF
-    else:
-        real_restype = restype
-
-    if error is _NOT_SPECIFIED:
-        if isinstance(real_restype, lltype.Ptr):
-            error = lltype.nullptr(real_restype.TO)
-        elif real_restype is lltype.Void:
-            error = CANNOT_FAIL
-    if type(error) is int:
-        error = rffi.cast(real_restype, error)
-    expect_integer = (isinstance(real_restype, lltype.Primitive) and
-                      rffi.cast(restype, 0) == 0)
-
-    def decorate(func):
-        func._always_inline_ = 'try'
-        func_name = func.func_name
-        if header is not None:
-            c_name = None
-            if func_name in FUNCTIONS_BY_HEADER[header]:
-                raise ValueError("%s already registered" % func_name)
-        else:
-            c_name = func_name
-        api_function = ApiFunction(argtypes, restype, func, error,
-                                   c_name=c_name, gil=gil,
-                                   result_borrowed=result_borrowed,
-                                   result_is_ll=result_is_ll)
-        func.api_func = api_function
-
-        if error is _NOT_SPECIFIED:
-            raise ValueError("function %s has no return value for exceptions"
-                             % func)
-        names = api_function.argnames
-        types_names_enum_ui = unrolling_iterable(enumerate(
-            zip(api_function.argtypes,
-                [tp_name.startswith("w_") for tp_name in names])))
+    def get_unwrapper(self):
+        names = self.argnames
+        argtypesw = zip(self.argtypes,
+                        [_name.startswith("w_") for _name in self.argnames])
+        types_names_enum_ui = unrolling_iterable(enumerate(argtypesw))
 
         @specialize.ll()
         def unwrapper(space, *args):
-            from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj
+            from pypy.module.cpyext.pyobject import is_pyobj
             from pypy.module.cpyext.pyobject import from_ref, as_pyobj
             newargs = ()
             keepalives = ()
-            assert len(args) == len(api_function.argtypes)
+            assert len(args) == len(self.argtypes)
             for i, (ARG, is_wrapped) in types_names_enum_ui:
                 input_arg = args[i]
                 if is_PyObject(ARG) and not is_wrapped:
@@ -416,33 +369,81 @@
                         arg = from_ref(space, input_arg)
                     else:
                         arg = input_arg
-
-                        ## ZZZ: for is_pyobj:
-                        ## try:
-                        ##     arg = from_ref(space,
-                        ##                rffi.cast(PyObject, input_arg))
-                        ## except TypeError, e:
-                        ##     err = oefmt(space.w_TypeError,
-                        ##                 "could not cast arg to PyObject")
-                        ##     if not catch_exception:
-                        ##         raise err
-                        ##     state = space.fromcache(State)
-                        ##     state.set_exception(err)
-                        ##     if is_PyObject(restype):
-                        ##         return None
-                        ##     else:
-                        ##         return api_function.error_value
                 else:
                     # arg is not declared as PyObject, no magic
                     arg = input_arg
                 newargs += (arg, )
             try:
-                return func(space, *newargs)
+                return self.callable(space, *newargs)
             finally:
                 keepalive_until_here(*keepalives)
+        return unwrapper
 
-        unwrapper.func = func
-        unwrapper.api_func = api_function
+    def get_c_restype(self, c_writer):
+        return c_writer.gettype(self.restype).replace('@', '').strip()
+
+    def get_c_args(self, c_writer):
+        args = []
+        for i, argtype in enumerate(self.argtypes):
+            if argtype is CONST_STRING:
+                arg = 'const char *@'
+            elif argtype is CONST_STRINGP:
+                arg = 'const char **@'
+            elif argtype is CONST_WSTRING:
+                arg = 'const wchar_t *@'
+            else:
+                arg = c_writer.gettype(argtype)
+            arg = arg.replace('@', 'arg%d' % (i,)).strip()
+            args.append(arg)
+        args = ', '.join(args) or "void"
+        return args
+
+    def get_api_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "PyAPI_FUNC({restype}) {name}({args});".format(**locals())
+
+    def get_ptr_decl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        return "{restype} (*{name})({args});".format(**locals())
+
+    def get_ctypes_impl(self, name, c_writer):
+        restype = self.get_c_restype(c_writer)
+        args = self.get_c_args(c_writer)
+        callargs = ', '.join('arg%d' % (i,)
+                            for i in range(len(self.argtypes)))
+        if self.restype is lltype.Void:
+            body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+        else:
+            body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+        return '%s %s(%s)\n%s' % (restype, name, args, body)
+
+
+DEFAULT_HEADER = 'pypy_decl.h'
+def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
+                gil=None, result_borrowed=False, result_is_ll=False):
+    """
+    Declares a function to be exported.
+    - `argtypes`, `restype` are lltypes and describe the function signature.
+    - `error` is the value returned when an applevel exception is raised. The
+      special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual
+      exception into a wrapped SystemError.  Unwrapped exceptions also cause a
+      SytemError.
+    - `header` is the header file to export the function in.
+    - set `gil` to "acquire", "release" or "around" to acquire the GIL,
+      release the GIL, or both
+    """
+    assert header is not None
+    def decorate(func):
+        if func.__name__ in FUNCTIONS_BY_HEADER[header]:
+            raise ValueError("%s already registered" % func.__name__)
+        func._always_inline_ = 'try'
+        api_function = ApiFunction(
+            argtypes, restype, func,
+            error=_compute_error(error, restype), gil=gil,
+            result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+        FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
 
         # ZZZ is this whole logic really needed???  It seems to be only
         # for RPython code calling PyXxx() functions directly.  I would
@@ -452,25 +453,61 @@
             try:
                 res = unwrapper(space, *args)
             except OperationError as e:
-                if not hasattr(api_function, "error_value"):
+                if not hasattr(unwrapper.api_func, "error_value"):
                     raise
                 state = space.fromcache(State)
                 state.set_exception(e)
                 if is_PyObject(restype):
                     return None
                 else:
-                    return api_function.error_value
+                    return unwrapper.api_func.error_value
             got_integer = isinstance(res, (int, long, float))
+            if isinstance(restype, lltype.Typedef):
+                real_restype = restype.OF
+            else:
+                real_restype = restype
+            expect_integer = (isinstance(real_restype, lltype.Primitive) and
+                            rffi.cast(restype, 0) == 0)
             assert got_integer == expect_integer, (
                 'got %r not integer' % (res,))
             return res
+        INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
 
-        if header is not None:
-            FUNCTIONS_BY_HEADER[header][func_name] = api_function
-        INTERPLEVEL_API[func_name] = unwrapper_catch  # used in tests
-        return unwrapper  # used in 'normal' RPython code.
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
     return decorate
 
+def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
+    def decorate(func):
+        func._always_inline_ = 'try'
+        api_function = ApiFunction(
+            argtypes, restype, func,
+            error=_compute_error(error, restype),
+            c_name=func.__name__)
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
+
+def _compute_error(error, restype):
+    """Convert error specification to actual error value of type restype."""
+    if isinstance(restype, lltype.Typedef):
+        real_restype = restype.OF
+    else:
+        real_restype = restype
+    if error is _NOT_SPECIFIED:
+        if isinstance(real_restype, lltype.Ptr):
+            error = lltype.nullptr(real_restype.TO)
+        elif real_restype is lltype.Void:
+            error = CANNOT_FAIL
+    if type(error) is int:
+        error = rffi.cast(real_restype, error)
+    return error
+
+
 def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
     if level == 1:
@@ -981,23 +1018,6 @@
     for func in BOOTSTRAP_FUNCTIONS:
         func(space)
 
-def c_function_signature(db, func):
-    restype = db.gettype(func.restype).replace('@', '').strip()
-    args = []
-    for i, argtype in enumerate(func.argtypes):
-        if argtype is CONST_STRING:
-            arg = 'const char *@'
-        elif argtype is CONST_STRINGP:
-            arg = 'const char **@'
-        elif argtype is CONST_WSTRING:
-            arg = 'const wchar_t *@'
-        else:
-            arg = db.gettype(argtype)
-        arg = arg.replace('@', 'arg%d' % (i,)).strip()
-        args.append(arg)
-    args = ', '.join(args) or "void"
-    return restype, args
-
 #_____________________________________________________
 # Build the bridge DLL, Allow extension DLLs to call
 # back into Pypy space functions
@@ -1017,15 +1037,8 @@
     structindex = {}
     for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         for name, func in header_functions.iteritems():
-            restype, args = c_function_signature(db, func)
-            callargs = ', '.join('arg%d' % (i,)
-                                for i in range(len(func.argtypes)))
-            if func.restype is lltype.Void:
-                body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
-            else:
-                body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
-            functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
-            members.append('%s (*%s)(%s);' % (restype, name, args))
+            functions.append(func.get_ctypes_impl(name, db))
+            members.append(func.get_ptr_decl(name, db))
             structindex[name] = len(structindex)
     structmembers = '\n'.join(members)
     struct_declaration_code = """\
@@ -1220,8 +1233,7 @@
         for name, func in sorted(header_functions.iteritems()):
             _name = mangle_name(prefix, name)
             header.append("#define %s %s" % (name, _name))
-            restype, args = c_function_signature(db, func)
-            header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
+            header.append(func.get_api_decl(name, db))
 
     for name, (typ, expr) in GLOBALS.iteritems():
         if '#' in name:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL)
+    PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -25,14 +25,14 @@
 ##
 ## In the PyBytesObject returned, the ob_sval buffer may be modified as
 ## long as the freshly allocated PyBytesObject is not "forced" via a call
-## to any of the more sophisticated C-API functions. 
+## to any of the more sophisticated C-API functions.
 ##
 ## Care has been taken in implementing the functions below, so that
-## if they are called with a non-forced PyBytesObject, they will not 
+## if they are called with a non-forced PyBytesObject, they will not
 ## unintentionally force the creation of a RPython object. As long as only these
 ## are used, the ob_sval buffer is still modifiable:
-## 
-## PyBytes_AsString / PyString_AsString 
+##
+## PyBytes_AsString / PyString_AsString
 ## PyBytes_AS_STRING / PyString_AS_STRING
 ## PyBytes_AsStringAndSize / PyString_AsStringAndSize
 ## PyBytes_Size / PyString_Size
@@ -40,7 +40,7 @@
 ## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string, 
+##   corresponds to the pypy gc-managed string,
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -105,7 +105,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def bytes_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -6,8 +6,8 @@
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
-    bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj, 
+    bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj,
         make_typedescr, track_reference, create_ref, from_ref, decref,
         Py_IncRef)
 from pypy.module.cpyext.object import _dealloc
@@ -36,7 +36,7 @@
     py_dict.c__tmpkeys = lltype.nullptr(PyObject.TO)
     # Problems: if this dict is a typedict, we may have unbound GetSetProperty
     # functions in the dict. The corresponding PyGetSetDescrObject must be
-    # bound to a class, but the actual w_type will be unavailable later on. 
+    # bound to a class, but the actual w_type will be unavailable later on.
     # Solution: use the w_userdata argument when assigning a PyTypeObject's
     # tp_dict slot to pass a w_type in, and force creation of the pair here
     if not space.is_w(w_userdata, space.gettypefor(GetSetProperty)):
@@ -55,7 +55,7 @@
     w_obj = space.newdict()
     track_reference(space, py_obj, w_obj)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def dict_dealloc(space, py_obj):
     py_dict = rffi.cast(PyDictObject, py_obj)
     decref(space, py_dict.c__tmpkeys)
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObjectFields, cpython_struct,
-    CANNOT_FAIL)
+    CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, Py_DecRef, make_ref, from_ref, track_reference,
     make_typedescr, get_typedescr)
@@ -39,7 +39,7 @@
     py_frame.c_f_locals = make_ref(space, frame.get_w_locals())
     rffi.setintfield(py_frame, 'c_f_lineno', frame.getorcreatedebug().f_lineno)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def frame_dealloc(space, py_obj):
     py_frame = rffi.cast(PyFrameObject, py_obj)
     py_code = rffi.cast(PyObject, py_frame.c_f_code)
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from rpython.rlib.unroll import unrolling_iterable
@@ -56,7 +57,7 @@
     assert isinstance(w_obj, Function)
     py_func.c_func_name = make_ref(space, space.wrap(w_obj.name))
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def function_dealloc(space, py_obj):
     py_func = rffi.cast(PyFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_func_name)
@@ -75,7 +76,7 @@
     rffi.setintfield(py_code, 'c_co_flags', co_flags)
     rffi.setintfield(py_code, 'c_co_argcount', w_obj.co_argcount)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def code_dealloc(space, py_obj):
     py_code = rffi.cast(PyCodeObject, py_obj)
     Py_DecRef(space, py_code.c_co_name)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,10 +1,10 @@
-from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers,
-                         Py_ssize_tP, PyObjectFields, cpython_struct,
-                         generic_cpy_call,
-                         bootstrap_function, Py_bufferP)
-from pypy.module.cpyext.pyobject import (PyObject, make_ref, as_pyobj, incref,
-             decref, from_ref, make_typedescr)
+from pypy.module.cpyext.api import (
+    cpython_api, Py_buffer, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS,
+    build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct,
+    bootstrap_function, Py_bufferP, generic_cpy_call, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr,
+    get_typedescr, track_reference)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.interpreter.error import oefmt
@@ -30,7 +30,7 @@
                    basestruct=PyMemoryViewObject.TO,
                    attach=memory_attach,
                    dealloc=memory_dealloc,
-                   #realize=memory_realize,
+                   realize=memory_realize,
                   )
 
 def memory_attach(space, py_obj, w_obj, w_userdata=None):
@@ -56,13 +56,37 @@
                                              track_allocation=False))
         rffi.setintfield(view, 'c_readonly', 1)
 
-def memory_realize(space, py_obj):
+def memory_realize(space, obj):
     """
     Creates the memory object in the interpreter
     """
-    raise oefmt(space.w_NotImplementedError, "cannot call this yet")
+    from pypy.module.cpyext.slotdefs import CPyBuffer, fq
+    py_mem = rffi.cast(PyMemoryViewObject, obj)
+    view = py_mem.c_view
+    ndim = widen(view.c_ndim)
+    shape = None
+    if view.c_shape:
+        shape = [view.c_shape[i] for i in range(ndim)]
+    strides = None
+    if view.c_strides:
+        strides = [view.c_strides[i] for i in range(ndim)]
+    format = 'B'
+    if view.c_format:
+        format = rffi.charp2str(view.c_format)
+    buf = CPyBuffer(space, view.c_buf, view.c_len, from_ref(space, view.c_obj),
+                    format=format, shape=shape, strides=strides,
+                    ndim=ndim, itemsize=view.c_itemsize,
+                    readonly=widen(view.c_readonly))
+    # Ensure view.c_buf is released upon object finalization
+    fq.register_finalizer(buf)
+    # Allow subclassing W_MemeoryView
+    w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
+    w_obj = space.allocate_instance(W_MemoryView, w_type)
+    w_obj.__init__(buf)
+    track_reference(space, obj, w_obj)
+    return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def memory_dealloc(space, py_obj):
     mem_obj = rffi.cast(PyMemoryViewObject, py_obj)
     if mem_obj.c_view.c_obj:
@@ -199,17 +223,41 @@
     py_memview = make_ref(space, w_memview, w_obj)
     return py_memview
 
- at cpython_api([Py_bufferP], PyObject)
+ at cpython_api([Py_bufferP], PyObject, result_is_ll=True)
 def PyMemoryView_FromBuffer(space, view):
     """Create a memoryview object wrapping the given buffer structure view.
     The memoryview object then owns the buffer represented by view, which
     means you shouldn't try to call PyBuffer_Release() yourself: it
     will be done on deallocation of the memoryview object."""
-    assert view.c_obj
-    w_obj = from_ref(space, view.c_obj)
-    if isinstance(w_obj, W_MemoryView):
-        return w_obj
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    # XXX this should allocate a PyMemoryViewObject and
+    # copy view into obj.c_view, without creating a new view.c_obj
+    typedescr = get_typedescr(W_MemoryView.typedef)
+    py_obj = typedescr.allocate(space, space.w_memoryview)
+    py_mem = rffi.cast(PyMemoryViewObject, py_obj)
+    mview = py_mem.c_view
+    mview.c_buf = view.c_buf
+    mview.c_obj = view.c_obj
+    mview.c_len = view.c_len
+    mview.c_itemsize = view.c_itemsize
+    mview.c_readonly = view.c_readonly
+    mview.c_ndim = view.c_ndim
+    mview.c_format = view.c_format
+    if view.c_strides == rffi.cast(Py_ssize_tP, view.c__strides):
+        py_mem.c_view.c_strides = rffi.cast(Py_ssize_tP, py_mem.c_view.c__strides)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_strides[i] = view.c_strides[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_strides = view.c_strides
+    if view.c_shape == rffi.cast(Py_ssize_tP, view.c__shape):
+        py_mem.c_view.c_shape = rffi.cast(Py_ssize_tP, py_mem.c_view.c__shape)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_shape[i] = view.c_shape[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_shape = view.c_shape
+    # XXX ignore suboffsets?
+    return py_obj
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,7 +11,7 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr)
+    PyTypeObjectPtr, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -51,7 +51,7 @@
     py_func.c_m_self = make_ref(space, w_obj.w_self)
     py_func.c_m_module = make_ref(space, w_obj.w_module)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def cfunction_dealloc(space, py_obj):
     py_func = rffi.cast(PyCFunctionObject, py_obj)
     Py_DecRef(space, py_func.c_m_self)
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t,
+    PyVarObject, Py_buffer, size_t, slot_function,
+    PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -58,7 +59,7 @@
         w_obj = PyObject_InitVar(space, py_objvar, type, itemcount)
     return py_obj
 
- at cpython_api([PyObject], lltype.Void)
+ at slot_function([PyObject], lltype.Void)
 def PyObject_dealloc(space, obj):
     return _dealloc(space, obj)
 
@@ -489,22 +490,28 @@
     Fills in a buffer-info structure correctly for an exporter that can only
     share a contiguous chunk of memory of "unsigned bytes" of the given
     length. Returns 0 on success and -1 (with raising an error) on error.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
     """
     if rffi.cast(lltype.Signed, flags) & PyBUF_WRITABLE and readonly:
         raise oefmt(space.w_ValueError, "Object is not writable")
     view.c_buf = buf
     view.c_len = length
     view.c_obj = obj
-    Py_IncRef(space, obj)
+    if obj:
+        Py_IncRef(space, obj)
     view.c_itemsize = 1
     rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
+    rffi.setintfield(view, 'c_ndim', 1)
     view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
     view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
     view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
 
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -87,7 +87,7 @@
     alloc     : allocate and basic initialization of a raw PyObject
     attach    : Function called to tie a raw structure to a pypy object
     realize   : Function called to create a pypy object from a raw struct
-    dealloc   : a cpython_api(header=None), similar to PyObject_dealloc
+    dealloc   : a @slot_function(), similar to PyObject_dealloc
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
+    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
@@ -40,7 +41,7 @@
     rffi.setintfield(py_traceback, 'c_tb_lasti', traceback.lasti)
     rffi.setintfield(py_traceback, 'c_tb_lineno',traceback.get_lineno())
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def traceback_dealloc(space, py_obj):
     py_traceback = rffi.cast(PyTracebackObject, py_obj)
     Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next))
diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py
--- a/pypy/module/cpyext/sliceobject.py
+++ b/pypy/module/cpyext/sliceobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
-    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields)
+    CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields, slot_function)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, PyObject, make_ref, make_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
@@ -36,7 +36,7 @@
     py_slice.c_stop = make_ref(space, w_obj.w_stop)
     py_slice.c_step = make_ref(space, w_obj.w_step)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def slice_dealloc(space, py_obj):
     """Frees allocated PySliceObject resources.
     """
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -6,7 +6,7 @@
 from rpython.rlib.rarithmetic import widen
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, PyObject, Py_ssize_t,
+    slot_function, generic_cpy_call, PyObject, Py_ssize_t,
     pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
@@ -64,7 +64,7 @@
 
 @not_rpython
 def llslot(space, func):
-    return llhelper(func.api_func.functype, func.api_func.get_wrapper(space))
+    return func.api_func.get_llhelper(space)
 
 @register_flow_sc(llslot)
 def sc_llslot(ctx, v_space, v_func):
@@ -315,7 +315,7 @@
 
     def __init__(self, space, ptr, size, w_obj, format='B', shape=None,
                 strides=None, ndim=1, itemsize=1, readonly=True,
-                releasebuffer=None):
+                releasebufferproc=rffi.cast(rffi.VOIDP, 0)):
         self.space = space
         self.ptr = ptr
         self.size = size
@@ -333,7 +333,7 @@
         self.ndim = ndim
         self.itemsize = itemsize
         self.readonly = readonly
-        self.releasebufferproc = releasebuffer
+        self.releasebufferproc = releasebufferproc
 
     def releasebuffer(self):
         if self.pyobj:
@@ -351,7 +351,10 @@
                 for i in range(self.ndim):
                     pybuf.c_shape[i] = self.shape[i]
                     pybuf.c_strides[i] = self.strides[i]
-                pybuf.c_format = rffi.str2charp(self.format)
+                if self.format:
+                    pybuf.c_format = rffi.str2charp(self.format)
+                else:
+                    pybuf.c_format = rffi.str2charp("B")
                 generic_cpy_call(self.space, func_target, self.pyobj, pybuf)
             self.releasebufferproc = rffi.cast(rffi.VOIDP, 0)
 
@@ -398,9 +401,9 @@
     func_target = rffi.cast(readbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
@@ -408,7 +411,7 @@
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -417,16 +420,16 @@
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
     decref(space, py_obj)
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -434,9 +437,9 @@
     func_target = rffi.cast(getbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(Py_buffer) as pybuf:
         _flags = 0
@@ -465,7 +468,7 @@
                             ndim=ndim, shape=shape, strides=strides,
                             itemsize=pybuf.c_itemsize,
                             readonly=widen(pybuf.c_readonly),
-                            releasebuffer = releasebuffer)
+                            releasebufferproc = rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -515,9 +518,6 @@
 def build_slot_tp_function(space, typedef, name):
     w_type = space.gettypeobject(typedef)
 
-    header = pypy_decl
-    if not (name.startswith('Py') or name.startswith('_Py')):
-        header = None
     handled = False
     # unary functions
     for tp_name, attr in [('tp_as_number.c_nb_int', '__int__'),
@@ -537,7 +537,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject], PyObject, header=header)
+            @slot_function([PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self):
                 return space.call_function(slot_fn, w_self)
@@ -563,7 +563,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg):
                 return space.call_function(slot_fn, w_self, w_arg)
@@ -580,7 +580,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, Py_ssize_t], PyObject, header=header)
+            @slot_function([PyObject, Py_ssize_t], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, arg):
                 return space.call_function(slot_fn, w_self, space.wrap(arg))
@@ -594,7 +594,7 @@
             if slot_fn is None:
                 return
 
-            @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+            @slot_function([PyObject, PyObject, PyObject], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
@@ -608,8 +608,8 @@
         if setattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real,
-                     error=-1, header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real,
+                     error=-1)
         @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,))
         def slot_tp_setattro(space, w_self, w_name, w_value):
             if w_value is not None:
@@ -623,7 +623,7 @@
         if getattr_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject], PyObject)
         @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,))
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
@@ -634,7 +634,7 @@
         if call_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], PyObject, header=header)
+        @slot_function([PyObject, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_call(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -647,7 +647,7 @@
         if iternext_fn is None:
             return
 
-        @cpython_api([PyObject], PyObject, header=header)
+        @slot_function([PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_iternext(space, w_self):
             try:
@@ -663,8 +663,7 @@
         if init_fn is None:
             return
 
-        @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1,
-                     header=header)
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_init(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -677,7 +676,7 @@
         if new_fn is None:
             return
 
-        @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
+        @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def slot_tp_new(space, w_self, w_args, w_kwds):
             args = Arguments(space, [w_self],
@@ -688,8 +687,8 @@
         buff_fn = w_type.getdictvalue(space, '__buffer__')
         if buff_fn is None:
             return
-        @cpython_api([PyObject, Py_bufferP, rffi.INT_real],
-                rffi.INT_real, header=None, error=-1)
+        @slot_function([PyObject, Py_bufferP, rffi.INT_real],
+                rffi.INT_real, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
         def buff_w(space, w_self, view, flags):
             args = Arguments(space, [space.newint(flags)])
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -2,20 +2,14 @@
 from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
-from pypy.module.cpyext import api
+from pypy.module.cpyext.api import (
+    slot_function, cpython_api, copy_header_files, INTERPLEVEL_API,
+    Py_ssize_t, Py_ssize_tP, PyObject)
 from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest
-PyObject = api.PyObject
 from pypy.interpreter.error import OperationError
 from rpython.rlib import rawrefcount
 import os
 
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetWrapped(space, w_arg):
-    assert isinstance(w_arg, W_Root)
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetReference(space, arg):
-    assert lltype.typeOf(arg) ==  PyObject
-
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
         space = cls.space
@@ -35,7 +29,7 @@
             def __getattr__(self, name):
                 return getattr(cls.space, name)
         cls.api = CAPI()
-        CAPI.__dict__.update(api.INTERPLEVEL_API)
+        CAPI.__dict__.update(INTERPLEVEL_API)
 
         print 'DONT_FREE_ANY_MORE'
         rawrefcount._dont_free_any_more()
@@ -71,31 +65,39 @@
         if self.check_and_print_leaks():
             assert False, "Test leaks or loses object(s)."
 
- at api.cpython_api([api.Py_ssize_t], api.Py_ssize_t, error=-1)
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetWrapped(space, w_arg):
+    assert isinstance(w_arg, W_Root)
+
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetReference(space, arg):
+    assert lltype.typeOf(arg) ==  PyObject
+
+ at cpython_api([Py_ssize_t], Py_ssize_t, error=-1)
 def PyPy_TypedefTest1(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_t
+    assert lltype.typeOf(arg) == Py_ssize_t
     return 0
 
- at api.cpython_api([api.Py_ssize_tP], api.Py_ssize_tP)
+ at cpython_api([Py_ssize_tP], Py_ssize_tP)
 def PyPy_TypedefTest2(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_tP
+    assert lltype.typeOf(arg) == Py_ssize_tP
     return None
 
 class TestConversion(BaseApiTest):
-    def test_conversions(self, space, api):
-        api.PyPy_GetWrapped(space.w_None)
-        api.PyPy_GetReference(space.w_None)
+    def test_conversions(self, space):
+        PyPy_GetWrapped(space, space.w_None)
+        PyPy_GetReference(space, space.w_None)
 
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
-                == ('Py_ssize_t', 'Py_ssize_t arg0'))
-        assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
-                == ('Py_ssize_t *', 'Py_ssize_t *arg0'))
+        assert PyPy_TypedefTest1.api_func.get_c_restype(db) == 'Py_ssize_t'
+        assert PyPy_TypedefTest1.api_func.get_c_args(db) == 'Py_ssize_t arg0'
+        assert PyPy_TypedefTest2.api_func.get_c_restype(db) == 'Py_ssize_t *'
+        assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Py_ssize_t *arg0'
 
         PyPy_TypedefTest1(space, 0)
-        ppos = lltype.malloc(api.Py_ssize_tP.TO, 1, flavor='raw')
+        ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
         ppos[0] = 0
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
@@ -103,7 +105,7 @@
 @pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
-    api.copy_header_files(tmpdir, True)
+    copy_header_files(tmpdir, True)
     def check(name):
         f = tmpdir.join(name)
         assert f.check(file=True)
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -30,7 +30,7 @@
         assert view.c_len == 5
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
-        w_mv = api.PyMemoryView_FromBuffer(view)
+        w_mv = from_ref(space, api.PyMemoryView_FromBuffer(view))
         for f in ('format', 'itemsize', 'ndim', 'readonly',
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)
@@ -43,6 +43,7 @@
                 ("fillinfo", "METH_VARARGS",
                  """
                  Py_buffer buf;
+                 PyObject * ret = NULL;
                  PyObject *str = PyBytes_FromString("hello, world.");
                  if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
                                        0, 0)) {
@@ -54,7 +55,14 @@
                   */
                  Py_DECREF(str);
 
-                 return PyMemoryView_FromBuffer(&buf);
+                 ret = PyMemoryView_FromBuffer(&buf);
+                 if (((PyMemoryViewObject*)ret)->view.obj != buf.obj)
+                 {
+                    PyErr_SetString(PyExc_ValueError, "leaked ref");
+                    Py_DECREF(ret);
+                    return NULL;
+                 }
+                 return ret;
                  """)])
         result = module.fillinfo()
         assert b"hello, world." == result
diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test/test_translate.py
--- a/pypy/module/cpyext/test/test_translate.py
+++ b/pypy/module/cpyext/test/test_translate.py
@@ -1,6 +1,6 @@
 from rpython.translator.c.test.test_genc import compile
 import pypy.module.cpyext.api
-from pypy.module.cpyext.api import cpython_api
+from pypy.module.cpyext.api import slot_function
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.objectmodel import specialize
@@ -19,7 +19,7 @@
 
     @specialize.memo()
     def get_tp_function(space, typedef):
-        @cpython_api([], lltype.Signed, error=-1, header=None)
+        @slot_function([], lltype.Signed, error=-1)
         def slot_tp_function(space):
             return typedef.value
 
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -1,11 +1,11 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import fatalerror_notb
-from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL,
-                                    build_type_checkers, PyVarObjectFields,
-                                    cpython_struct, bootstrap_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef,
-    make_ref, from_ref, decref, incref, pyobj_has_w_obj,
+from pypy.module.cpyext.api import (
+    cpython_api, Py_ssize_t, build_type_checkers,
+    PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import (
+    PyObject, PyObjectP, make_ref, from_ref, decref, incref,
     track_reference, make_typedescr, get_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
@@ -74,7 +74,7 @@
     if py_tup.c_ob_size < length:
         raise oefmt(space.w_ValueError,
             "tuple_attach called on object with ob_size %d but trying to store %d",
-            py_tup.c_ob_size, length) 
+            py_tup.c_ob_size, length)
     i = 0
     try:
         while i < length:
@@ -113,7 +113,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def tuple_dealloc(space, py_obj):
     """Frees allocated PyTupleObject resources.
     """
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -2,21 +2,20 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rstring import rsplit
 from rpython.rtyper.lltypesystem import rffi, lltype
 
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.typedef import (GetSetProperty, TypeDef,
-        interp_attrproperty, interp_attrproperty, interp2app)
+from pypy.interpreter.typedef import (
+    GetSetProperty, TypeDef, interp_attrproperty, interp2app)
 from pypy.module.__builtin__.abstractinst import abstract_issubclass_w
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_buffer,
+    slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_buffer,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
+    PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
@@ -345,7 +344,7 @@
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
 
- at cpython_api([PyObject, PyObject, PyObject], PyObject, header=None)
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
 def tp_new_wrapper(space, self, w_args, w_kwds):
     self_pytype = rffi.cast(PyTypeObjectPtr, self)
     tp_new = self_pytype.c_tp_new
@@ -504,7 +503,7 @@
                    realize=type_realize,
                    dealloc=type_dealloc)
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def subtype_dealloc(space, obj):
     pto = obj.c_ob_type
     base = pto
@@ -519,8 +518,7 @@
     # hopefully this does not clash with the memory model assumed in
     # extension modules
 
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real,
-              header=None, error=-1)
+ at slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bytes_getbuffer(space, w_str, view, flags):
     from pypy.module.cpyext.bytesobject import PyBytes_AsString
     from pypy.module.cpyext.object import PyBuffer_FillInfo
@@ -528,8 +526,7 @@
     return PyBuffer_FillInfo(space, view, w_str, c_buf,
                              space.len_w(w_str), 1, flags)
 
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real,
-              header=None, error=-1)
+ at slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bf_getbuffer(space, w_obj, view, flags):
     from pypy.module.cpyext.object import PyBuffer_FillInfo
     buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags))
@@ -550,7 +547,7 @@
     pto.c_tp_as_buffer = c_buf
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def type_dealloc(space, obj):
     from pypy.module.cpyext.object import _dealloc
     obj_pto = rffi.cast(PyTypeObjectPtr, obj)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
     bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
-    CONST_WSTRING, Py_CLEANUP_SUPPORTED)
+    CONST_WSTRING, Py_CLEANUP_SUPPORTED, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -78,7 +78,7 @@
     track_reference(space, py_obj, w_obj)
     return w_obj
 
- at cpython_api([PyObject], lltype.Void, header=None)
+ at slot_function([PyObject], lltype.Void)
 def unicode_dealloc(space, py_obj):
     py_unicode = rffi.cast(PyUnicodeObject, py_obj)
     if py_unicode.c_buffer:
diff --git a/rpython/rlib/test/test_rposix_stat.py b/rpython/rlib/test/test_rposix_stat.py
--- a/rpython/rlib/test/test_rposix_stat.py
+++ b/rpython/rlib/test/test_rposix_stat.py
@@ -29,7 +29,10 @@
             check(os.environ['TEMP'])
         else:
             check('/')
-            check('/tmp')
+            check('/dev')
+            # don't test with /tmp, because if another process is also
+            # creating files in /tmp it is more likely that the mtime
+            # we get during successive calls was actually changed
         check(sys.executable)
 
     def test_fstat(self):
diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py
--- a/rpython/rlib/unicodedata/test/test_unicodedata.py
+++ b/rpython/rlib/unicodedata/test/test_unicodedata.py
@@ -5,7 +5,8 @@
 import py
 
 from rpython.rlib.unicodedata import (
-    unicodedb_3_2_0, unicodedb_5_2_0, unicodedb_6_0_0, unicodedb_6_2_0)
+    unicodedb_3_2_0, unicodedb_5_2_0, unicodedb_6_0_0, unicodedb_6_2_0,
+    unicodedb_8_0_0)
 
 
 class TestUnicodeData(object):
@@ -141,3 +142,9 @@
 
     def test_islower(self):
         assert unicodedb_6_2_0.islower(0x2177)
+
+
+class TestUnicodeData800(object):
+    def test_changed_in_version_8(self):
+        assert unicodedb_6_2_0.toupper_full(0x025C) == [0x025C]
+        assert unicodedb_8_0_0.toupper_full(0x025C) == [0xA7AB]

From pypy.commits at gmail.com  Wed Jan 11 13:49:46 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 11 Jan 2017 10:49:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58767e4a.4f831c0a.8bab3.ddd7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89501:f4f9c3b10d64
Date: 2017-01-11 18:49 +0000
http://bitbucket.org/pypy/pypy/changeset/f4f9c3b10d64/

Log:	hg merge default

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -8,6 +8,7 @@
 
 from rpython.rlib.jit import promote
 from rpython.rlib.objectmodel import compute_identity_hash, specialize
+from rpython.rlib.objectmodel import instantiate
 from rpython.tool.sourcetools import compile2, func_with_new_name
 
 
@@ -221,10 +222,6 @@
     exec source.compile() in miniglobals
     return miniglobals['descr_typecheck_%s' % func.__name__]
 
-def unknown_objclass_getter(space):
-    # NB. this is an AttributeError to make inspect.py happy
-    raise oefmt(space.w_AttributeError, "generic property has no __objclass__")
-
 @specialize.arg(0)
 def make_objclass_getter(tag, func, cls):
     if func and hasattr(func, 'im_func'):
@@ -235,7 +232,7 @@
 @specialize.memo()
 def _make_objclass_getter(cls):
     if not cls:
-        return unknown_objclass_getter, cls
+        return None, cls
     miniglobals = {}
     if isinstance(cls, str):
         assert cls.startswith('<'), "pythontype typecheck should begin with <"
@@ -254,6 +251,8 @@
 
 class GetSetProperty(W_Root):
     _immutable_fields_ = ["fget", "fset", "fdel"]
+    name = '<generic property>'
+    w_objclass = None
 
     @specialize.arg(7)
     def __init__(self, fget, fset=None, fdel=None, doc=None,
@@ -265,16 +264,26 @@
                                             cls=cls, use_closure=use_closure)
         fdel = make_descr_typecheck_wrapper((tag, 2), fdel,
                                             cls=cls, use_closure=use_closure)
+        self._init(fget, fset, fdel, doc, cls, objclass_getter, use_closure)
+
+    def _init(self, fget, fset, fdel, doc, cls, objclass_getter, use_closure):
         self.fget = fget
         self.fset = fset
         self.fdel = fdel
         self.doc = doc
         self.reqcls = cls
-        self.name = '<generic property>'
         self.qualname = None
         self.objclass_getter = objclass_getter
         self.use_closure = use_closure
 
+    def copy_for_type(self, w_objclass):
+        new = instantiate(GetSetProperty)
+        new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
+                  None, self.use_closure)
+        new.name = self.name
+        new.w_objclass = w_objclass
+        return new
+
     @unwrap_spec(w_cls = WrappedDefault(None))
     def descr_property_get(self, space, w_obj, w_cls=None):
         """property.__get__(obj[, type]) -> value
@@ -338,7 +347,14 @@
         return space.wrap(qualname)
 
     def descr_get_objclass(space, property):
-        return property.objclass_getter(space)
+        if property.w_objclass is not None:
+            return property.w_objclass
+        if property.objclass_getter is not None:
+            return property.objclass_getter(space)
+        # NB. this is an AttributeError to make inspect.py happy
+        raise oefmt(space.w_AttributeError,
+                    "generic property has no __objclass__")
+
 
 def interp_attrproperty(name, cls, doc=None):
     "NOT_RPYTHON: initialization-time only"
@@ -497,7 +513,7 @@
     return lifeline.get_any_weakref(space)
 
 dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
-                            doc="dictionary for instance variables (if defined)")
+                            doc="dictionary for instance variables")
 dict_descr.name = '__dict__'
 
 
@@ -532,7 +548,7 @@
     return space.newtuple([w_docstring])
 
 weakref_descr = GetSetProperty(descr_get_weakref,
-                    doc="list of weak references to the object (if defined)")
+                               doc="list of weak references to the object")
 weakref_descr.name = '__weakref__'
 
 def make_weakref_descr(cls):
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -118,12 +118,16 @@
                     """ % (type_name,)))
         except OperationError:
             return 0
+    return check, check_exact
 
-make_check_function("PyDateTime_Check", "datetime")
-make_check_function("PyDate_Check", "date")
-make_check_function("PyTime_Check", "time")
-make_check_function("PyDelta_Check", "timedelta")
-make_check_function("PyTZInfo_Check", "tzinfo")
+PyDateTime_Check, PyDateTime_CheckExact = make_check_function(
+    "PyDateTime_Check", "datetime")
+PyDate_Check, PyDate_CheckExact = make_check_function("PyDate_Check", "date")
+PyTime_Check, PyTime_CheckExact = make_check_function("PyTime_Check", "time")
+PyDelta_Check, PyDelta_CheckExact = make_check_function(
+    "PyDelta_Check", "timedelta")
+PyTZInfo_Check, PyTZInfo_CheckExact = make_check_function(
+    "PyTZInfo_Check", "tzinfo")
 
 # Constructors. They are better used as macros.
 
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,4 +1,5 @@
 import py, pytest
+import contextlib
 from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
@@ -10,6 +11,13 @@
 from rpython.rlib import rawrefcount
 import os
 
+ at contextlib.contextmanager
+def raises_w(space, expected_exc):
+    with pytest.raises(OperationError) as excinfo:
+        yield
+    operror = excinfo.value
+    assert operror.w_type is getattr(space, 'w_' + expected_exc.__name__)
+
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
         space = cls.space
diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py
--- a/pypy/module/cpyext/test/test_boolobject.py
+++ b/pypy/module/cpyext/test/test_boolobject.py
@@ -1,20 +1,22 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong
+from pypy.module.cpyext.floatobject import PyFloat_FromDouble
 
 class TestBoolObject(BaseApiTest):
-    def test_fromlong(self, space, api):
+    def test_fromlong(self, space):
         for i in range(-3, 3):
-            obj = api.PyBool_FromLong(i)
+            obj = PyBool_FromLong(space, i)
             if i:
                 assert obj is space.w_True
             else:
                 assert obj is space.w_False
 
-    def test_check(self, space, api):
-        assert api.PyBool_Check(space.w_True)
-        assert api.PyBool_Check(space.w_False)
-        assert not api.PyBool_Check(space.w_None)
-        assert not api.PyBool_Check(api.PyFloat_FromDouble(1.0))
+    def test_check(self, space):
+        assert PyBool_Check(space, space.w_True)
+        assert PyBool_Check(space, space.w_False)
+        assert not PyBool_Check(space, space.w_None)
+        assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0))
 
 class AppTestBoolMacros(AppTestCpythonExtensionBase):
     def test_macros(self):
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -1,14 +1,14 @@
 # encoding: utf-8
+import pytest
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.interpreter.error import OperationError
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject
 from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call, Py_buffer
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref, as_pyobj
 from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
 
-import py
-import sys
 
 
 class AppTestBytesObject(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_codecs.py b/pypy/module/cpyext/test/test_codecs.py
--- a/pypy/module/cpyext/test/test_codecs.py
+++ b/pypy/module/cpyext/test/test_codecs.py
@@ -1,14 +1,15 @@
 # encoding: iso-8859-15
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.codecs import (
+    PyCodec_IncrementalEncoder, PyCodec_IncrementalDecoder)
 
 class TestCodecs(BaseApiTest):
-    def test_incremental(self, space, api):
+    def test_incremental(self, space):
         utf8 = rffi.str2charp('utf-8')
-        w_encoder = api.PyCodec_IncrementalEncoder(utf8, None)
+        w_encoder = PyCodec_IncrementalEncoder(space, utf8, None)
         w_encoded = space.call_method(w_encoder, 'encode', space.wrap(u'sp&#228;m'))
-        w_decoder = api.PyCodec_IncrementalDecoder(utf8, None)
+        w_decoder = PyCodec_IncrementalDecoder(space, utf8, None)
         w_decoded = space.call_method(w_decoder, 'decode', w_encoded)
         assert space.unwrap(w_decoded) == u'sp&#228;m'
         rffi.free_charp(utf8)
-
diff --git a/pypy/module/cpyext/test/test_complexobject.py b/pypy/module/cpyext/test/test_complexobject.py
--- a/pypy/module/cpyext/test/test_complexobject.py
+++ b/pypy/module/cpyext/test/test_complexobject.py
@@ -1,23 +1,23 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.complexobject import (
+    PyComplex_FromDoubles, PyComplex_RealAsDouble, PyComplex_ImagAsDouble)
 
 class TestComplexObject(BaseApiTest):
-    def test_complexobject(self, space, api):
-        w_value = api.PyComplex_FromDoubles(1.2, 3.4)
+    def test_complexobject(self, space):
+        w_value = PyComplex_FromDoubles(space, 1.2, 3.4)
         assert space.unwrap(w_value) == 1.2+3.4j
-        assert api.PyComplex_RealAsDouble(w_value) == 1.2
-        assert api.PyComplex_ImagAsDouble(w_value) == 3.4
+        assert PyComplex_RealAsDouble(space, w_value) == 1.2
+        assert PyComplex_ImagAsDouble(space, w_value) == 3.4
 
-        assert api.PyComplex_RealAsDouble(space.wrap(42)) == 42
-        assert api.PyComplex_RealAsDouble(space.wrap(1.5)) == 1.5
-        assert api.PyComplex_ImagAsDouble(space.wrap(1.5)) == 0.0
+        assert PyComplex_RealAsDouble(space, space.wrap(42)) == 42
+        assert PyComplex_RealAsDouble(space, space.wrap(1.5)) == 1.5
+        assert PyComplex_ImagAsDouble(space, space.wrap(1.5)) == 0.0
 
         # cpython accepts anything for PyComplex_ImagAsDouble
-        assert api.PyComplex_ImagAsDouble(space.w_None) == 0.0
-        assert not api.PyErr_Occurred()
-        assert api.PyComplex_RealAsDouble(space.w_None) == -1.0
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
+        assert PyComplex_ImagAsDouble(space, space.w_None) == 0.0
+        with raises_w(space, TypeError):
+            PyComplex_RealAsDouble(space, space.w_None)
 
 class AppTestCComplex(AppTestCpythonExtensionBase):
     def test_AsCComplex(self):
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -1,92 +1,96 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.cdatetime import *
+from pypy.module.cpyext.cdatetime import (
+    _PyDateTime_Import, _PyDateTime_FromDateAndTime, _PyDate_FromDate,
+    _PyTime_FromTime, _PyDelta_FromDelta)
 import datetime
 
 class TestDatetime(BaseApiTest):
-    def test_date(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_date = api._PyDate_FromDate(2010, 06, 03, date_api.c_DateType)
+    def test_date(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_date = _PyDate_FromDate(space, 2010, 06, 03, date_api.c_DateType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03'
 
-        assert api.PyDate_Check(w_date)
-        assert api.PyDate_CheckExact(w_date)
+        assert PyDate_Check(space, w_date)
+        assert PyDate_CheckExact(space, w_date)
 
-        assert api.PyDateTime_GET_YEAR(w_date) == 2010
-        assert api.PyDateTime_GET_MONTH(w_date) == 6
-        assert api.PyDateTime_GET_DAY(w_date) == 3
+        assert PyDateTime_GET_YEAR(space, w_date) == 2010
+        assert PyDateTime_GET_MONTH(space, w_date) == 6
+        assert PyDateTime_GET_DAY(space, w_date) == 3
 
-    def test_time(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_time = api._PyTime_FromTime(23, 15, 40, 123456,
-                                      space.w_None, date_api.c_TimeType)
+    def test_time(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_time = _PyTime_FromTime(
+            space, 23, 15, 40, 123456, space.w_None, date_api.c_TimeType)
         assert space.unwrap(space.str(w_time)) == '23:15:40.123456'
 
-        assert api.PyTime_Check(w_time)
-        assert api.PyTime_CheckExact(w_time)
+        assert PyTime_Check(space, w_time)
+        assert PyTime_CheckExact(space, w_time)
 
-        assert api.PyDateTime_TIME_GET_HOUR(w_time) == 23
-        assert api.PyDateTime_TIME_GET_MINUTE(w_time) == 15
-        assert api.PyDateTime_TIME_GET_SECOND(w_time) == 40
-        assert api.PyDateTime_TIME_GET_MICROSECOND(w_time) == 123456
+        assert PyDateTime_TIME_GET_HOUR(space, w_time) == 23
+        assert PyDateTime_TIME_GET_MINUTE(space, w_time) == 15
+        assert PyDateTime_TIME_GET_SECOND(space, w_time) == 40
+        assert PyDateTime_TIME_GET_MICROSECOND(space, w_time) == 123456
 
-    def test_datetime(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_date = api._PyDateTime_FromDateAndTime(
-            2010, 06, 03, 23, 15, 40, 123456,
-            space.w_None, date_api.c_DateTimeType)
+    def test_datetime(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_date = _PyDateTime_FromDateAndTime(
+            space, 2010, 06, 03, 23, 15, 40, 123456, space.w_None,
+            date_api.c_DateTimeType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03 23:15:40.123456'
 
-        assert api.PyDateTime_Check(w_date)
-        assert api.PyDateTime_CheckExact(w_date)
-        assert api.PyDate_Check(w_date)
-        assert not api.PyDate_CheckExact(w_date)
+        assert PyDateTime_Check(space, w_date)
+        assert PyDateTime_CheckExact(space, w_date)
+        assert PyDate_Check(space, w_date)
+        assert not PyDate_CheckExact(space, w_date)
 
-        assert api.PyDateTime_GET_YEAR(w_date) == 2010
-        assert api.PyDateTime_GET_MONTH(w_date) == 6
-        assert api.PyDateTime_GET_DAY(w_date) == 3
-        assert api.PyDateTime_DATE_GET_HOUR(w_date) == 23
-        assert api.PyDateTime_DATE_GET_MINUTE(w_date) == 15
-        assert api.PyDateTime_DATE_GET_SECOND(w_date) == 40
-        assert api.PyDateTime_DATE_GET_MICROSECOND(w_date) == 123456
+        assert PyDateTime_GET_YEAR(space, w_date) == 2010
+        assert PyDateTime_GET_MONTH(space, w_date) == 6
+        assert PyDateTime_GET_DAY(space, w_date) == 3
+        assert PyDateTime_DATE_GET_HOUR(space, w_date) == 23
+        assert PyDateTime_DATE_GET_MINUTE(space, w_date) == 15
+        assert PyDateTime_DATE_GET_SECOND(space, w_date) == 40
+        assert PyDateTime_DATE_GET_MICROSECOND(space, w_date) == 123456
 
-    def test_delta(self, space, api):
-        date_api = api._PyDateTime_Import()
+    def test_delta(self, space):
+        date_api = _PyDateTime_Import(space)
         w_delta = space.appexec(
             [space.wrap(3), space.wrap(15)], """(days, seconds):
             from datetime import timedelta
             return timedelta(days, seconds)
         """)
-        assert api.PyDelta_Check(w_delta)
-        assert api.PyDelta_CheckExact(w_delta)
+        assert PyDelta_Check(space, w_delta)
+        assert PyDelta_CheckExact(space, w_delta)
 
-        w_delta = api._PyDelta_FromDelta(10, 20, 30, True, date_api.c_DeltaType)
-        assert api.PyDelta_Check(w_delta)
-        assert api.PyDelta_CheckExact(w_delta)
+        w_delta = _PyDelta_FromDelta(space, 10, 20, 30, True, date_api.c_DeltaType)
+        assert PyDelta_Check(space, w_delta)
+        assert PyDelta_CheckExact(space, w_delta)
 
-        assert api.PyDateTime_DELTA_GET_DAYS(w_delta) == 10
-        assert api.PyDateTime_DELTA_GET_SECONDS(w_delta) == 20
-        assert api.PyDateTime_DELTA_GET_MICROSECONDS(w_delta) == 30
+        assert PyDateTime_DELTA_GET_DAYS(space, w_delta) == 10
+        assert PyDateTime_DELTA_GET_SECONDS(space, w_delta) == 20
+        assert PyDateTime_DELTA_GET_MICROSECONDS(space, w_delta) == 30
 
-    def test_fromtimestamp(self, space, api):
+    def test_fromtimestamp(self, space):
         w_args = space.wrap((0,))
-        w_date = api.PyDate_FromTimestamp(w_args)
+        w_date = PyDate_FromTimestamp(space, w_args)
         date = datetime.date.fromtimestamp(0)
         assert space.unwrap(space.str(w_date)) == str(date)
 
         w_args = space.wrap((0,))
-        w_date = api.PyDateTime_FromTimestamp(w_args)
+        w_date = PyDateTime_FromTimestamp(space, w_args)
         date = datetime.datetime.fromtimestamp(0)
         assert space.unwrap(space.str(w_date)) == str(date)
 
-    def test_tzinfo(self, space, api):
+    def test_tzinfo(self, space):
         w_tzinfo = space.appexec(
             [], """():
             from datetime import tzinfo
             return tzinfo()
         """)
-        assert api.PyTZInfo_Check(w_tzinfo)
-        assert api.PyTZInfo_CheckExact(w_tzinfo)
-        assert not api.PyTZInfo_Check(space.w_None)
+        assert PyTZInfo_Check(space, w_tzinfo)
+        assert PyTZInfo_CheckExact(space, w_tzinfo)
+        assert not PyTZInfo_Check(space, space.w_None)
 
 class AppTestDatetime(AppTestCpythonExtensionBase):
     def test_CAPI(self):
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1456,3 +1456,9 @@
     def test_duplicate_slot_name(self):
         class X:   # does not raise
             __slots__ = 'a', 'a'
+
+    def test_descriptor_objclass(self):
+        class X(object):
+            pass
+        assert X.__dict__['__dict__'].__objclass__ is X
+        assert X.__dict__['__weakref__'].__objclass__ is X
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1160,14 +1160,16 @@
 
 def create_dict_slot(w_self):
     if not w_self.hasdict:
+        descr = dict_descr.copy_for_type(w_self)
         w_self.dict_w.setdefault('__dict__',
-                                 w_self.space.wrap(dict_descr))
+                                 w_self.space.wrap(descr))
         w_self.hasdict = True
 
 def create_weakref_slot(w_self):
     if not w_self.weakrefable:
+        descr = weakref_descr.copy_for_type(w_self)
         w_self.dict_w.setdefault('__weakref__',
-                                 w_self.space.wrap(weakref_descr))
+                                 w_self.space.wrap(descr))
         w_self.weakrefable = True
 
 def setup_user_defined_type(w_self, force_new_layout):

From pypy.commits at gmail.com  Wed Jan 11 14:03:47 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 11 Jan 2017 11:03:47 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove uses of api.XXX magic in
 test_dictobject
Message-ID: <58768193.2717c30a.b5839.ce05@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89502:6ba9b4214af5
Date: 2017-01-11 19:01 +0000
http://bitbucket.org/pypy/pypy/changeset/6ba9b4214af5/

Log:	Remove uses of api.XXX magic in test_dictobject

diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -1,83 +1,81 @@
 import py
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP, PyTypeObjectPtr
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.dictobject import *
+from pypy.module.cpyext.pyobject import decref
 
 class TestDictObject(BaseApiTest):
-    def test_dict(self, space, api):
-        d = api.PyDict_New()
+    def test_dict(self, space):
+        d = PyDict_New(space)
         assert space.eq_w(d, space.newdict())
 
-        assert space.eq_w(api.PyDict_GetItem(space.wrap({"a": 72}),
+        assert space.eq_w(PyDict_GetItem(space, space.wrap({"a": 72}),
                                              space.wrap("a")),
                           space.wrap(72))
 
-        assert api.PyDict_SetItem(d, space.wrap("c"), space.wrap(42)) >= 0
+        PyDict_SetItem(space, d, space.wrap("c"), space.wrap(42))
         assert space.eq_w(space.getitem(d, space.wrap("c")),
                           space.wrap(42))
 
         space.setitem(d, space.wrap("name"), space.wrap(3))
-        assert space.eq_w(api.PyDict_GetItem(d, space.wrap("name")),
+        assert space.eq_w(PyDict_GetItem(space, d, space.wrap("name")),
                           space.wrap(3))
 
         space.delitem(d, space.wrap("name"))
-        assert not api.PyDict_GetItem(d, space.wrap("name"))
-        assert not api.PyErr_Occurred()
+        assert not PyDict_GetItem(space, d, space.wrap("name"))
 
         buf = rffi.str2charp("name")
-        assert not api.PyDict_GetItemString(d, buf)
+        assert not PyDict_GetItemString(space, d, buf)
         rffi.free_charp(buf)
-        assert not api.PyErr_Occurred()
 
-        assert api.PyDict_Contains(d, space.wrap("c"))
-        assert not api.PyDict_Contains(d, space.wrap("z"))
+        assert PyDict_Contains(space, d, space.wrap("c"))
+        assert not PyDict_Contains(space, d, space.wrap("z"))
 
-        assert api.PyDict_DelItem(d, space.wrap("c")) == 0
-        assert api.PyDict_DelItem(d, space.wrap("name")) < 0
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
-        assert api.PyDict_Size(d) == 0
+        PyDict_DelItem(space, d, space.wrap("c"))
+        with raises_w(space, KeyError):
+            PyDict_DelItem(space, d, space.wrap("name"))
+        assert PyDict_Size(space, d) == 0
 
         space.setitem(d, space.wrap("some_key"), space.wrap(3))
         buf = rffi.str2charp("some_key")
-        assert api.PyDict_DelItemString(d, buf) == 0
-        assert api.PyDict_Size(d) == 0
-        assert api.PyDict_DelItemString(d, buf) < 0
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
+        PyDict_DelItemString(space, d, buf)
+        assert PyDict_Size(space, d) == 0
+        with raises_w(space, KeyError):
+            PyDict_DelItemString(space, d, buf)
         rffi.free_charp(buf)
 
         d = space.wrap({'a': 'b'})
-        api.PyDict_Clear(d)
-        assert api.PyDict_Size(d) == 0
+        PyDict_Clear(space, d)
+        assert PyDict_Size(space, d) == 0
 
-    def test_check(self, space, api):
-        d = api.PyDict_New()
-        assert api.PyDict_Check(d)
-        assert api.PyDict_CheckExact(d)
+    def test_check(self, space):
+        d = PyDict_New(space, )
+        assert PyDict_Check(space, d)
+        assert PyDict_CheckExact(space, d)
         sub = space.appexec([], """():
             class D(dict):
                 pass
             return D""")
         d = space.call_function(sub)
-        assert api.PyDict_Check(d)
-        assert not api.PyDict_CheckExact(d)
+        assert PyDict_Check(space, d)
+        assert not PyDict_CheckExact(space, d)
         i = space.wrap(2)
-        assert not api.PyDict_Check(i)
-        assert not api.PyDict_CheckExact(i)
+        assert not PyDict_Check(space, i)
+        assert not PyDict_CheckExact(space, i)
 
-    def test_keys(self, space, api):
+    def test_keys(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
-        assert space.eq_w(api.PyDict_Keys(w_d), space.wrap(["a"]))
-        assert space.eq_w(api.PyDict_Values(w_d), space.wrap(["b"]))
-        assert space.eq_w(api.PyDict_Items(w_d), space.wrap([("a", "b")]))
+        assert space.eq_w(PyDict_Keys(space, w_d), space.wrap(["a"]))
+        assert space.eq_w(PyDict_Values(space, w_d), space.wrap(["b"]))
+        assert space.eq_w(PyDict_Items(space, w_d), space.wrap([("a", "b")]))
 
-    def test_merge(self, space, api):
+    def test_merge(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
@@ -86,35 +84,34 @@
         space.setitem(w_d2, space.wrap("c"), space.wrap("d"))
         space.setitem(w_d2, space.wrap("e"), space.wrap("f"))
 
-        api.PyDict_Merge(w_d, w_d2, 0)
+        PyDict_Merge(space, w_d, w_d2, 0)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
-        api.PyDict_Merge(w_d, w_d2, 1)
+        PyDict_Merge(space, w_d, w_d2, 1)
         assert space.unwrap(w_d) == dict(a='c', c='d', e='f')
 
-    def test_update(self, space, api):
+    def test_update(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
-        w_d2 = api.PyDict_Copy(w_d)
+        w_d2 = PyDict_Copy(space, w_d)
         assert not space.is_w(w_d2, w_d)
         space.setitem(w_d, space.wrap("c"), space.wrap("d"))
         space.setitem(w_d2, space.wrap("e"), space.wrap("f"))
 
-        api.PyDict_Update(w_d, w_d2)
+        PyDict_Update(space, w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
-    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+    def test_update_doesnt_accept_list_of_tuples(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
         w_d2 = space.wrap([("c", "d"), ("e", "f")])
 
-        api.PyDict_Update(w_d, w_d2)
-        assert api.PyErr_Occurred() is space.w_AttributeError
-        api.PyErr_Clear()
+        with raises_w(space, AttributeError):
+            PyDict_Update(space, w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b') # unchanged
 
-    def test_iter(self, space, api):
+    def test_iter(self, space):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
 
@@ -125,7 +122,7 @@
 
         try:
             w_copy = space.newdict()
-            while api.PyDict_Next(w_dict, ppos, pkey, pvalue):
+            while PyDict_Next(space, w_dict, ppos, pkey, pvalue):
                 w_key = from_ref(space, pkey[0])
                 w_value = from_ref(space, pvalue[0])
                 space.setitem(w_copy, w_key, w_value)
@@ -134,12 +131,12 @@
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
 
-        api.Py_DecRef(py_dict) # release borrowed references
+        decref(space, py_dict) # release borrowed references
 
         assert space.eq_w(space.len(w_copy), space.len(w_dict))
         assert space.eq_w(w_copy, w_dict)
 
-    def test_iterkeys(self, space, api):
+    def test_iterkeys(self, space):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
 
@@ -151,11 +148,11 @@
         values_w = []
         try:
             ppos[0] = 0
-            while api.PyDict_Next(w_dict, ppos, pkey, None):
+            while PyDict_Next(space, w_dict, ppos, pkey, None):
                 w_key = from_ref(space, pkey[0])
                 keys_w.append(w_key)
             ppos[0] = 0
-            while api.PyDict_Next(w_dict, ppos, None, pvalue):
+            while PyDict_Next(space, w_dict, ppos, None, pvalue):
                 w_value = from_ref(space, pvalue[0])
                 values_w.append(w_value)
         finally:
@@ -163,25 +160,25 @@
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
 
-        api.Py_DecRef(py_dict) # release borrowed references
+        decref(space, py_dict) # release borrowed references
 
         assert space.eq_w(space.newlist(keys_w),
                           space.call_method(w_dict, "keys"))
         assert space.eq_w(space.newlist(values_w),
                           space.call_method(w_dict, "values"))
 
-    def test_dictproxy(self, space, api):
+    def test_dictproxy(self, space):
         w_dict = space.sys.get('modules')
-        w_proxy = api.PyDictProxy_New(w_dict)
+        w_proxy = PyDictProxy_New(space, w_dict)
         assert space.contains_w(w_proxy, space.wrap('sys'))
         raises(OperationError, space.setitem,
                w_proxy, space.wrap('sys'), space.w_None)
         raises(OperationError, space.delitem,
                w_proxy, space.wrap('sys'))
         raises(OperationError, space.call_method, w_proxy, 'clear')
-        assert api.PyDictProxy_Check(w_proxy)
+        assert PyDictProxy_Check(space, w_proxy)
 
-    def test_typedict1(self, space, api):
+    def test_typedict1(self, space):
         py_type = make_ref(space, space.w_int)
         py_dict = rffi.cast(PyTypeObjectPtr, py_type).c_tp_dict
         ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
@@ -191,7 +188,7 @@
         pvalue = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         try:
             w_copy = space.newdict()
-            while api.PyDict_Next(py_dict, ppos, pkey, pvalue):
+            while PyDict_Next(space, py_dict, ppos, pkey, pvalue):
                 w_key = from_ref(space, pkey[0])
                 w_value = from_ref(space, pvalue[0])
                 space.setitem(w_copy, w_key, w_value)
@@ -199,7 +196,7 @@
             lltype.free(ppos, flavor='raw')
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
-        api.Py_DecRef(py_type) # release borrowed references
+        decref(space, py_type) # release borrowed references
         # do something with w_copy ?
 
 class AppTestDictObject(AppTestCpythonExtensionBase):

From pypy.commits at gmail.com  Wed Jan 11 14:08:07 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 11 Jan 2017 11:08:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix translation
Message-ID: <58768297.0e821c0a.f4ac4.fcb8@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89503:73e4cf94fc54
Date: 2017-01-11 19:07 +0000
http://bitbucket.org/pypy/pypy/changeset/73e4cf94fc54/

Log:	fix translation

diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -491,7 +491,8 @@
     share a contiguous chunk of memory of "unsigned bytes" of the given
     length. Returns 0 on success and -1 (with raising an error) on error.
     """
-    if rffi.cast(lltype.Signed, flags) & PyBUF_WRITABLE and readonly:
+    flags = rffi.cast(lltype.Signed, flags)
+    if flags & PyBUF_WRITABLE and readonly:
         raise oefmt(space.w_ValueError, "Object is not writable")
     view.c_buf = buf
     view.c_len = length

From pypy.commits at gmail.com  Wed Jan 11 15:40:05 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 11 Jan 2017 12:40:05 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove more uses of api.XXX magic
Message-ID: <58769825.06891c0a.fffe2.0690@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89504:094c5770114b
Date: 2017-01-11 20:39 +0000
http://bitbucket.org/pypy/pypy/changeset/094c5770114b/

Log:	Remove more uses of api.XXX magic

diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -1,17 +1,26 @@
+import sys
+import os
+import pytest
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.object import PyObject_Size, PyObject_GetItem
+from pypy.module.cpyext.pythonrun import Py_AtExit
 from pypy.module.cpyext.eval import (
-    Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import (c_fopen, c_fclose, c_fileno, 
-                Py_ssize_tP, is_valid_fd)
+    Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags,
+    PyEval_CallObjectWithKeywords, PyObject_CallObject, PyEval_EvalCode,
+    PyRun_SimpleString, PyRun_String, PyRun_StringFlags, PyRun_File,
+    PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals,
+    _PyEval_SliceIndex)
+from pypy.module.cpyext.api import (
+    c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd)
 from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
-import sys, os
 
 class TestEval(BaseApiTest):
-    def test_eval(self, space, api):
+    def test_eval(self, space):
         w_l, w_f = space.fixedview(space.appexec([], """():
         l = []
         def f(arg1, arg2):
@@ -22,7 +31,7 @@
         """))
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, None)
+        w_res = PyEval_CallObjectWithKeywords(space, w_f, w_t, None)
         assert space.int_w(w_res) == 2
         assert space.len_w(w_l) == 2
         w_f = space.appexec([], """():
@@ -35,10 +44,10 @@
         w_t = space.newtuple([space.w_None, space.w_None])
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("xyz"), space.wrap(3))
-        w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, w_d)
+        w_res = PyEval_CallObjectWithKeywords(space, w_f, w_t, w_d)
         assert space.int_w(w_res) == 21
 
-    def test_call_object(self, space, api):
+    def test_call_object(self, space):
         w_l, w_f = space.fixedview(space.appexec([], """():
         l = []
         def f(arg1, arg2):
@@ -49,7 +58,7 @@
         """))
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyObject_CallObject(w_f, w_t)
+        w_res = PyObject_CallObject(space, w_f, w_t)
         assert space.int_w(w_res) == 2
         assert space.len_w(w_l) == 2
 
@@ -61,11 +70,11 @@
             """)
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyObject_CallObject(w_f, w_t)
+        w_res = PyObject_CallObject(space, w_f, w_t)
 
         assert space.int_w(w_res) == 10
 
-    def test_evalcode(self, space, api):
+    def test_evalcode(self, space):
         w_f = space.appexec([], """():
             def f(*args):
                 assert isinstance(args, tuple)
@@ -77,84 +86,78 @@
         w_globals = space.newdict()
         w_locals = space.newdict()
         space.setitem(w_locals, space.wrap("args"), w_t)
-        w_res = api.PyEval_EvalCode(w_f.code, w_globals, w_locals)
+        w_res = PyEval_EvalCode(space, w_f.code, w_globals, w_locals)
 
         assert space.int_w(w_res) == 10
 
-    def test_run_simple_string(self, space, api):
+    def test_run_simple_string(self, space):
         def run(code):
             buf = rffi.str2charp(code)
             try:
-                return api.PyRun_SimpleString(buf)
+                return PyRun_SimpleString(space, buf)
             finally:
                 rffi.free_charp(buf)
 
-        assert 0 == run("42 * 43")
+        assert run("42 * 43") == 0  # no error
+        with pytest.raises(OperationError):
+            run("4..3 * 43")
 
-        assert -1 == run("4..3 * 43")
-
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
-
-    def test_run_string(self, space, api):
+    def test_run_string(self, space):
         def run(code, start, w_globals, w_locals):
             buf = rffi.str2charp(code)
             try:
-                return api.PyRun_String(buf, start, w_globals, w_locals)
+                return PyRun_String(space, buf, start, w_globals, w_locals)
             finally:
                 rffi.free_charp(buf)
 
         w_globals = space.newdict()
         assert 42 * 43 == space.unwrap(
             run("42 * 43", Py_eval_input, w_globals, w_globals))
-        assert api.PyObject_Size(w_globals) == 0
+        assert PyObject_Size(space, w_globals) == 0
 
         assert run("a = 42 * 43", Py_single_input,
                    w_globals, w_globals) == space.w_None
         assert 42 * 43 == space.unwrap(
-            api.PyObject_GetItem(w_globals, space.wrap("a")))
+            PyObject_GetItem(space, w_globals, space.wrap("a")))
 
-    def test_run_string_flags(self, space, api):
+    def test_run_string_flags(self, space):
         flags = lltype.malloc(PyCompilerFlags, flavor='raw')
         flags.c_cf_flags = rffi.cast(rffi.INT, consts.PyCF_SOURCE_IS_UTF8)
         w_globals = space.newdict()
         buf = rffi.str2charp("a = u'caf\xc3\xa9'")
         try:
-            api.PyRun_StringFlags(buf, Py_single_input,
-                                  w_globals, w_globals, flags)
+            PyRun_StringFlags(space, buf, Py_single_input, w_globals,
+                              w_globals, flags)
         finally:
             rffi.free_charp(buf)
         w_a = space.getitem(w_globals, space.wrap("a"))
         assert space.unwrap(w_a) == u'caf\xe9'
         lltype.free(flags, flavor='raw')
 
-    def test_run_file(self, space, api):
+    def test_run_file(self, space):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
         fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
-        api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
+        with raises_w(space, ZeroDivisionError):
+            PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
         c_fclose(fp)
-        assert api.PyErr_Occurred() is space.w_ZeroDivisionError
-        api.PyErr_Clear()
 
         # try again, but with a closed file
         fp = c_fopen(str(filepath), "rb")
         os.close(c_fileno(fp))
-        api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
+        with raises_w(space, IOError):
+            PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
         if is_valid_fd(c_fileno(fp)):
             c_fclose(fp)
-        assert api.PyErr_Occurred() is space.w_IOError
-        api.PyErr_Clear()
-
         rffi.free_charp(filename)
 
-    def test_getbuiltins(self, space, api):
-        assert api.PyEval_GetBuiltins() is space.builtin.w_dict
+    def test_getbuiltins(self, space):
+        assert PyEval_GetBuiltins(space) is space.builtin.w_dict
 
         def cpybuiltins(space):
-            return api.PyEval_GetBuiltins()
+            return PyEval_GetBuiltins(space)
         w_cpybuiltins = space.wrap(interp2app(cpybuiltins))
 
         w_result = space.appexec([w_cpybuiltins], """(cpybuiltins):
@@ -168,13 +171,13 @@
         """)
         assert space.len_w(w_result) == 1
 
-    def test_getglobals(self, space, api):
-        assert api.PyEval_GetLocals() is None
-        assert api.PyEval_GetGlobals() is None
+    def test_getglobals(self, space):
+        assert PyEval_GetLocals(space) is None
+        assert PyEval_GetGlobals(space) is None
 
         def cpyvars(space):
-            return space.newtuple([api.PyEval_GetGlobals(),
-                                   api.PyEval_GetLocals()])
+            return space.newtuple([PyEval_GetGlobals(space),
+                                   PyEval_GetLocals(space)])
         w_cpyvars = space.wrap(interp2app(cpyvars))
 
         w_result = space.appexec([w_cpyvars], """(cpyvars):
@@ -186,26 +189,26 @@
         assert sorted(locals) == ['cpyvars', 'x']
         assert sorted(globals) == ['__builtins__', 'anonymous', 'y']
 
-    def test_sliceindex(self, space, api):
+    def test_sliceindex(self, space):
         pi = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
-        assert api._PyEval_SliceIndex(space.w_None, pi) == 0
-        api.PyErr_Clear()
+        with pytest.raises(OperationError):
+            _PyEval_SliceIndex(space, space.w_None, pi)
 
-        assert api._PyEval_SliceIndex(space.wrap(123), pi) == 1
+        assert _PyEval_SliceIndex(space, space.wrap(123), pi) == 1
         assert pi[0] == 123
 
-        assert api._PyEval_SliceIndex(space.wrap(1 << 66), pi) == 1
+        assert _PyEval_SliceIndex(space, space.wrap(1 << 66), pi) == 1
         assert pi[0] == sys.maxint
 
         lltype.free(pi, flavor='raw')
 
-    def test_atexit(self, space, api):
+    def test_atexit(self, space):
         lst = []
         def func():
             lst.append(42)
-        api.Py_AtExit(func)
+        Py_AtExit(space, func)
         cpyext = space.getbuiltinmodule('cpyext')
-        cpyext.shutdown(space) # simulate shutdown
+        cpyext.shutdown(space)  # simulate shutdown
         assert lst == [42]
 
 class AppTestCall(AppTestCpythonExtensionBase):
@@ -269,6 +272,7 @@
                 return res;
              """),
             ])
+
         def f(*args):
             return args
         assert module.call_func(f) == (None,)
@@ -322,7 +326,7 @@
             ])
         assert module.get_flags() == (0, 0)
 
-        ns = {'module':module}
+        ns = {'module': module}
         exec """from __future__ import division    \nif 1:
                 def nested_flags():
                     return module.get_flags()""" in ns
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ b/pypy/module/cpyext/test/test_floatobject.py
@@ -1,36 +1,40 @@
+import pytest
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.floatobject import (
+    PyFloat_FromDouble, PyFloat_AsDouble, PyFloat_AS_DOUBLE, PyNumber_Float,
+    _PyFloat_Unpack4, _PyFloat_Unpack8)
 
 class TestFloatObject(BaseApiTest):
-    def test_floatobject(self, space, api):
-        assert space.unwrap(api.PyFloat_FromDouble(3.14)) == 3.14
-        assert api.PyFloat_AsDouble(space.wrap(23.45)) == 23.45
-        assert api.PyFloat_AS_DOUBLE(space.wrap(23.45)) == 23.45
+    def test_floatobject(self, space):
+        assert space.unwrap(PyFloat_FromDouble(space, 3.14)) == 3.14
+        assert PyFloat_AsDouble(space, space.wrap(23.45)) == 23.45
+        assert PyFloat_AS_DOUBLE(space, space.wrap(23.45)) == 23.45
+        with pytest.raises(OperationError):
+            PyFloat_AsDouble(space, space.w_None)
 
-        assert api.PyFloat_AsDouble(space.w_None) == -1
-        api.PyErr_Clear()
-
-    def test_coerce(self, space, api):
-        assert space.type(api.PyNumber_Float(space.wrap(3))) is space.w_float
-        assert space.type(api.PyNumber_Float(space.wrap("3"))) is space.w_float
+    def test_coerce(self, space):
+        assert space.type(PyNumber_Float(space, space.wrap(3))) is space.w_float
+        assert space.type(PyNumber_Float(space, space.wrap("3"))) is space.w_float
 
         w_obj = space.appexec([], """():
             class Coerce(object):
                 def __float__(self):
                     return 42.5
             return Coerce()""")
-        assert space.eq_w(api.PyNumber_Float(w_obj), space.wrap(42.5))
+        assert space.eq_w(PyNumber_Float(space, w_obj), space.wrap(42.5))
 
-    def test_unpack(self, space, api):
+    def test_unpack(self, space):
         with rffi.scoped_str2charp("\x9a\x99\x99?") as ptr:
-            assert abs(api._PyFloat_Unpack4(ptr, 1) - 1.2) < 1e-7
+            assert abs(_PyFloat_Unpack4(space, ptr, 1) - 1.2) < 1e-7
         with rffi.scoped_str2charp("?\x99\x99\x9a") as ptr:
-            assert abs(api._PyFloat_Unpack4(ptr, 0) - 1.2) < 1e-7
+            assert abs(_PyFloat_Unpack4(space, ptr, 0) - 1.2) < 1e-7
         with rffi.scoped_str2charp("\x1f\x85\xebQ\xb8\x1e\t@") as ptr:
-            assert abs(api._PyFloat_Unpack8(ptr, 1) - 3.14) < 1e-15
+            assert abs(_PyFloat_Unpack8(space, ptr, 1) - 3.14) < 1e-15
         with rffi.scoped_str2charp("@\t\x1e\xb8Q\xeb\x85\x1f") as ptr:
-            assert abs(api._PyFloat_Unpack8(ptr, 0) - 3.14) < 1e-15
+            assert abs(_PyFloat_Unpack8(space, ptr, 0) - 3.14) < 1e-15
 
 class AppTestFloatObject(AppTestCpythonExtensionBase):
     def test_fromstring(self):
@@ -79,8 +83,6 @@
         assert math.isinf(neginf)
 
     def test_macro_accepts_wrong_pointer_type(self):
-        import math
-
         module = self.import_extension('foo', [
             ("test_macros", "METH_NOARGS",
              """
diff --git a/pypy/module/cpyext/test/test_funcobject.py b/pypy/module/cpyext/test/test_funcobject.py
--- a/pypy/module/cpyext/test/test_funcobject.py
+++ b/pypy/module/cpyext/test/test_funcobject.py
@@ -1,16 +1,18 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
+from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, decref
+from pypy.module.cpyext.methodobject import PyClassMethod_New
 from pypy.module.cpyext.funcobject import (
-    PyFunctionObject, PyCodeObject, CODE_FLAGS)
-from pypy.interpreter.function import Function, Method
+    PyFunctionObject, PyCodeObject, CODE_FLAGS, PyMethod_Function,
+    PyMethod_Self, PyMethod_Class, PyMethod_New, PyFunction_GetCode,
+    PyCode_NewEmpty, PyCode_GetNumFree)
+from pypy.interpreter.function import Function
 from pypy.interpreter.pycode import PyCode
 
 globals().update(CODE_FLAGS)
 
 class TestFunctionObject(BaseApiTest):
-    def test_function(self, space, api):
+    def test_function(self, space):
         w_function = space.appexec([], """():
             def f(): pass
             return f
@@ -19,10 +21,10 @@
         assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is
                 space.gettypeobject(Function.typedef))
         assert "f" == space.unwrap(
-           from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name))
-        api.Py_DecRef(ref)
+            from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name))
+        decref(space, ref)
 
-    def test_method(self, space, api):
+    def test_method(self, space):
         w_method = space.appexec([], """():
             class C(list):
                 def method(self): pass
@@ -33,30 +35,30 @@
         w_self = space.getattr(w_method, space.wrap("im_self"))
         w_class = space.getattr(w_method, space.wrap("im_class"))
 
-        assert space.is_w(api.PyMethod_Function(w_method), w_function)
-        assert space.is_w(api.PyMethod_Self(w_method), w_self)
-        assert space.is_w(api.PyMethod_Class(w_method), w_class)
+        assert space.is_w(PyMethod_Function(space, w_method), w_function)
+        assert space.is_w(PyMethod_Self(space, w_method), w_self)
+        assert space.is_w(PyMethod_Class(space, w_method), w_class)
 
-        w_method2 = api.PyMethod_New(w_function, w_self, w_class)
+        w_method2 = PyMethod_New(space, w_function, w_self, w_class)
         assert space.eq_w(w_method, w_method2)
 
-    def test_getcode(self, space, api):
+    def test_getcode(self, space):
         w_function = space.appexec([], """():
             def func(x, y, z): return x
             return func
         """)
-        w_code = api.PyFunction_GetCode(w_function)
+        w_code = PyFunction_GetCode(space, w_function)
         assert w_code.co_name == "func"
 
         ref = make_ref(space, w_code)
         assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is
                 space.gettypeobject(PyCode.typedef))
         assert "func" == space.unwrap(
-           from_ref(space, rffi.cast(PyCodeObject, ref).c_co_name))
+            from_ref(space, rffi.cast(PyCodeObject, ref).c_co_name))
         assert 3 == rffi.cast(PyCodeObject, ref).c_co_argcount
-        api.Py_DecRef(ref)
+        decref(space, ref)
 
-    def test_co_flags(self, space, api):
+    def test_co_flags(self, space):
         def get_flags(signature, body="pass"):
             w_code = space.appexec([], """():
                 def func(%s): %s
@@ -64,7 +66,7 @@
             """ % (signature, body))
             ref = make_ref(space, w_code)
             co_flags = rffi.cast(PyCodeObject, ref).c_co_flags
-            api.Py_DecRef(ref)
+            decref(space, ref)
             return co_flags
         assert get_flags("x") == CO_NESTED | CO_OPTIMIZED | CO_NEWLOCALS
         assert get_flags("x", "exec x") == CO_NESTED | CO_NEWLOCALS
@@ -72,29 +74,29 @@
         assert get_flags("x, **kw") & CO_VARKEYWORDS
         assert get_flags("x", "yield x") & CO_GENERATOR
 
-    def test_newcode(self, space, api):
+    def test_newcode(self, space):
         filename = rffi.str2charp('filename')
         funcname = rffi.str2charp('funcname')
-        w_code = api.PyCode_NewEmpty(filename, funcname, 3)
+        w_code = PyCode_NewEmpty(space, filename, funcname, 3)
         assert w_code.co_filename == 'filename'
         assert w_code.co_firstlineno == 3
 
         ref = make_ref(space, w_code)
         assert "filename" == space.unwrap(
             from_ref(space, rffi.cast(PyCodeObject, ref).c_co_filename))
-        api.Py_DecRef(ref)
+        decref(space, ref)
         rffi.free_charp(filename)
         rffi.free_charp(funcname)
 
-    def test_getnumfree(self, space, api):
+    def test_getnumfree(self, space):
         w_function = space.appexec([], """():
             a = 5
             def method(x): return a, x
             return method
         """)
-        assert api.PyCode_GetNumFree(w_function.code) == 1
+        assert PyCode_GetNumFree(space, w_function.code) == 1
 
-    def test_classmethod(self, space, api):
+    def test_classmethod(self, space):
         w_function = space.appexec([], """():
             def method(x): return x
             return method
@@ -106,6 +108,7 @@
         space.setattr(w_class, space.wrap("method"), w_function)
         assert space.is_w(space.call_method(w_instance, "method"), w_instance)
         # now a classmethod
-        w_classmethod = api.PyClassMethod_New(w_function)
+        w_classmethod = PyClassMethod_New(space, w_function)
         space.setattr(w_class, space.wrap("classmethod"), w_classmethod)
-        assert space.is_w(space.call_method(w_instance, "classmethod"), w_class)
+        assert space.is_w(
+            space.call_method(w_instance, "classmethod"), w_class)
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -1,48 +1,51 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.import_ import *
+from pypy.module.cpyext.import_ import (
+    _PyImport_AcquireLock, _PyImport_ReleaseLock)
 from rpython.rtyper.lltypesystem import rffi
 
 class TestImport(BaseApiTest):
-    def test_import(self, space, api):
-        stat = api.PyImport_Import(space.wrap("stat"))
+    def test_import(self, space):
+        stat = PyImport_Import(space, space.wrap("stat"))
         assert stat
         assert space.getattr(stat, space.wrap("S_IMODE"))
 
-    def test_addmodule(self, space, api):
+    def test_addmodule(self, space):
         with rffi.scoped_str2charp("sys") as modname:
-            w_sys = api.PyImport_AddModule(modname)
+            w_sys = PyImport_AddModule(space, modname)
         assert w_sys is space.sys
 
         with rffi.scoped_str2charp("foobar") as modname:
-            w_foobar = api.PyImport_AddModule(modname)
+            w_foobar = PyImport_AddModule(space, modname)
         assert space.str_w(space.getattr(w_foobar,
                                          space.wrap('__name__'))) == 'foobar'
 
-    def test_getmoduledict(self, space, api):
+    def test_getmoduledict(self, space):
         testmod = "_functools"
-        w_pre_dict = api.PyImport_GetModuleDict()
+        w_pre_dict = PyImport_GetModuleDict(space, )
         assert not space.contains_w(w_pre_dict, space.wrap(testmod))
 
         with rffi.scoped_str2charp(testmod) as modname:
-            w_module = api.PyImport_ImportModule(modname)
+            w_module = PyImport_ImportModule(space, modname)
             print w_module
             assert w_module
 
-        w_dict = api.PyImport_GetModuleDict()
+        w_dict = PyImport_GetModuleDict(space, )
         assert space.contains_w(w_dict, space.wrap(testmod))
 
-    def test_reload(self, space, api):
-        stat = api.PyImport_Import(space.wrap("stat"))
+    def test_reload(self, space):
+        stat = PyImport_Import(space, space.wrap("stat"))
         space.delattr(stat, space.wrap("S_IMODE"))
-        stat = api.PyImport_ReloadModule(stat)
+        stat = PyImport_ReloadModule(space, stat)
         assert space.getattr(stat, space.wrap("S_IMODE"))
 
-    def test_lock(self, space, api):
+    def test_lock(self, space):
         # "does not crash"
-        api._PyImport_AcquireLock()
-        api._PyImport_AcquireLock()
-        api._PyImport_ReleaseLock()
-        api._PyImport_ReleaseLock()
+        _PyImport_AcquireLock(space, )
+        _PyImport_AcquireLock(space, )
+        _PyImport_ReleaseLock(space, )
+        _PyImport_ReleaseLock(space, )
 
 
 class AppTestImportLogic(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py
--- a/pypy/module/cpyext/test/test_intobject.py
+++ b/pypy/module/cpyext/test/test_intobject.py
@@ -1,51 +1,52 @@
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.intobject import (
+    PyInt_Check, PyInt_AsLong, PyInt_AS_LONG, PyInt_FromLong,
+    PyInt_AsUnsignedLong, PyInt_AsUnsignedLongMask,
+    PyInt_AsUnsignedLongLongMask)
 import sys
 
 class TestIntObject(BaseApiTest):
-    def test_intobject(self, space, api):
-        assert api.PyInt_Check(space.wrap(3))
-        assert api.PyInt_Check(space.w_True)
-        assert not api.PyInt_Check(space.wrap((1, 2, 3)))
+    def test_intobject(self, space):
+        assert PyInt_Check(space, space.wrap(3))
+        assert PyInt_Check(space, space.w_True)
+        assert not PyInt_Check(space, space.wrap((1, 2, 3)))
         for i in [3, -5, -1, -sys.maxint, sys.maxint - 1]:
-            x = api.PyInt_AsLong(space.wrap(i))
-            y = api.PyInt_AS_LONG(space.wrap(i))
+            x = PyInt_AsLong(space, space.wrap(i))
+            y = PyInt_AS_LONG(space, space.wrap(i))
             assert x == i
             assert y == i
-            w_x = api.PyInt_FromLong(x + 1)
+            w_x = PyInt_FromLong(space, x + 1)
             assert space.type(w_x) is space.w_int
             assert space.eq_w(w_x, space.wrap(i + 1))
 
-        assert api.PyInt_AsLong(space.w_None) == -1
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyInt_AsLong(space, space.w_None)
 
-        assert api.PyInt_AsLong(None) == -1
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyInt_AsLong(space, None)
 
-        assert api.PyInt_AsUnsignedLong(space.wrap(sys.maxint)) == sys.maxint
-        assert api.PyInt_AsUnsignedLong(space.wrap(-5)) == sys.maxint * 2 + 1
-        assert api.PyErr_Occurred() is space.w_ValueError
-        api.PyErr_Clear()
+        assert PyInt_AsUnsignedLong(space, space.wrap(sys.maxint)) == sys.maxint
+        with raises_w(space, ValueError):
+            PyInt_AsUnsignedLong(space, space.wrap(-5))
 
-        assert (api.PyInt_AsUnsignedLongMask(space.wrap(sys.maxint))
+        assert (PyInt_AsUnsignedLongMask(space, space.wrap(sys.maxint))
                 == sys.maxint)
-        assert (api.PyInt_AsUnsignedLongMask(space.wrap(10**30))
-                == 10**30 % ((sys.maxint + 1) * 2))
+        assert (PyInt_AsUnsignedLongMask(space, space.wrap(10 ** 30))
+                == 10 ** 30 % ((sys.maxint + 1) * 2))
 
-        assert (api.PyInt_AsUnsignedLongLongMask(space.wrap(sys.maxint))
+        assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(sys.maxint))
                 == sys.maxint)
-        assert (api.PyInt_AsUnsignedLongLongMask(space.wrap(10**30))
-                == 10**30 % (2**64))
+        assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(10 ** 30))
+                == 10 ** 30 % (2 ** 64))
 
-    def test_coerce(self, space, api):
+    def test_coerce(self, space):
         w_obj = space.appexec([], """():
             class Coerce(object):
                 def __int__(self):
                     return 42
             return Coerce()""")
-        assert api.PyInt_AsLong(w_obj) == 42
+        assert PyInt_AsLong(space, w_obj) == 42
 
 class AppTestIntObject(AppTestCpythonExtensionBase):
     def test_fromstring(self):
@@ -203,4 +204,3 @@
                 """
                 ),
                 ])
-
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -1,5 +1,6 @@
 # encoding: utf-8
-from pypy.module.cpyext.test.test_api import BaseApiTest
+import pytest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.unicodeobject import (
     Py_UNICODE, PyUnicodeObject, new_empty_unicode)
@@ -7,6 +8,7 @@
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref
 from rpython.rtyper.lltypesystem import rffi, lltype
 import sys, py
+from pypy.module.cpyext.unicodeobject import *
 
 class AppTestUnicodeObject(AppTestCpythonExtensionBase):
     def test_unicodeobject(self):
@@ -131,13 +133,13 @@
         assert module.test_macro_invocations() == u''
 
 class TestUnicode(BaseApiTest):
-    def test_unicodeobject(self, space, api):
-        assert api.PyUnicode_GET_SIZE(space.wrap(u'sp&#65533;m')) == 4
-        assert api.PyUnicode_GetSize(space.wrap(u'sp&#65533;m')) == 4
+    def test_unicodeobject(self, space):
+        assert PyUnicode_GET_SIZE(space, space.wrap(u'sp&#65533;m')) == 4
+        assert PyUnicode_GetSize(space, space.wrap(u'sp&#65533;m')) == 4
         unichar = rffi.sizeof(Py_UNICODE)
-        assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp&#65533;m')) == 4 * unichar
+        assert PyUnicode_GET_DATA_SIZE(space, space.wrap(u'sp&#65533;m')) == 4 * unichar
 
-        encoding = rffi.charp2str(api.PyUnicode_GetDefaultEncoding())
+        encoding = rffi.charp2str(PyUnicode_GetDefaultEncoding(space, ))
         w_default_encoding = space.call_function(
             space.sys.get('getdefaultencoding')
         )
@@ -145,43 +147,45 @@
         invalid = rffi.str2charp('invalid')
         utf_8 = rffi.str2charp('utf-8')
         prev_encoding = rffi.str2charp(space.unwrap(w_default_encoding))
-        self.raises(space, api, TypeError, api.PyUnicode_SetDefaultEncoding, lltype.nullptr(rffi.CCHARP.TO))
-        assert api.PyUnicode_SetDefaultEncoding(invalid) == -1
-        assert api.PyErr_Occurred() is space.w_LookupError
-        api.PyErr_Clear()
-        assert api.PyUnicode_SetDefaultEncoding(utf_8) == 0
-        assert rffi.charp2str(api.PyUnicode_GetDefaultEncoding()) == 'utf-8'
-        assert api.PyUnicode_SetDefaultEncoding(prev_encoding) == 0
+        with raises_w(space, TypeError):
+            PyUnicode_SetDefaultEncoding(space, lltype.nullptr(rffi.CCHARP.TO))
+        with raises_w(space, LookupError):
+            PyUnicode_SetDefaultEncoding(space, invalid)
+
+        assert PyUnicode_SetDefaultEncoding(space, utf_8) == 0
+        assert rffi.charp2str(PyUnicode_GetDefaultEncoding(space, )) == 'utf-8'
+        assert PyUnicode_SetDefaultEncoding(space, prev_encoding) == 0
         rffi.free_charp(invalid)
         rffi.free_charp(utf_8)
         rffi.free_charp(prev_encoding)
 
-    def test_AS(self, space, api):
+    def test_AS(self, space):
         word = space.wrap(u'spam')
-        array = rffi.cast(rffi.CWCHARP, api.PyUnicode_AS_DATA(word))
-        array2 = api.PyUnicode_AS_UNICODE(word)
-        array3 = api.PyUnicode_AsUnicode(word)
+        array = rffi.cast(rffi.CWCHARP, PyUnicode_AS_DATA(space, word))
+        array2 = PyUnicode_AS_UNICODE(space, word)
+        array3 = PyUnicode_AsUnicode(space, word)
         for (i, char) in enumerate(space.unwrap(word)):
             assert array[i] == char
             assert array2[i] == char
             assert array3[i] == char
-        self.raises(space, api, TypeError, api.PyUnicode_AsUnicode,
-                    space.wrap('spam'))
+        with raises_w(space, TypeError):
+            PyUnicode_AsUnicode(space, space.wrap('spam'))
 
         utf_8 = rffi.str2charp('utf-8')
-        encoded = api.PyUnicode_AsEncodedString(space.wrap(u'sp&#65533;m'),
+        encoded = PyUnicode_AsEncodedString(space, space.wrap(u'sp&#65533;m'),
                                                 utf_8, None)
         assert space.unwrap(encoded) == 'sp\xef\xbf\xbdm'
-        encoded_obj = api.PyUnicode_AsEncodedObject(space.wrap(u'sp&#65533;m'),
+        encoded_obj = PyUnicode_AsEncodedObject(space, space.wrap(u'sp&#65533;m'),
                                                 utf_8, None)
         assert space.eq_w(encoded, encoded_obj)
-        self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString,
-               space.newtuple([1, 2, 3]), None, None)
-        self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString,
-               space.wrap(''), None, None)
+        with raises_w(space, TypeError):
+            PyUnicode_AsEncodedString(
+                space, space.newtuple([1, 2, 3]), None, None)
+        with raises_w(space, TypeError):
+            PyUnicode_AsEncodedString(space, space.wrap(''), None, None)
         ascii = rffi.str2charp('ascii')
         replace = rffi.str2charp('replace')
-        encoded = api.PyUnicode_AsEncodedString(space.wrap(u'sp&#65533;m'),
+        encoded = PyUnicode_AsEncodedString(space, space.wrap(u'sp&#65533;m'),
                                                 ascii, replace)
         assert space.unwrap(encoded) == 'sp?m'
         rffi.free_charp(utf_8)
@@ -189,38 +193,38 @@
         rffi.free_charp(ascii)
 
         buf = rffi.unicode2wcharp(u"12345")
-        api.PyUnicode_AsWideChar(space.wrap(u'longword'), buf, 5)
+        PyUnicode_AsWideChar(space, space.wrap(u'longword'), buf, 5)
         assert rffi.wcharp2unicode(buf) == 'longw'
-        api.PyUnicode_AsWideChar(space.wrap(u'a'), buf, 5)
+        PyUnicode_AsWideChar(space, space.wrap(u'a'), buf, 5)
         assert rffi.wcharp2unicode(buf) == 'a'
         rffi.free_wcharp(buf)
 
-    def test_fromstring(self, space, api):
+    def test_fromstring(self, space):
         s = rffi.str2charp(u'sp\x09m'.encode("utf-8"))
-        w_res = api.PyUnicode_FromString(s)
+        w_res = PyUnicode_FromString(space, s)
         assert space.unwrap(w_res) == u'sp\x09m'
 
-        res = api.PyUnicode_FromStringAndSize(s, 4)
+        res = PyUnicode_FromStringAndSize(space, s, 4)
         w_res = from_ref(space, res)
-        api.Py_DecRef(res)
+        Py_DecRef(space, res)
         assert space.unwrap(w_res) == u'sp\x09m'
         rffi.free_charp(s)
 
-    def test_unicode_resize(self, space, api):
+    def test_unicode_resize(self, space):
         py_uni = new_empty_unicode(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         py_uni.c_str[0] = u'a'
         py_uni.c_str[1] = u'b'
         py_uni.c_str[2] = u'c'
         ar[0] = rffi.cast(PyObject, py_uni)
-        api.PyUnicode_Resize(ar, 3)
+        PyUnicode_Resize(space, ar, 3)
         py_uni = rffi.cast(PyUnicodeObject, ar[0])
         assert py_uni.c_length == 3
         assert py_uni.c_str[1] == u'b'
         assert py_uni.c_str[3] == u'\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_uni)
-        api.PyUnicode_Resize(ar, 10)
+        PyUnicode_Resize(space, ar, 10)
         py_uni = rffi.cast(PyUnicodeObject, ar[0])
         assert py_uni.c_length == 10
         assert py_uni.c_str[1] == 'b'
@@ -228,47 +232,46 @@
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
-    def test_AsUTF8String(self, space, api):
+    def test_AsUTF8String(self, space):
         w_u = space.wrap(u'sp\x09m')
-        w_res = api.PyUnicode_AsUTF8String(w_u)
+        w_res = PyUnicode_AsUTF8String(space, w_u)
         assert space.type(w_res) is space.w_str
         assert space.unwrap(w_res) == 'sp\tm'
 
-    def test_decode_utf8(self, space, api):
+    def test_decode_utf8(self, space):
         u = rffi.str2charp(u'sp\x134m'.encode("utf-8"))
-        w_u = api.PyUnicode_DecodeUTF8(u, 5, None)
+        w_u = PyUnicode_DecodeUTF8(space, u, 5, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == u'sp\x134m'
 
-        w_u = api.PyUnicode_DecodeUTF8(u, 2, None)
+        w_u = PyUnicode_DecodeUTF8(space, u, 2, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == 'sp'
         rffi.free_charp(u)
 
-    def test_encode_utf8(self, space, api):
+    def test_encode_utf8(self, space):
         u = rffi.unicode2wcharp(u'sp\x09m')
-        w_s = api.PyUnicode_EncodeUTF8(u, 4, None)
+        w_s = PyUnicode_EncodeUTF8(space, u, 4, None)
         assert space.unwrap(w_s) == u'sp\x09m'.encode('utf-8')
         rffi.free_wcharp(u)
 
-    def test_encode_decimal(self, space, api):
+    def test_encode_decimal(self, space):
         with rffi.scoped_unicode2wcharp(u' (12, 35 ABC)') as u:
             with rffi.scoped_alloc_buffer(20) as buf:
-                res = api.PyUnicode_EncodeDecimal(u, 13, buf.raw, None)
+                res = PyUnicode_EncodeDecimal(space, u, 13, buf.raw, None)
                 s = rffi.charp2str(buf.raw)
         assert res == 0
         assert s == ' (12, 35 ABC)'
 
         with rffi.scoped_unicode2wcharp(u' (12, \u1234\u1235)') as u:
             with rffi.scoped_alloc_buffer(20) as buf:
-                res = api.PyUnicode_EncodeDecimal(u, 9, buf.raw, None)
-        assert res == -1
-        api.PyErr_Clear()
+                with pytest.raises(OperationError):
+                    PyUnicode_EncodeDecimal(space, u, 9, buf.raw, None)
 
         with rffi.scoped_unicode2wcharp(u' (12, \u1234\u1235)') as u:
             with rffi.scoped_alloc_buffer(20) as buf:
                 with rffi.scoped_str2charp("replace") as errors:
-                    res = api.PyUnicode_EncodeDecimal(u, 9, buf.raw,
+                    res = PyUnicode_EncodeDecimal(space, u, 9, buf.raw,
                                                       errors)
                 s = rffi.charp2str(buf.raw)
         assert res == 0
@@ -277,140 +280,140 @@
         with rffi.scoped_unicode2wcharp(u'12\u1234') as u:
             with rffi.scoped_alloc_buffer(20) as buf:
                 with rffi.scoped_str2charp("xmlcharrefreplace") as errors:
-                    res = api.PyUnicode_EncodeDecimal(u, 3, buf.raw,
+                    res = PyUnicode_EncodeDecimal(space, u, 3, buf.raw,
                                                       errors)
                 s = rffi.charp2str(buf.raw)
         assert res == 0
         assert s == "12&#4660;"
 
 
-    def test_IS(self, space, api):
+    def test_IS(self, space):
         for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f,
                      0x20, 0x85, 0xa0, 0x1680, 0x2000, 0x2001, 0x2002,
                      0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008,
                      0x2009, 0x200a,
                      #0x200b is in Other_Default_Ignorable_Code_Point in 4.1.0
                      0x2028, 0x2029, 0x202f, 0x205f, 0x3000]:
-            assert api.Py_UNICODE_ISSPACE(unichr(char))
-        assert not api.Py_UNICODE_ISSPACE(u'a')
+            assert Py_UNICODE_ISSPACE(space, unichr(char))
+        assert not Py_UNICODE_ISSPACE(space, u'a')
 
-        assert api.Py_UNICODE_ISALPHA(u'a')
-        assert not api.Py_UNICODE_ISALPHA(u'0')
-        assert api.Py_UNICODE_ISALNUM(u'a')
-        assert api.Py_UNICODE_ISALNUM(u'0')
-        assert not api.Py_UNICODE_ISALNUM(u'+')
+        assert Py_UNICODE_ISALPHA(space, u'a')
+        assert not Py_UNICODE_ISALPHA(space, u'0')
+        assert Py_UNICODE_ISALNUM(space, u'a')
+        assert Py_UNICODE_ISALNUM(space, u'0')
+        assert not Py_UNICODE_ISALNUM(space, u'+')
 
-        assert api.Py_UNICODE_ISDECIMAL(u'\u0660')
-        assert not api.Py_UNICODE_ISDECIMAL(u'a')
-        assert api.Py_UNICODE_ISDIGIT(u'9')
-        assert not api.Py_UNICODE_ISDIGIT(u'@')
-        assert api.Py_UNICODE_ISNUMERIC(u'9')
-        assert not api.Py_UNICODE_ISNUMERIC(u'@')
+        assert Py_UNICODE_ISDECIMAL(space, u'\u0660')
+        assert not Py_UNICODE_ISDECIMAL(space, u'a')
+        assert Py_UNICODE_ISDIGIT(space, u'9')
+        assert not Py_UNICODE_ISDIGIT(space, u'@')
+        assert Py_UNICODE_ISNUMERIC(space, u'9')
+        assert not Py_UNICODE_ISNUMERIC(space, u'@')
 
         for char in [0x0a, 0x0d, 0x1c, 0x1d, 0x1e, 0x85, 0x2028, 0x2029]:
-            assert api.Py_UNICODE_ISLINEBREAK(unichr(char))
+            assert Py_UNICODE_ISLINEBREAK(space, unichr(char))
 
-        assert api.Py_UNICODE_ISLOWER(u'\xdf') # sharp s
-        assert api.Py_UNICODE_ISUPPER(u'\xde') # capital thorn
-        assert api.Py_UNICODE_ISLOWER(u'a')
-        assert not api.Py_UNICODE_ISUPPER(u'a')
-        assert not api.Py_UNICODE_ISTITLE(u'\xce')
-        assert api.Py_UNICODE_ISTITLE(
+        assert Py_UNICODE_ISLOWER(space, u'\xdf') # sharp s
+        assert Py_UNICODE_ISUPPER(space, u'\xde') # capital thorn
+        assert Py_UNICODE_ISLOWER(space, u'a')
+        assert not Py_UNICODE_ISUPPER(space, u'a')
+        assert not Py_UNICODE_ISTITLE(space, u'\xce')
+        assert Py_UNICODE_ISTITLE(space,
             u'\N{LATIN CAPITAL LETTER L WITH SMALL LETTER J}')
 
-    def test_TOLOWER(self, space, api):
-        assert api.Py_UNICODE_TOLOWER(u'&#65533;') == u'&#65533;'
-        assert api.Py_UNICODE_TOLOWER(u'&#65533;') == u'&#65533;'
+    def test_TOLOWER(self, space):
+        assert Py_UNICODE_TOLOWER(space, u'&#65533;') == u'&#65533;'
+        assert Py_UNICODE_TOLOWER(space, u'&#65533;') == u'&#65533;'
 
-    def test_TOUPPER(self, space, api):
-        assert api.Py_UNICODE_TOUPPER(u'&#65533;') == u'&#65533;'
-        assert api.Py_UNICODE_TOUPPER(u'&#65533;') == u'&#65533;'
+    def test_TOUPPER(self, space):
+        assert Py_UNICODE_TOUPPER(space, u'&#65533;') == u'&#65533;'
+        assert Py_UNICODE_TOUPPER(space, u'&#65533;') == u'&#65533;'
 
-    def test_TOTITLE(self, space, api):
-        assert api.Py_UNICODE_TOTITLE(u'/') == u'/'
-        assert api.Py_UNICODE_TOTITLE(u'&#65533;') == u'&#65533;'
-        assert api.Py_UNICODE_TOTITLE(u'&#65533;') == u'&#65533;'
+    def test_TOTITLE(self, space):
+        assert Py_UNICODE_TOTITLE(space, u'/') == u'/'
+        assert Py_UNICODE_TOTITLE(space, u'&#65533;') == u'&#65533;'
+        assert Py_UNICODE_TOTITLE(space, u'&#65533;') == u'&#65533;'
 
-    def test_TODECIMAL(self, space, api):
-        assert api.Py_UNICODE_TODECIMAL(u'6') == 6
-        assert api.Py_UNICODE_TODECIMAL(u'A') == -1
+    def test_TODECIMAL(self, space):
+        assert Py_UNICODE_TODECIMAL(space, u'6') == 6
+        assert Py_UNICODE_TODECIMAL(space, u'A') == -1
 
-    def test_TODIGIT(self, space, api):
-        assert api.Py_UNICODE_TODIGIT(u'6') == 6
-        assert api.Py_UNICODE_TODIGIT(u'A') == -1
+    def test_TODIGIT(self, space):
+        assert Py_UNICODE_TODIGIT(space, u'6') == 6
+        assert Py_UNICODE_TODIGIT(space, u'A') == -1
 
-    def test_TONUMERIC(self, space, api):
-        assert api.Py_UNICODE_TONUMERIC(u'6') == 6.0
-        assert api.Py_UNICODE_TONUMERIC(u'A') == -1.0
-        assert api.Py_UNICODE_TONUMERIC(u'\N{VULGAR FRACTION ONE HALF}') == .5
+    def test_TONUMERIC(self, space):
+        assert Py_UNICODE_TONUMERIC(space, u'6') == 6.0
+        assert Py_UNICODE_TONUMERIC(space, u'A') == -1.0
+        assert Py_UNICODE_TONUMERIC(space, u'\N{VULGAR FRACTION ONE HALF}') == .5
 
-    def test_fromobject(self, space, api):
+    def test_fromobject(self, space):
         w_u = space.wrap(u'a')
-        assert api.PyUnicode_FromObject(w_u) is w_u
+        assert PyUnicode_FromObject(space, w_u) is w_u
         assert space.unwrap(
-            api.PyUnicode_FromObject(space.wrap('test'))) == 'test'
+            PyUnicode_FromObject(space, space.wrap('test'))) == 'test'
 
-    def test_decode(self, space, api):
+    def test_decode(self, space):
         b_text = rffi.str2charp('caf\x82xx')
         b_encoding = rffi.str2charp('cp437')
         assert space.unwrap(
-            api.PyUnicode_Decode(b_text, 4, b_encoding, None)) == u'caf\xe9'
+            PyUnicode_Decode(space, b_text, 4, b_encoding, None)) == u'caf\xe9'
 
-        w_text = api.PyUnicode_FromEncodedObject(space.wrap("test"), b_encoding, None)
+        w_text = PyUnicode_FromEncodedObject(space, space.wrap("test"), b_encoding, None)
         assert space.isinstance_w(w_text, space.w_unicode)
         assert space.unwrap(w_text) == "test"
 
-        assert api.PyUnicode_FromEncodedObject(space.wrap(u"test"), b_encoding, None) is None
-        assert api.PyErr_Occurred() is space.w_TypeError
-        assert api.PyUnicode_FromEncodedObject(space.wrap(1), b_encoding, None) is None
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyUnicode_FromEncodedObject(space, space.wrap(u"test"),
+                                        b_encoding, None)
+        with raises_w(space, TypeError):
+            PyUnicode_FromEncodedObject(space, space.wrap(1), b_encoding, None)
 
         rffi.free_charp(b_text)
         rffi.free_charp(b_encoding)
 
-    def test_decode_null_encoding(self, space, api):
+    def test_decode_null_encoding(self, space):
         null_charp = lltype.nullptr(rffi.CCHARP.TO)
         u_text = u'abcdefg'
-        s_text = space.str_w(api.PyUnicode_AsEncodedString(space.wrap(u_text), null_charp, null_charp))
+        s_text = space.str_w(PyUnicode_AsEncodedString(space, space.wrap(u_text), null_charp, null_charp))
         b_text = rffi.str2charp(s_text)
-        assert space.unwrap(api.PyUnicode_Decode(b_text, len(s_text), null_charp, null_charp)) == u_text
-        self.raises(space, api, TypeError, api.PyUnicode_FromEncodedObject, space.wrap(u_text), null_charp, None)
+        assert space.unwrap(PyUnicode_Decode(space, b_text, len(s_text), null_charp, null_charp)) == u_text
+        with raises_w(space, TypeError):
+            PyUnicode_FromEncodedObject(
+                space, space.wrap(u_text), null_charp, None)
         rffi.free_charp(b_text)
 
-    def test_mbcs(self, space, api):
+    def test_mbcs(self, space):
         if sys.platform != 'win32':
             py.test.skip("mcbs encoding only exists on Windows")
         # unfortunately, mbcs is locale-dependent.
         # This tests works at least on a Western Windows.
         unichars = u"abc" + unichr(12345)
         wbuf = rffi.unicode2wcharp(unichars)
-        w_str = api.PyUnicode_EncodeMBCS(wbuf, 4, None)
+        w_str = PyUnicode_EncodeMBCS(space, wbuf, 4, None)
         rffi.free_wcharp(wbuf)
         assert space.type(w_str) is space.w_str
         assert space.str_w(w_str) == "abc?"
 
-    def test_escape(self, space, api):
+    def test_escape(self, space):
         def test(ustr):
             w_ustr = space.wrap(ustr.decode('Unicode-Escape'))
-            result = api.PyUnicode_AsUnicodeEscapeString(w_ustr)
+            result = PyUnicode_AsUnicodeEscapeString(space, w_ustr)
             assert space.eq_w(space.wrap(ustr), result)
 
         test('\\u674f\\u7f8e')
         test('\\u0105\\u0107\\u017c\\u017a')
         test('El Ni\\xf1o')
 
-    def test_ascii(self, space, api):
+    def test_ascii(self, space):
         ustr = "abcdef"
         w_ustr = space.wrap(ustr.decode("ascii"))
-        result = api.PyUnicode_AsASCIIString(w_ustr)
+        result = PyUnicode_AsASCIIString(space, w_ustr)
+        assert space.eq_w(space.wrap(ustr), result)
+        with raises_w(space, UnicodeEncodeError):
+            PyUnicode_AsASCIIString(space, space.wrap(u"abcd\xe9f"))
 
-        assert space.eq_w(space.wrap(ustr), result)
-
-        w_ustr = space.wrap(u"abcd\xe9f")
-        self.raises(space, api, UnicodeEncodeError, api.PyUnicode_AsASCIIString, w_ustr)
-
-    def test_decode_utf16(self, space, api):
+    def test_decode_utf16(self, space):
         def test(encoded, endian, realendian=None):
             encoded_charp = rffi.str2charp(encoded)
             strict_charp = rffi.str2charp("strict")
@@ -426,7 +429,7 @@
             else:
                 pendian = None
 
-            w_ustr = api.PyUnicode_DecodeUTF16(encoded_charp, len(encoded), strict_charp, pendian)
+            w_ustr = PyUnicode_DecodeUTF16(space, encoded_charp, len(encoded), strict_charp, pendian)
             assert space.eq_w(space.call_method(w_ustr, 'encode', space.wrap('ascii')),
                               space.wrap("abcd"))
 
@@ -446,7 +449,7 @@
         test("\xFE\xFF\x00\x61\x00\x62\x00\x63\x00\x64", 0, 1)
         test("\xFF\xFE\x61\x00\x62\x00\x63\x00\x64\x00", 0, -1)
 
-    def test_decode_utf32(self, space, api):
+    def test_decode_utf32(self, space):
         def test(encoded, endian, realendian=None):
             encoded_charp = rffi.str2charp(encoded)
             strict_charp = rffi.str2charp("strict")
@@ -462,7 +465,8 @@
             else:
                 pendian = None
 
-            w_ustr = api.PyUnicode_DecodeUTF32(encoded_charp, len(encoded), strict_charp, pendian)
+            w_ustr = PyUnicode_DecodeUTF32(space, encoded_charp, len(encoded),
+                                           strict_charp, pendian)
             assert space.eq_w(space.call_method(w_ustr, 'encode', space.wrap('ascii')),
                               space.wrap("ab"))
 
@@ -485,148 +489,150 @@
         test("\x00\x00\xFE\xFF\x00\x00\x00\x61\x00\x00\x00\x62", 0, 1)
         test("\xFF\xFE\x00\x00\x61\x00\x00\x00\x62\x00\x00\x00", 0, -1)
 
-    def test_compare(self, space, api):
-        assert api.PyUnicode_Compare(space.wrap('a'), space.wrap('b')) == -1
+    def test_compare(self, space):
+        assert PyUnicode_Compare(space, space.wrap('a'), space.wrap('b')) == -1
 
-    def test_concat(self, space, api):
-        w_res = api.PyUnicode_Concat(space.wrap(u'a'), space.wrap(u'b'))
+    def test_concat(self, space):
+        w_res = PyUnicode_Concat(space, space.wrap(u'a'), space.wrap(u'b'))
         assert space.unwrap(w_res) == u'ab'
 
-    def test_copy(self, space, api):
+    def test_copy(self, space):
         w_x = space.wrap(u"abcd\u0660")
         count1 = space.int_w(space.len(w_x))
         target_chunk = lltype.malloc(rffi.CWCHARP.TO, count1, flavor='raw')
 
-        x_chunk = api.PyUnicode_AS_UNICODE(w_x)
-        api.Py_UNICODE_COPY(target_chunk, x_chunk, 4)
+        x_chunk = PyUnicode_AS_UNICODE(space, w_x)
+        Py_UNICODE_COPY(space, target_chunk, x_chunk, 4)
         w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, 4))
 
         assert space.eq_w(w_y, space.wrap(u"abcd"))
 
-        size = api.PyUnicode_GET_SIZE(w_x)
-        api.Py_UNICODE_COPY(target_chunk, x_chunk, size)
+        size = PyUnicode_GET_SIZE(space, w_x)
+        Py_UNICODE_COPY(space, target_chunk, x_chunk, size)
         w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, size))
 
         assert space.eq_w(w_y, w_x)
 
         lltype.free(target_chunk, flavor='raw')
 
-    def test_ascii_codec(self, space, api):
+    def test_ascii_codec(self, space):
         s = 'abcdefg'
         data = rffi.str2charp(s)
-        w_u = api.PyUnicode_DecodeASCII(data, len(s), lltype.nullptr(rffi.CCHARP.TO))
+        NULL = lltype.nullptr(rffi.CCHARP.TO)
+        w_u = PyUnicode_DecodeASCII(space, data, len(s), NULL)
         assert space.eq_w(w_u, space.wrap(u"abcdefg"))
         rffi.free_charp(data)
 
         s = 'abcd\xFF'
         data = rffi.str2charp(s)
-        self.raises(space, api, UnicodeDecodeError, api.PyUnicode_DecodeASCII,
-                    data, len(s), lltype.nullptr(rffi.CCHARP.TO))
+        with raises_w(space, UnicodeDecodeError):
+            PyUnicode_DecodeASCII(space, data, len(s), NULL)
         rffi.free_charp(data)
 
         uni = u'abcdefg'
         data = rffi.unicode2wcharp(uni)
-        w_s = api.PyUnicode_EncodeASCII(data, len(uni), lltype.nullptr(rffi.CCHARP.TO))
+        w_s = PyUnicode_EncodeASCII(space, data, len(uni), NULL)
         assert space.eq_w(space.wrap("abcdefg"), w_s)
         rffi.free_wcharp(data)
 
         u = u'&#65533;bcd&#65533;fg'
         data = rffi.unicode2wcharp(u)
-        w_s = api.PyUnicode_EncodeASCII(data, len(u), lltype.nullptr(rffi.CCHARP.TO))
-        self.raises(space, api, UnicodeEncodeError, api.PyUnicode_EncodeASCII,
-                    data, len(u), lltype.nullptr(rffi.CCHARP.TO))
+        with raises_w(space, UnicodeEncodeError):
+            PyUnicode_EncodeASCII(space, data, len(u), NULL)
         rffi.free_wcharp(data)
 
-    def test_latin1(self, space, api):
+    def test_latin1(self, space):
         s = 'abcdefg'
         data = rffi.str2charp(s)
-        w_u = api.PyUnicode_DecodeLatin1(data, len(s), lltype.nullptr(rffi.CCHARP.TO))
+        w_u = PyUnicode_DecodeLatin1(space, data, len(s),
+                                     lltype.nullptr(rffi.CCHARP.TO))
         assert space.eq_w(w_u, space.wrap(u"abcdefg"))
         rffi.free_charp(data)
 
         uni = u'abcdefg'
         data = rffi.unicode2wcharp(uni)
-        w_s = api.PyUnicode_EncodeLatin1(data, len(uni), lltype.nullptr(rffi.CCHARP.TO))
+        w_s = PyUnicode_EncodeLatin1(space, data, len(uni),
+                                     lltype.nullptr(rffi.CCHARP.TO))
         assert space.eq_w(space.wrap("abcdefg"), w_s)
         rffi.free_wcharp(data)
 
         ustr = "abcdef"
         w_ustr = space.wrap(ustr.decode("ascii"))
-        result = api.PyUnicode_AsLatin1String(w_ustr)
+        result = PyUnicode_AsLatin1String(space, w_ustr)
         assert space.eq_w(space.wrap(ustr), result)
 
-    def test_format(self, space, api):
+    def test_format(self, space):
         w_format = space.wrap(u'hi %s')
         w_args = space.wrap((u'test',))
-        w_formated = api.PyUnicode_Format(w_format, w_args)
+        w_formated = PyUnicode_Format(space, w_format, w_args)
         assert space.unwrap(w_formated) == space.unwrap(space.mod(w_format, w_args))
 
-    def test_join(self, space, api):
+    def test_join(self, space):
         w_sep = space.wrap(u'<sep>')
         w_seq = space.wrap([u'a', u'b'])
-        w_joined = api.PyUnicode_Join(w_sep, w_seq)
+        w_joined = PyUnicode_Join(space, w_sep, w_seq)
         assert space.unwrap(w_joined) == u'a<sep>b'
 
-    def test_fromordinal(self, space, api):
-        w_char = api.PyUnicode_FromOrdinal(65)
+    def test_fromordinal(self, space):
+        w_char = PyUnicode_FromOrdinal(space, 65)
         assert space.unwrap(w_char) == u'A'
-        w_char = api.PyUnicode_FromOrdinal(0)
+        w_char = PyUnicode_FromOrdinal(space, 0)
         assert space.unwrap(w_char) == u'\0'
-        w_char = api.PyUnicode_FromOrdinal(0xFFFF)
+        w_char = PyUnicode_FromOrdinal(space, 0xFFFF)
         assert space.unwrap(w_char) == u'\uFFFF'
 
-    def test_replace(self, space, api):
+    def test_replace(self, space):
         w_str = space.wrap(u"abababab")
         w_substr = space.wrap(u"a")
         w_replstr = space.wrap(u"z")
         assert u"zbzbabab" == space.unwrap(
-            api.PyUnicode_Replace(w_str, w_substr, w_replstr, 2))
+            PyUnicode_Replace(space, w_str, w_substr, w_replstr, 2))
         assert u"zbzbzbzb" == space.unwrap(
-            api.PyUnicode_Replace(w_str, w_substr, w_replstr, -1))
+            PyUnicode_Replace(space, w_str, w_substr, w_replstr, -1))
 
-    def test_tailmatch(self, space, api):
+    def test_tailmatch(self, space):
         w_str = space.wrap(u"abcdef")
         # prefix match
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 9, -1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cdf"), 2, 6, -1) == 0
+        assert PyUnicode_Tailmatch(space, w_str, space.wrap("cde"), 2, 9, -1) == 1
+        assert PyUnicode_Tailmatch(space, w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
+        assert PyUnicode_Tailmatch(space, w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
+        assert PyUnicode_Tailmatch(space, w_str, space.wrap("cdf"), 2, 6, -1) == 0
         # suffix match
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5,  1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("bde"), 1, 5,  1) == 0
+        assert PyUnicode_Tailmatch(space, w_str, space.wrap("cde"), 1, 5,  1) == 1
+        assert PyUnicode_Tailmatch(space, w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
+        assert PyUnicode_Tailmatch(space, w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
+        assert PyUnicode_Tailmatch(space, w_str, space.wrap("bde"), 1, 5,  1) == 0
         # type checks
-        self.raises(space, api, TypeError,
-                    api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
-        self.raises(space, api, TypeError,
-                    api.PyUnicode_Tailmatch, space.wrap(3), space.wrap("abc"),
-                    2, 10, 1)
+        with raises_w(space, TypeError):
+            PyUnicode_Tailmatch(space, w_str, space.wrap(3), 2, 10, 1)
+        with raises_w(space, TypeError):
+            PyUnicode_Tailmatch(
+                space, space.wrap(3), space.wrap("abc"), 2, 10, 1)
 
-    def test_count(self, space, api):
+    def test_count(self, space):
         w_str = space.wrap(u"abcabdab")
-        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, -1) == 2
-        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, 2) == 1
-        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), -5, 30) == 2
+        assert PyUnicode_Count(space, w_str, space.wrap(u"ab"), 0, -1) == 2
+        assert PyUnicode_Count(space, w_str, space.wrap(u"ab"), 0, 2) == 1
+        assert PyUnicode_Count(space, w_str, space.wrap(u"ab"), -5, 30) == 2
 
-    def test_find(self, space, api):
+    def test_find(self, space):
         w_str = space.wrap(u"abcabcd")
-        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, 1) == 2
-        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, 1) == 5
-        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, -1) == 5
-        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, -1) == 5
-        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 4, -1) == 2
-        assert api.PyUnicode_Find(w_str, space.wrap(u"z"), 0, 4, -1) == -1
+        assert PyUnicode_Find(space, w_str, space.wrap(u"c"), 0, 7, 1) == 2
+        assert PyUnicode_Find(space, w_str, space.wrap(u"c"), 3, 7, 1) == 5
+        assert PyUnicode_Find(space, w_str, space.wrap(u"c"), 0, 7, -1) == 5
+        assert PyUnicode_Find(space, w_str, space.wrap(u"c"), 3, 7, -1) == 5
+        assert PyUnicode_Find(space, w_str, space.wrap(u"c"), 0, 4, -1) == 2
+        assert PyUnicode_Find(space, w_str, space.wrap(u"z"), 0, 4, -1) == -1
 
-    def test_split(self, space, api):
+    def test_split(self, space):
         w_str = space.wrap(u"a\nb\nc\nd")
         assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
-                api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
+                PyUnicode_Split(space, w_str, space.wrap('\n'), -1)))
         assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
-                api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+                PyUnicode_Split(space, w_str, space.wrap('\n'), 2)))
         assert r"[u'a', u'b', u'c d']" == space.unwrap(space.repr(
-                api.PyUnicode_Split(space.wrap(u'a\nb  c d'), None, 2)))
+                PyUnicode_Split(space, space.wrap(u'a\nb  c d'), None, 2)))
         assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
-                api.PyUnicode_Splitlines(w_str, 0)))
+                PyUnicode_Splitlines(space, w_str, 0)))
         assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
-                api.PyUnicode_Splitlines(w_str, 1)))
+                PyUnicode_Splitlines(space, w_str, 1)))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -487,6 +487,7 @@
         if not PyUnicode_Check(space, w_unicode):
             PyErr_BadArgument(space)
         return unicodeobject.encode_object(space, w_unicode, encoding, "strict")
+    globals()['PyUnicode_As%sString' % suffix] = PyUnicode_AsXXXString
 
     @cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING], PyObject)
     @func_renamer('PyUnicode_Decode%s' % suffix)

From pypy.commits at gmail.com  Thu Jan 12 08:53:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 05:53:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: DeprecationWarning when opening in 'U'
 mode
Message-ID: <58778a71.c745c20a.7f31b.eb62@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89505:623575878821
Date: 2017-01-12 14:53 +0100
http://bitbucket.org/pypy/pypy/changeset/623575878821/

Log:	DeprecationWarning when opening in 'U' mode

diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -67,8 +67,13 @@
     if updating:
         rawmode += "+"
 
-    if universal and (writing or appending):
-        raise oefmt(space.w_ValueError, "can't use U and writing mode at once")
+    if universal:
+        if writing or appending:
+            raise oefmt(space.w_ValueError,
+                        "can't use U and writing mode at once")
+        space.warn(space.wrap("'U' mode is deprecated ('r' has the same "
+                              "effect in Python 3.x)"),
+                   space.w_DeprecationWarning)
     if text and binary:
         raise oefmt(space.w_ValueError,
                     "can't have text and binary mode at once")
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -225,17 +225,21 @@
 
     def test_attributes(self):
         import _io
+        import warnings
 
         with _io.open(self.tmpfile, "wb", buffering=0) as f:
             assert f.mode == "wb"
 
-        with _io.open(self.tmpfile, "U") as f:
-            assert f.name == self.tmpfile
-            assert f.buffer.name == self.tmpfile
-            assert f.buffer.raw.name == self.tmpfile
-            assert f.mode == "U"
-            assert f.buffer.mode == "rb"
-            assert f.buffer.raw.mode == "rb"
+        with warnings.catch_warnings(record=True) as l:
+            warnings.simplefilter("always")
+            with _io.open(self.tmpfile, "U") as f:
+                assert f.name == self.tmpfile
+                assert f.buffer.name == self.tmpfile
+                assert f.buffer.raw.name == self.tmpfile
+                assert f.mode == "U"
+                assert f.buffer.mode == "rb"
+                assert f.buffer.raw.mode == "rb"
+        assert isinstance(l[0].message, DeprecationWarning)
 
         with _io.open(self.tmpfile, "w+") as f:
             assert f.mode == "w+"

From pypy.commits at gmail.com  Thu Jan 12 09:20:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 06:20:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Detect null bytes in unicode or byte
 strings for the OS functions
Message-ID: <587790a5.e4361c0a.e7624.2dab@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89506:6e3116ffd73c
Date: 2017-01-12 15:17 +0100
http://bitbucket.org/pypy/pypy/changeset/6e3116ffd73c/

Log:	Detect null bytes in unicode or byte strings for the OS functions

diff --git a/pypy/interpreter/test/test_fsencode.py b/pypy/interpreter/test/test_fsencode.py
--- a/pypy/interpreter/test/test_fsencode.py
+++ b/pypy/interpreter/test/test_fsencode.py
@@ -78,3 +78,8 @@
 
             assert space.fsencode_w(w_enc) == space.bytes_w(w_enc)
             assert space.eq_w(space.wrap_fsdecoded(space.bytes_w(w_enc)), w_st2)
+
+    def test_null_byte(self):
+        space = self.space
+        w_u = space.newunicode(u'abc\x00def')
+        space.raises_w(space.w_ValueError, space.fsencode, w_u)
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1,5 +1,5 @@
 import sys
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import runicode
 from pypy.module._codecs import interp_codecs
@@ -104,6 +104,8 @@
         from pypy.module._codecs.locale import (
             unicode_encode_locale_surrogateescape)
         uni = space.unicode_w(w_uni)
+        if u'\x00' in uni:
+            raise oefmt(space.w_ValueError, "embedded null character")
         bytes = unicode_encode_locale_surrogateescape(
             uni, errorhandler=encode_error_handler(space))
     else:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -148,11 +148,15 @@
     try:
         path_b = space.fsencode_w(w_value)
         return Path(-1, path_b, None, w_value)
-    except OperationError:
+    except OperationError as e:
+        if not e.match(space, space.w_TypeError):
+            raise
         if allow_fd:
             fd = unwrap_fd(space, w_value, "string, bytes or integer")
             return Path(fd, None, None, w_value)
-    raise oefmt(space.w_TypeError, "illegal type for path parameter")
+    raise oefmt(space.w_TypeError,
+                "illegal type for path parameter (expected "
+                "string or bytes, got %T)", w_value)
 
 class _PathOrFd(Unwrapper):
     def unwrap(self, space, w_value):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -202,6 +202,8 @@
         excinfo = raises(TypeError, self.posix.stat, 2.)
         assert "should be string, bytes or integer, not float" in str(excinfo.value)
         raises(ValueError, self.posix.stat, -1)
+        raises(ValueError, self.posix.stat, b"abc\x00def")
+        raises(ValueError, self.posix.stat, u"abc\x00def")
 
     if hasattr(__import__(os.name), "statvfs"):
         def test_statvfs(self):

From pypy.commits at gmail.com  Thu Jan 12 09:35:11 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 12 Jan 2017 06:35:11 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Expand pseudo-header
Message-ID: <5877941f.8c7e1c0a.b09b3.323e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89507:02d7b8efbcaa
Date: 2016-12-18 01:38 +0000
http://bitbucket.org/pypy/pypy/changeset/02d7b8efbcaa/

Log:	Expand pseudo-header

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -674,6 +674,51 @@
 typedef struct _typeobject PyTypeObject;
 
 typedef void (*freefunc)(void *);
+typedef void (*destructor)(PyObject *);
+typedef int (*printfunc)(PyObject *, FILE *, int);
+typedef PyObject *(*getattrfunc)(PyObject *, char *);
+typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
+typedef int (*setattrfunc)(PyObject *, char *, PyObject *);
+typedef int (*setattrofunc)(PyObject *, PyObject *, PyObject *);
+typedef int (*cmpfunc)(PyObject *, PyObject *);
+typedef PyObject *(*reprfunc)(PyObject *);
+typedef long (*hashfunc)(PyObject *);
+typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
+typedef PyObject *(*getiterfunc) (PyObject *);
+typedef PyObject *(*iternextfunc) (PyObject *);
+typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);
+typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *);
+typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
+typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
+typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t);
+
+typedef PyObject * (*unaryfunc)(PyObject *);
+typedef PyObject * (*binaryfunc)(PyObject *, PyObject *);
+typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
+typedef int (*inquiry)(PyObject *);
+typedef Py_ssize_t (*lenfunc)(PyObject *);
+typedef int (*coercion)(PyObject **, PyObject **);
+typedef PyObject *(*intargfunc)(PyObject *, int);
+typedef PyObject *(*intintargfunc)(PyObject *, int, int);
+typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
+typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
+typedef int(*intobjargproc)(PyObject *, int, PyObject *);
+typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *);
+typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
+typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
+typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
+
+
+/* int-based buffer interface */
+typedef int (*getreadbufferproc)(PyObject *, int, void **);
+typedef int (*getwritebufferproc)(PyObject *, int, void **);
+typedef int (*getsegcountproc)(PyObject *, int *);
+typedef int (*getcharbufferproc)(PyObject *, int, char **);
+/* ssize_t-based buffer interface */
+typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
+typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
+typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
+typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 
 /* Py3k buffer interface, adapted for PyPy */
 #define Py_MAX_NDIMS 32

From pypy.commits at gmail.com  Thu Jan 12 09:35:15 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 12 Jan 2017 06:35:15 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Add missing declarations needed
 by PyTypeObject
Message-ID: <58779423.6a7ac20a.f5ed4.119d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89509:7b3731d591e4
Date: 2016-12-18 03:40 +0000
http://bitbucket.org/pypy/pypy/changeset/7b3731d591e4/

Log:	Add missing declarations needed by PyTypeObject

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -840,11 +840,47 @@
 	releasebufferproc bf_releasebuffer;
 } PyBufferProcs;
 
+/* from descrobject.h */
+typedef PyObject *(*getter)(PyObject *, void *);
+typedef int (*setter)(PyObject *, PyObject *, void *);
+
+typedef struct PyGetSetDef {
+	char *name;
+	getter get;
+	setter set;
+	char *doc;
+	void *closure;
+} PyGetSetDef;
+
+/* from methodobject.h */
+typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
+typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
+                                             PyObject *);
+typedef PyObject *(*PyNoArgsFunction)(PyObject *);
+
+struct PyMethodDef {
+    const char  *ml_name;   /* The name of the built-in function/method */
+    PyCFunction  ml_meth;   /* The C function that implements it */
+    int          ml_flags;  /* Combination of METH_xxx flags, which mostly
+                               describe the args expected by the C func */
+    const char  *ml_doc;    /* The __doc__ attribute, or NULL */
+};
+typedef struct PyMethodDef PyMethodDef;
+
+/* from structmember.h */
+typedef struct PyMemberDef {
+    /* Current version, use this */
+    char *name;
+    int type;
+    Py_ssize_t offset;
+    int flags;
+    char *doc;
+} PyMemberDef;
 
 
 typedef struct _typeobject {
 	PyObject_VAR_HEAD
-	const char *tp_name; /* For printing, in format "<module>.<name>" */
+	/* const */ char *tp_name; /* For printing, in format "<module>.<name>" */
 	Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
 
 	/* Methods to implement standard operations */
@@ -876,7 +912,7 @@
 	/* Flags to define presence of optional/expanded features */
 	long tp_flags;
 
-	const char *tp_doc; /* Documentation string */
+	/*const*/ char *tp_doc; /* Documentation string */
 
 	/* Assigned meaning in release 2.0 */
 	/* call function for all accessible objects */
@@ -923,7 +959,7 @@
 
 } PyTypeObject;
 
-""")
+""", configure_now=True)
 
 Py_ssize_t = object_h.gettype('Py_ssize_t')
 Py_ssize_tP = object_h.gettype('Py_ssize_t *')
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,23 +11,15 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr, slot_function)
+    PyTypeObjectPtr, slot_function, object_h)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
 PyCFunction_typedef = rffi.COpaquePtr(typedef='PyCFunction')
-PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject))
-PyCFunctionKwArgs = lltype.Ptr(lltype.FuncType([PyObject, PyObject, PyObject],
-                                               PyObject))
 
-PyMethodDef = cpython_struct(
-    'PyMethodDef',
-    [('ml_name', rffi.CONST_CCHARP),
-     ('ml_meth', PyCFunction_typedef),
-     ('ml_flags', rffi.INT_real),
-     ('ml_doc', rffi.CONST_CCHARP),
-     ])
-
+PyMethodDef = object_h.gettype('PyMethodDef')
+PyCFunction = object_h.gettype('PyCFunction')
+PyCFunctionKwArgs = object_h.gettype('PyCFunctionWithKeywords')
 PyCFunctionObjectStruct = cpython_struct(
     'PyCFunctionObject',
     PyObjectFields + (
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -50,7 +50,7 @@
     cache.  CPython includes some extra checking here to make sure the module
     being initialized lines up with what's expected, but we don't.
     """
-    from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
+    from pypy.module.cpyext.api import PyTypeObjectPtr
     modname = rffi.charp2str(name)
     state = space.fromcache(State)
     f_name, f_path = state.package_context
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,9 +7,9 @@
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
     slot_function, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    pypy_decl, Py_buffer, Py_bufferP)
+    pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr)
 from pypy.module.cpyext.typeobjectdefs import (
-    unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
+    unaryfunc, ternaryfunc, binaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -5,7 +5,7 @@
 from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject
 from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref
-from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
+from pypy.module.cpyext.api import PyTypeObjectPtr
 
 import py
 import sys
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -1,237 +1,59 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rtyper.lltypesystem.lltype import Ptr, FuncType, Void
-from pypy.module.cpyext.api import (cpython_struct, Py_ssize_t, Py_ssize_tP,
-    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP,
-    Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
-from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
-from pypy.module.cpyext.modsupport import PyMethodDef
-from pypy.module.cpyext.api import Py_bufferP, object_h
+from pypy.module.cpyext.api import object_h
 
 
-P, FT, PyO = Ptr, FuncType, PyObject
-PyOPtr = Ptr(lltype.Array(PyO, hints={'nolength': True}))
+freefunc = object_h.definitions['freefunc']
+destructor = object_h.definitions['destructor']
+printfunc = object_h.definitions['printfunc']
+getattrfunc = object_h.definitions['getattrfunc']
+getattrofunc = object_h.definitions['getattrofunc']
+setattrfunc = object_h.definitions['setattrfunc']
+setattrofunc = object_h.definitions['setattrofunc']
+cmpfunc = object_h.definitions['cmpfunc']
+reprfunc = object_h.definitions['reprfunc']
+hashfunc = object_h.definitions['hashfunc']
+richcmpfunc = object_h.definitions['richcmpfunc']
+getiterfunc = object_h.definitions['getiterfunc']
+iternextfunc = object_h.definitions['iternextfunc']
+descrgetfunc = object_h.definitions['descrgetfunc']
+descrsetfunc = object_h.definitions['descrsetfunc']
+initproc = object_h.definitions['initproc']
+newfunc = object_h.definitions['newfunc']
+allocfunc = object_h.definitions['allocfunc']
 
-#freefunc = P(FT([rffi.VOIDP], Void))
-freefunc = object_h.definitions['freefunc']
+unaryfunc = object_h.definitions['unaryfunc']
+binaryfunc = object_h.definitions['binaryfunc']
+ternaryfunc = object_h.definitions['ternaryfunc']
+inquiry = object_h.definitions['inquiry']
+lenfunc = object_h.definitions['lenfunc']
+coercion = object_h.definitions['coercion']
+intargfunc = object_h.definitions['intargfunc']
+intintargfunc = object_h.definitions['intintargfunc']
+ssizeargfunc = object_h.definitions['ssizeargfunc']
+ssizessizeargfunc = object_h.definitions['ssizessizeargfunc']
+intobjargproc = object_h.definitions['intobjargproc']
+intintobjargproc = object_h.definitions['intintobjargproc']
+ssizeobjargproc = object_h.definitions['ssizeobjargproc']
+ssizessizeobjargproc = object_h.definitions['ssizessizeobjargproc']
+objobjargproc = object_h.definitions['objobjargproc']
 
-destructor = P(FT([PyO], Void))
-printfunc = P(FT([PyO, FILEP, rffi.INT_real], rffi.INT))
-getattrfunc = P(FT([PyO, rffi.CCHARP], PyO))
-getattrofunc = P(FT([PyO, PyO], PyO))
-setattrfunc = P(FT([PyO, rffi.CCHARP, PyO], rffi.INT_real))
-setattrofunc = P(FT([PyO, PyO, PyO], rffi.INT_real))
-cmpfunc = P(FT([PyO, PyO], rffi.INT_real))
-reprfunc = P(FT([PyO], PyO))
-hashfunc = P(FT([PyO], lltype.Signed))
-richcmpfunc = P(FT([PyO, PyO, rffi.INT_real], PyO))
-getiterfunc = P(FT([PyO], PyO))
-iternextfunc = P(FT([PyO], PyO))
-descrgetfunc = P(FT([PyO, PyO, PyO], PyO))
-descrsetfunc = P(FT([PyO, PyO, PyO], rffi.INT_real))
-initproc = P(FT([PyO, PyO, PyO], rffi.INT_real))
-newfunc = P(FT([PyTypeObjectPtr, PyO, PyO], PyO))
-allocfunc = P(FT([PyTypeObjectPtr, Py_ssize_t], PyO))
+objobjproc = object_h.definitions['objobjproc']
+visitproc = object_h.definitions['visitproc']
+traverseproc = object_h.definitions['traverseproc']
 
-unaryfunc = P(FT([PyO], PyO))
-binaryfunc = P(FT([PyO, PyO], PyO))
-ternaryfunc = P(FT([PyO, PyO, PyO], PyO))
-inquiry = P(FT([PyO], rffi.INT_real))
-lenfunc = P(FT([PyO], Py_ssize_t))
-coercion = P(FT([PyOPtr, PyOPtr], rffi.INT_real))
-intargfunc = P(FT([PyO, rffi.INT_real], PyO))
-intintargfunc = P(FT([PyO, rffi.INT_real, rffi.INT], PyO))
-ssizeargfunc = P(FT([PyO, Py_ssize_t], PyO))
-ssizessizeargfunc = P(FT([PyO, Py_ssize_t, Py_ssize_t], PyO))
-intobjargproc = P(FT([PyO, rffi.INT_real, PyO], rffi.INT))
-intintobjargproc = P(FT([PyO, rffi.INT_real, rffi.INT, PyO], rffi.INT))
-ssizeobjargproc = P(FT([PyO, Py_ssize_t, PyO], rffi.INT_real))
-ssizessizeobjargproc = P(FT([PyO, Py_ssize_t, Py_ssize_t, PyO], rffi.INT_real))
-objobjargproc = P(FT([PyO, PyO, PyO], rffi.INT_real))
+getter = object_h.definitions['getter']
+setter = object_h.definitions['setter']
 
-objobjproc = P(FT([PyO, PyO], rffi.INT_real))
-visitproc = P(FT([PyO, rffi.VOIDP], rffi.INT_real))
-traverseproc = P(FT([PyO, visitproc, rffi.VOIDP], rffi.INT_real))
+readbufferproc = object_h.definitions['readbufferproc']
+writebufferproc = object_h.definitions['writebufferproc']
+segcountproc = object_h.definitions['segcountproc']
+charbufferproc = object_h.definitions['charbufferproc']
+getbufferproc = object_h.definitions['getbufferproc']
+releasebufferproc = object_h.definitions['releasebufferproc']
 
-getter = P(FT([PyO, rffi.VOIDP], PyO))
-setter = P(FT([PyO, PyO, rffi.VOIDP], rffi.INT_real))
 
-wrapperfunc = P(FT([PyO, PyO, rffi.VOIDP], PyO))
-wrapperfunc_kwds = P(FT([PyO, PyO, rffi.VOIDP, PyO], PyO))
-
-readbufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
-writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
-segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
-charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
-getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
-releasebufferproc = P(FT([PyO, Py_bufferP], Void))
-
-
-PyGetSetDef = cpython_struct("PyGetSetDef", (
-    ("name", rffi.CCHARP),
-    ("get", getter),
-    ("set", setter),
-    ("doc", rffi.CCHARP),
-    ("closure", rffi.VOIDP),
-))
-
-PyNumberMethods = cpython_struct("PyNumberMethods", (
-    ("nb_add", binaryfunc),
-    ("nb_subtract", binaryfunc),
-    ("nb_multiply", binaryfunc),
-    ("nb_divide", binaryfunc),
-    ("nb_remainder", binaryfunc),
-    ("nb_divmod", binaryfunc),
-    ("nb_power", ternaryfunc),
-    ("nb_negative", unaryfunc),
-    ("nb_positive", unaryfunc),
-    ("nb_absolute", unaryfunc),
-    ("nb_nonzero", inquiry),
-    ("nb_invert", unaryfunc),
-    ("nb_lshift", binaryfunc),
-    ("nb_rshift", binaryfunc),
-    ("nb_and", binaryfunc),
-    ("nb_xor", binaryfunc),
-    ("nb_or", binaryfunc),
-    ("nb_coerce", coercion),
-    ("nb_int", unaryfunc),
-    ("nb_long", unaryfunc),
-    ("nb_float", unaryfunc),
-    ("nb_oct", unaryfunc),
-    ("nb_hex", unaryfunc),
-    ("nb_inplace_add", binaryfunc),
-    ("nb_inplace_subtract", binaryfunc),
-    ("nb_inplace_multiply", binaryfunc),
-    ("nb_inplace_divide", binaryfunc),
-    ("nb_inplace_remainder", binaryfunc),
-    ("nb_inplace_power", ternaryfunc),
-    ("nb_inplace_lshift", binaryfunc),
-    ("nb_inplace_rshift", binaryfunc),
-    ("nb_inplace_and", binaryfunc),
-    ("nb_inplace_xor", binaryfunc),
-    ("nb_inplace_or", binaryfunc),
-
-    ("nb_floor_divide", binaryfunc),
-    ("nb_true_divide", binaryfunc),
-    ("nb_inplace_floor_divide", binaryfunc),
-    ("nb_inplace_true_divide", binaryfunc),
-
-    ("nb_index", unaryfunc),
-))
-
-PySequenceMethods = cpython_struct("PySequenceMethods", (
-    ("sq_length", lenfunc),
-    ("sq_concat", binaryfunc),
-    ("sq_repeat", ssizeargfunc),
-    ("sq_item", ssizeargfunc),
-    ("sq_slice", ssizessizeargfunc),
-    ("sq_ass_item", ssizeobjargproc),
-    ("sq_ass_slice", ssizessizeobjargproc),
-    ("sq_contains", objobjproc),
-    ("sq_inplace_concat", binaryfunc),
-    ("sq_inplace_repeat", ssizeargfunc),
-))
-
-PyMappingMethods = cpython_struct("PyMappingMethods", (
-    ("mp_length", lenfunc),
-    ("mp_subscript", binaryfunc),
-    ("mp_ass_subscript", objobjargproc),
-))
-
-PyBufferProcs = cpython_struct("PyBufferProcs", (
-    ("bf_getreadbuffer", readbufferproc),
-    ("bf_getwritebuffer", writebufferproc),
-    ("bf_getsegcount", segcountproc),
-    ("bf_getcharbuffer", charbufferproc),
-    ("bf_getbuffer", getbufferproc),
-    ("bf_releasebuffer", releasebufferproc),
-))
-
-PyMemberDef = cpython_struct("PyMemberDef", (
-    ("name", rffi.CCHARP),
-    ("type",  rffi.INT_real),
-    ("offset", Py_ssize_t),
-    ("flags", rffi.INT_real),
-    ("doc", rffi.CCHARP),
-))
-
-# These fields are supported and used in different ways
-# The following comments mean:
-#    #E    essential, initialized for all PTOs
-#    #S    supported
-#    #U    unsupported
-#    #N    not yet implemented
-PyTypeObjectFields = []
-PyTypeObjectFields.extend(PyVarObjectFields)
-PyTypeObjectFields.extend([
-    ("tp_name", rffi.CCHARP), #E For printing, in format "<module>.<name>"
-    ("tp_basicsize", Py_ssize_t), #E  For allocation
-    ("tp_itemsize", Py_ssize_t),  #E       "
-
-    # Methods to implement standard operations
-    ("tp_dealloc", destructor),   #E
-    ("tp_print", printfunc),      #U
-    ("tp_getattr", getattrfunc),  #U
-    ("tp_setattr", setattrfunc),  #U
-    ("tp_compare", cmpfunc),      #N
-    ("tp_repr", reprfunc),        #N
-
-    # Method suites for standard classes
-    ("tp_as_number", Ptr(PyNumberMethods)), #N
-    ("tp_as_sequence", Ptr(PySequenceMethods)), #N
-    ("tp_as_mapping", Ptr(PyMappingMethods)), #N
-
-    # More standard operations (here for binary compatibility)
-    ("tp_hash", hashfunc),        #N
-    ("tp_call", ternaryfunc),     #N
-    ("tp_str", reprfunc),         #N
-    ("tp_getattro", getattrofunc),#N
-    ("tp_setattro", setattrofunc),#N
-
-    # Functions to access object as input/output buffer
-    ("tp_as_buffer", Ptr(PyBufferProcs)), #U
-
-    # Flags to define presence of optional/expanded features
-    ("tp_flags", lltype.Signed),  #E
-
-    ("tp_doc", rffi.CCHARP),      #N Documentation string
-
-    # Assigned meaning in release 2.0
-    # call function for all accessible objects
-    ("tp_traverse", traverseproc),#U
-
-    # delete references to contained objects
-    ("tp_clear", inquiry),        #U
-
-    # Assigned meaning in release 2.1
-    # rich comparisons
-    ("tp_richcompare", richcmpfunc), #N
-
-    # weak reference enabler
-    ("tp_weaklistoffset", Py_ssize_t), #U
-
-    # Added in release 2.2
-    # Iterators
-    ("tp_iter", getiterfunc),       #N
-    ("tp_iternext", iternextfunc),  #N
-
-    # Attribute descriptor and subclassing stuff
-    ("tp_methods", Ptr(PyMethodDef)), #S
-    ("tp_members", Ptr(PyMemberDef)), #S
-    ("tp_getset", Ptr(PyGetSetDef)),  #S
-    ("tp_base", Ptr(PyTypeObject)),   #E
-    ("tp_dict", PyObject),            #U
-    ("tp_descr_get", descrgetfunc),   #N
-    ("tp_descr_set", descrsetfunc),   #N
-    ("tp_dictoffset", Py_ssize_t),    #U
-    ("tp_init", initproc),            #N
-    ("tp_alloc", allocfunc),          #N
-    ("tp_new", newfunc),              #S
-    ("tp_free", freefunc), #E Low-level free-memory routine
-    ("tp_is_gc", inquiry), #U For PyObject_IS_GC
-    ("tp_bases", PyObject),#E
-    ("tp_mro", PyObject),  #U method resolution order
-    ("tp_cache", PyObject),#S
-    ("tp_subclasses", PyObject), #U
-    ("tp_weaklist", PyObject),   #U
-    ("tp_del", destructor),      #N
-    ])
+PyGetSetDef = object_h.definitions['PyGetSetDef']
+PyNumberMethods = object_h.definitions['PyNumberMethods']
+PySequenceMethods = object_h.definitions['PySequenceMethods']
+PyMappingMethods = object_h.definitions['PyMappingMethods']
+PyBufferProcs = object_h.definitions['PyBufferProcs']
+PyMemberDef = object_h.definitions['PyMemberDef']

From pypy.commits at gmail.com  Thu Jan 12 09:35:17 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 12 Jan 2017 06:35:17 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: hg merge default
Message-ID: <58779425.4a0dc20a.f8fdf.c8af@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89510:421ea2f68c13
Date: 2017-01-12 14:34 +0000
http://bitbucket.org/pypy/pypy/changeset/421ea2f68c13/

Log:	hg merge default

diff too long, truncating to 2000 out of 2826 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -97,3 +97,7 @@
 
 Fix a test failure introduced by strbuf-as-buffer
 
+.. branch: cpyext-FromBuffer
+
+Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to
+the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak.
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -8,6 +8,7 @@
 
 from rpython.rlib.jit import promote
 from rpython.rlib.objectmodel import compute_identity_hash, specialize
+from rpython.rlib.objectmodel import instantiate
 from rpython.tool.sourcetools import compile2, func_with_new_name
 
 
@@ -221,10 +222,6 @@
     exec source.compile() in miniglobals
     return miniglobals['descr_typecheck_%s' % func.__name__]
 
-def unknown_objclass_getter(space):
-    # NB. this is an AttributeError to make inspect.py happy
-    raise oefmt(space.w_AttributeError, "generic property has no __objclass__")
-
 @specialize.arg(0)
 def make_objclass_getter(tag, func, cls):
     if func and hasattr(func, 'im_func'):
@@ -235,7 +232,7 @@
 @specialize.memo()
 def _make_objclass_getter(cls):
     if not cls:
-        return unknown_objclass_getter, cls
+        return None, cls
     miniglobals = {}
     if isinstance(cls, str):
         assert cls.startswith('<'), "pythontype typecheck should begin with <"
@@ -254,6 +251,8 @@
 
 class GetSetProperty(W_Root):
     _immutable_fields_ = ["fget", "fset", "fdel"]
+    name = '<generic property>'
+    w_objclass = None
 
     @specialize.arg(7)
     def __init__(self, fget, fset=None, fdel=None, doc=None,
@@ -265,15 +264,25 @@
                                             cls=cls, use_closure=use_closure)
         fdel = make_descr_typecheck_wrapper((tag, 2), fdel,
                                             cls=cls, use_closure=use_closure)
+        self._init(fget, fset, fdel, doc, cls, objclass_getter, use_closure)
+
+    def _init(self, fget, fset, fdel, doc, cls, objclass_getter, use_closure):
         self.fget = fget
         self.fset = fset
         self.fdel = fdel
         self.doc = doc
         self.reqcls = cls
-        self.name = '<generic property>'
         self.objclass_getter = objclass_getter
         self.use_closure = use_closure
 
+    def copy_for_type(self, w_objclass):
+        new = instantiate(GetSetProperty)
+        new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
+                  None, self.use_closure)
+        new.name = self.name
+        new.w_objclass = w_objclass
+        return new
+
     @unwrap_spec(w_cls = WrappedDefault(None))
     def descr_property_get(self, space, w_obj, w_cls=None):
         """property.__get__(obj[, type]) -> value
@@ -322,7 +331,14 @@
                                                space.wrap(self.name)]))
 
     def descr_get_objclass(space, property):
-        return property.objclass_getter(space)
+        if property.w_objclass is not None:
+            return property.w_objclass
+        if property.objclass_getter is not None:
+            return property.objclass_getter(space)
+        # NB. this is an AttributeError to make inspect.py happy
+        raise oefmt(space.w_AttributeError,
+                    "generic property has no __objclass__")
+
 
 def interp_attrproperty(name, cls, doc=None):
     "NOT_RPYTHON: initialization-time only"
@@ -467,7 +483,7 @@
     return lifeline.get_any_weakref(space)
 
 dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
-                            doc="dictionary for instance variables (if defined)")
+                            doc="dictionary for instance variables")
 dict_descr.name = '__dict__'
 
 
@@ -499,7 +515,7 @@
     return space.newtuple([w_docstring])
 
 weakref_descr = GetSetProperty(descr_get_weakref,
-                    doc="list of weak references to the object (if defined)")
+                               doc="list of weak references to the object")
 weakref_descr.name = '__weakref__'
 
 def make_weakref_descr(cls):
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -23,6 +23,10 @@
 def pytest_ignore_collect(path, config):
     if py.path.local.sysfind('genreflex') is None and config.option.runappdirect:
         return True          # "can't run dummy tests in -A"
+    if disabled:
+        return True
+
+disabled = None
 
 def pytest_configure(config):
     if py.path.local.sysfind('genreflex') is None:
@@ -37,7 +41,7 @@
             # build dummy backend (which has reflex info and calls hard-wired)
             import os
             from rpython.translator.tool.cbuild import ExternalCompilationInfo
-            from rpython.translator.platform import platform
+            from rpython.translator.platform import platform, CompilationError
             from rpython.translator import cdir
 
             from rpython.rtyper.lltypesystem import rffi
@@ -55,9 +59,16 @@
                 use_cpp_linker=True,
             )
 
-            soname = platform.compile(
-                [], eci,
-                outputfilename='libcppyy_dummy_backend',
-                standalone=False)
+            try:
+                soname = platform.compile(
+                    [], eci,
+                    outputfilename='libcppyy_dummy_backend',
+                    standalone=False)
+            except CompilationError as e:
+                if '-std=c++11' in str(e):
+                    global disabled
+                    disabled = str(e)
+                    return
+                raise
 
             lcapi.reflection_library = str(soname)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -123,6 +123,7 @@
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -244,14 +245,13 @@
 cpyext_namespace = NameManager('cpyext_')
 
 class ApiFunction(object):
-    def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED,
+    def __init__(self, argtypes, restype, callable, error=CANNOT_FAIL,
                  c_name=None, gil=None, result_borrowed=False, result_is_ll=False):
         self.argtypes = argtypes
         self.restype = restype
         self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
         self.callable = callable
-        if error is not _NOT_SPECIFIED:
-            self.error_value = error
+        self.error_value = error
         self.c_name = c_name
 
         # extract the signature from the (CPython-level) code object
@@ -291,7 +291,7 @@
 
         argtypesw = zip(self.argtypes,
                         [_name.startswith("w_") for _name in self.argnames])
-        error_value = getattr(self, "error_value", CANNOT_FAIL)
+        error_value = self.error_value
         if (isinstance(self.restype, lltype.Ptr)
                 and error_value is not CANNOT_FAIL):
             assert lltype.typeOf(error_value) == self.restype
@@ -429,12 +429,12 @@
     def decorate(func):
         if func.__name__ in FUNCTIONS_BY_HEADER[header]:
             raise ValueError("%s already registered" % func.__name__)
-        api_function = _create_api_func(
-            func, argtypes, restype, error, gil=gil,
+        func._always_inline_ = 'try'
+        api_function = ApiFunction(
+            argtypes, restype, func,
+            error=_compute_error(error, restype), gil=gil,
             result_borrowed=result_borrowed, result_is_ll=result_is_ll)
-        unwrapper = api_function.get_unwrapper()
-        unwrapper.func = func
-        unwrapper.api_func = api_function
+        FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
 
         # ZZZ is this whole logic really needed???  It seems to be only
         # for RPython code calling PyXxx() functions directly.  I would
@@ -462,32 +462,33 @@
             assert got_integer == expect_integer, (
                 'got %r not integer' % (res,))
             return res
+        INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
 
-        if header is not None:
-            FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
-            INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
-        return unwrapper
-    return decorate
-
-def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
-    def decorate(func):
-        c_name = func.__name__
-        api_function = _create_api_func(func, argtypes, restype, error, c_name)
         unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
         return unwrapper
     return decorate
 
+def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
+    def decorate(func):
+        func._always_inline_ = 'try'
+        api_function = ApiFunction(
+            argtypes, restype, func,
+            error=_compute_error(error, restype),
+            c_name=func.__name__)
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
 
-def _create_api_func(
-        func, argtypes, restype, error=_NOT_SPECIFIED, c_name=None,
-        gil=None, result_borrowed=False, result_is_ll=False):
+def _compute_error(error, restype):
+    """Convert error specification to actual error value of type restype."""
     if isinstance(restype, lltype.Typedef):
         real_restype = restype.OF
     else:
         real_restype = restype
-
     if error is _NOT_SPECIFIED:
         if isinstance(real_restype, lltype.Ptr):
             error = lltype.nullptr(real_restype.TO)
@@ -495,11 +496,7 @@
             error = CANNOT_FAIL
     if type(error) is int:
         error = rffi.cast(real_restype, error)
-
-    func._always_inline_ = 'try'
-    return ApiFunction(
-        argtypes, restype, func, error, c_name=c_name, gil=gil,
-        result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+    return error
 
 
 def cpython_struct(name, fields, forward=None, level=1):
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -118,12 +118,16 @@
                     """ % (type_name,)))
         except OperationError:
             return 0
+    return check, check_exact
 
-make_check_function("PyDateTime_Check", "datetime")
-make_check_function("PyDate_Check", "date")
-make_check_function("PyTime_Check", "time")
-make_check_function("PyDelta_Check", "timedelta")
-make_check_function("PyTZInfo_Check", "tzinfo")
+PyDateTime_Check, PyDateTime_CheckExact = make_check_function(
+    "PyDateTime_Check", "datetime")
+PyDate_Check, PyDate_CheckExact = make_check_function("PyDate_Check", "date")
+PyTime_Check, PyTime_CheckExact = make_check_function("PyTime_Check", "time")
+PyDelta_Check, PyDelta_CheckExact = make_check_function(
+    "PyDelta_Check", "timedelta")
+PyTZInfo_Check, PyTZInfo_CheckExact = make_check_function(
+    "PyTZInfo_Check", "tzinfo")
 
 # Constructors. They are better used as macros.
 
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -3,7 +3,8 @@
     build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct,
     bootstrap_function, Py_bufferP, slot_function)
 from pypy.module.cpyext.pyobject import (
-    PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr)
+    PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr,
+    get_typedescr, track_reference)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -28,7 +29,7 @@
                    basestruct=PyMemoryViewObject.TO,
                    attach=memory_attach,
                    dealloc=memory_dealloc,
-                   #realize=memory_realize,
+                   realize=memory_realize,
                   )
 
 def memory_attach(space, py_obj, w_obj, w_userdata=None):
@@ -54,11 +55,35 @@
                                              track_allocation=False))
         rffi.setintfield(view, 'c_readonly', 1)
 
-def memory_realize(space, py_obj):
+def memory_realize(space, obj):
     """
     Creates the memory object in the interpreter
     """
-    raise oefmt(space.w_NotImplementedError, "cannot call this yet")
+    from pypy.module.cpyext.slotdefs import CPyBuffer, fq
+    py_mem = rffi.cast(PyMemoryViewObject, obj)
+    view = py_mem.c_view
+    ndim = widen(view.c_ndim)
+    shape = None
+    if view.c_shape:
+        shape = [view.c_shape[i] for i in range(ndim)]
+    strides = None
+    if view.c_strides:
+        strides = [view.c_strides[i] for i in range(ndim)]
+    format = 'B'
+    if view.c_format:
+        format = rffi.charp2str(view.c_format)
+    buf = CPyBuffer(space, view.c_buf, view.c_len, from_ref(space, view.c_obj),
+                    format=format, shape=shape, strides=strides,
+                    ndim=ndim, itemsize=view.c_itemsize,
+                    readonly=widen(view.c_readonly))
+    # Ensure view.c_buf is released upon object finalization
+    fq.register_finalizer(buf)
+    # Allow subclassing W_MemeoryView
+    w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
+    w_obj = space.allocate_instance(W_MemoryView, w_type)
+    w_obj.__init__(buf)
+    track_reference(space, obj, w_obj)
+    return w_obj
 
 @slot_function([PyObject], lltype.Void)
 def memory_dealloc(space, py_obj):
@@ -208,17 +233,41 @@
     py_memview = make_ref(space, w_memview, w_obj)
     return py_memview
 
- at cpython_api([Py_bufferP], PyObject)
+ at cpython_api([Py_bufferP], PyObject, result_is_ll=True)
 def PyMemoryView_FromBuffer(space, view):
     """Create a memoryview object wrapping the given buffer-info structure view.
     The memoryview object then owns the buffer, which means you shouldn't
     try to release it yourself: it will be released on deallocation of the
     memoryview object."""
-    assert view.c_obj
-    w_obj = from_ref(space, view.c_obj)
-    if isinstance(w_obj, W_MemoryView):
-        return w_obj
-    return space.call_method(space.builtin, "memoryview", w_obj)
+    # XXX this should allocate a PyMemoryViewObject and
+    # copy view into obj.c_view, without creating a new view.c_obj
+    typedescr = get_typedescr(W_MemoryView.typedef)
+    py_obj = typedescr.allocate(space, space.w_memoryview)
+    py_mem = rffi.cast(PyMemoryViewObject, py_obj)
+    mview = py_mem.c_view
+    mview.c_buf = view.c_buf
+    mview.c_obj = view.c_obj
+    mview.c_len = view.c_len
+    mview.c_itemsize = view.c_itemsize
+    mview.c_readonly = view.c_readonly
+    mview.c_ndim = view.c_ndim
+    mview.c_format = view.c_format
+    if view.c_strides == rffi.cast(Py_ssize_tP, view.c__strides):
+        py_mem.c_view.c_strides = rffi.cast(Py_ssize_tP, py_mem.c_view.c__strides)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_strides[i] = view.c_strides[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_strides = view.c_strides
+    if view.c_shape == rffi.cast(Py_ssize_tP, view.c__shape):
+        py_mem.c_view.c_shape = rffi.cast(Py_ssize_tP, py_mem.c_view.c__shape)
+        for i in range(view.c_ndim):
+            py_mem.c_view.c_shape[i] = view.c_shape[i]
+    else:
+        # some externally allocated memory chunk
+        py_mem.c_view.c_shape = view.c_shape
+    # XXX ignore suboffsets?
+    return py_obj
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -2,6 +2,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
     PyVarObject, Py_buffer, size_t, slot_function,
+    PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -486,22 +487,28 @@
     Fills in a buffer-info structure correctly for an exporter that can only
     share a contiguous chunk of memory of "unsigned bytes" of the given
     length. Returns 0 on success and -1 (with raising an error) on error.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
     """
     if flags & PyBUF_WRITABLE and readonly:
         raise oefmt(space.w_ValueError, "Object is not writable")
     view.c_buf = buf
     view.c_len = length
     view.c_obj = obj
-    Py_IncRef(space, obj)
+    if obj:
+        Py_IncRef(space, obj)
     view.c_itemsize = 1
     rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
+    rffi.setintfield(view, 'c_ndim', 1)
     view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
     view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
     view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
 
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -324,7 +324,7 @@
 
     def __init__(self, space, ptr, size, w_obj, format='B', shape=None,
                 strides=None, ndim=1, itemsize=1, readonly=True,
-                releasebuffer=None):
+                releasebufferproc=rffi.cast(rffi.VOIDP, 0)):
         self.space = space
         self.ptr = ptr
         self.size = size
@@ -342,7 +342,7 @@
         self.ndim = ndim
         self.itemsize = itemsize
         self.readonly = readonly
-        self.releasebufferproc = releasebuffer
+        self.releasebufferproc = releasebufferproc
 
     def releasebuffer(self):
         if self.pyobj:
@@ -360,7 +360,10 @@
                 for i in range(self.ndim):
                     pybuf.c_shape[i] = self.shape[i]
                     pybuf.c_strides[i] = self.strides[i]
-                pybuf.c_format = rffi.str2charp(self.format)
+                if self.format:
+                    pybuf.c_format = rffi.str2charp(self.format)
+                else:
+                    pybuf.c_format = rffi.str2charp("B")
                 generic_cpy_call(self.space, func_target, self.pyobj, pybuf)
             self.releasebufferproc = rffi.cast(rffi.VOIDP, 0)
 
@@ -407,9 +410,9 @@
     func_target = rffi.cast(readbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
@@ -417,7 +420,7 @@
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -426,16 +429,16 @@
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
     decref(space, py_obj)
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
         index = rffi.cast(Py_ssize_t, 0)
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
-                               releasebuffer=releasebuffer)
+                               releasebufferproc=rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
@@ -443,9 +446,9 @@
     func_target = rffi.cast(getbufferproc, func)
     py_obj = make_ref(space, w_self)
     py_type = py_obj.c_ob_type
-    releasebuffer = rffi.cast(rffi.VOIDP, 0)
+    rbp = rffi.cast(rffi.VOIDP, 0)
     if py_type.c_tp_as_buffer:
-        releasebuffer = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
     decref(space, py_obj)
     with lltype.scoped_alloc(Py_buffer) as pybuf:
         _flags = 0
@@ -471,7 +474,7 @@
                             ndim=ndim, shape=shape, strides=strides,
                             itemsize=pybuf.c_itemsize,
                             readonly=widen(pybuf.c_readonly),
-                            releasebuffer = releasebuffer)
+                            releasebufferproc = rbp)
         fq.register_finalizer(buf)
         return space.newbuffer(buf)
 
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,20 +1,22 @@
 import py, pytest
+import contextlib
 from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
-from pypy.module.cpyext import api
+from pypy.module.cpyext.api import (
+    slot_function, cpython_api, copy_header_files, INTERPLEVEL_API,
+    Py_ssize_t, Py_ssize_tP, PyObject)
 from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest
-PyObject = api.PyObject
 from pypy.interpreter.error import OperationError
 from rpython.rlib import rawrefcount
 import os
 
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetWrapped(space, w_arg):
-    assert isinstance(w_arg, W_Root)
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetReference(space, arg):
-    assert lltype.typeOf(arg) ==  PyObject
+ at contextlib.contextmanager
+def raises_w(space, expected_exc):
+    with pytest.raises(OperationError) as excinfo:
+        yield
+    operror = excinfo.value
+    assert operror.w_type is getattr(space, 'w_' + expected_exc.__name__)
 
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
@@ -35,7 +37,7 @@
             def __getattr__(self, name):
                 return getattr(cls.space, name)
         cls.api = CAPI()
-        CAPI.__dict__.update(api.INTERPLEVEL_API)
+        CAPI.__dict__.update(INTERPLEVEL_API)
 
         print 'DONT_FREE_ANY_MORE'
         rawrefcount._dont_free_any_more()
@@ -71,20 +73,28 @@
         if self.check_and_print_leaks():
             assert False, "Test leaks or loses object(s)."
 
- at api.cpython_api([api.Py_ssize_t], api.Py_ssize_t, error=-1)
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetWrapped(space, w_arg):
+    assert isinstance(w_arg, W_Root)
+
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetReference(space, arg):
+    assert lltype.typeOf(arg) ==  PyObject
+
+ at cpython_api([Py_ssize_t], Py_ssize_t, error=-1)
 def PyPy_TypedefTest1(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_t
+    assert lltype.typeOf(arg) == Py_ssize_t
     return 0
 
- at api.cpython_api([api.Py_ssize_tP], api.Py_ssize_tP)
+ at cpython_api([Py_ssize_tP], Py_ssize_tP)
 def PyPy_TypedefTest2(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_tP
+    assert lltype.typeOf(arg) == Py_ssize_tP
     return None
 
 class TestConversion(BaseApiTest):
-    def test_conversions(self, space, api):
-        api.PyPy_GetWrapped(space.w_None)
-        api.PyPy_GetReference(space.w_None)
+    def test_conversions(self, space):
+        PyPy_GetWrapped(space, space.w_None)
+        PyPy_GetReference(space, space.w_None)
 
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
@@ -95,7 +105,7 @@
         assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Signed *arg0'
 
         PyPy_TypedefTest1(space, 0)
-        ppos = lltype.malloc(api.Py_ssize_tP.TO, 1, flavor='raw')
+        ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
         ppos[0] = 0
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
@@ -103,7 +113,7 @@
 @pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
-    api.copy_header_files(tmpdir, True)
+    copy_header_files(tmpdir, True)
     def check(name):
         f = tmpdir.join(name)
         assert f.check(file=True)
diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py
--- a/pypy/module/cpyext/test/test_boolobject.py
+++ b/pypy/module/cpyext/test/test_boolobject.py
@@ -1,20 +1,22 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong
+from pypy.module.cpyext.floatobject import PyFloat_FromDouble
 
 class TestBoolObject(BaseApiTest):
-    def test_fromlong(self, space, api):
+    def test_fromlong(self, space):
         for i in range(-3, 3):
-            obj = api.PyBool_FromLong(i)
+            obj = PyBool_FromLong(space, i)
             if i:
                 assert obj is space.w_True
             else:
                 assert obj is space.w_False
 
-    def test_check(self, space, api):
-        assert api.PyBool_Check(space.w_True)
-        assert api.PyBool_Check(space.w_False)
-        assert not api.PyBool_Check(space.w_None)
-        assert not api.PyBool_Check(api.PyFloat_FromDouble(1.0))
+    def test_check(self, space):
+        assert PyBool_Check(space, space.w_True)
+        assert PyBool_Check(space, space.w_False)
+        assert not PyBool_Check(space, space.w_None)
+        assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0))
 
 class AppTestBoolMacros(AppTestCpythonExtensionBase):
     def test_macros(self):
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -1,14 +1,19 @@
 # encoding: utf-8
+import pytest
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.interpreter.error import OperationError
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject
+from pypy.module.cpyext.bytesobject import (
+    new_empty_str, PyBytesObject, _PyString_Resize, PyString_Concat,
+    PyString_ConcatAndDel, PyString_Format, PyString_InternFromString,
+    PyString_AsEncodedObject, PyString_AsDecodedObject, _PyString_Eq,
+    _PyString_Join)
 from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref
+from pypy.module.cpyext.object import PyObject_AsCharBuffer
 from pypy.module.cpyext.api import PyTypeObjectPtr
 
-import py
-import sys
 
 class AppTestBytesObject(AppTestCpythonExtensionBase):
     def test_bytesobject(self):
@@ -441,21 +446,21 @@
         assert a == 'abc'
 
 class TestBytes(BaseApiTest):
-    def test_bytes_resize(self, space, api):
+    def test_bytes_resize(self, space):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         py_str.c_ob_sval[0] = 'a'
         py_str.c_ob_sval[1] = 'b'
         py_str.c_ob_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
-        api._PyString_Resize(ar, 3)
+        _PyString_Resize(space, ar, 3)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 3
         assert py_str.c_ob_sval[1] == 'b'
         assert py_str.c_ob_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
-        api._PyString_Resize(ar, 10)
+        _PyString_Resize(space, ar, 10)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 10
         assert py_str.c_ob_sval[1] == 'b'
@@ -463,7 +468,7 @@
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
-    def test_string_buffer(self, space, api):
+    def test_string_buffer(self, space):
         py_str = new_empty_str(space, 10)
         c_buf = py_str.c_ob_type.c_tp_as_buffer
         assert c_buf
@@ -481,108 +486,106 @@
         lltype.free(ref, flavor='raw')
         Py_DecRef(space, py_obj)
 
-    def test_Concat(self, space, api):
+    def test_Concat(self, space):
         ref = make_ref(space, space.wrap('abc'))
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref
         prev_refcnt = ref.c_ob_refcnt
-        api.PyString_Concat(ptr, space.wrap('def'))
+        PyString_Concat(space, ptr, space.wrap('def'))
         assert ref.c_ob_refcnt == prev_refcnt - 1
         assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
-        api.PyString_Concat(ptr, space.w_None)
+        with pytest.raises(OperationError):
+            PyString_Concat(space, ptr, space.w_None)
         assert not ptr[0]
-        api.PyErr_Clear()
         ptr[0] = lltype.nullptr(PyObject.TO)
-        api.PyString_Concat(ptr, space.wrap('def')) # should not crash
+        PyString_Concat(space, ptr, space.wrap('def')) # should not crash
         lltype.free(ptr, flavor='raw')
 
-    def test_ConcatAndDel(self, space, api):
+    def test_ConcatAndDel(self, space):
         ref1 = make_ref(space, space.wrap('abc'))
         ref2 = make_ref(space, space.wrap('def'))
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref1
         prev_refcnf = ref2.c_ob_refcnt
-        api.PyString_ConcatAndDel(ptr, ref2)
+        PyString_ConcatAndDel(space, ptr, ref2)
         assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
         assert ref2.c_ob_refcnt == prev_refcnf - 1
         Py_DecRef(space, ptr[0])
         ptr[0] = lltype.nullptr(PyObject.TO)
         ref2 = make_ref(space, space.wrap('foo'))
         prev_refcnf = ref2.c_ob_refcnt
-        api.PyString_ConcatAndDel(ptr, ref2) # should not crash
+        PyString_ConcatAndDel(space, ptr, ref2) # should not crash
         assert ref2.c_ob_refcnt == prev_refcnf - 1
         lltype.free(ptr, flavor='raw')
 
-    def test_format(self, space, api):
+    def test_format(self, space):
         assert "1 2" == space.unwrap(
-            api.PyString_Format(space.wrap('%s %d'), space.wrap((1, 2))))
+            PyString_Format(space, space.wrap('%s %d'), space.wrap((1, 2))))
 
-    def test_asbuffer(self, space, api):
+    def test_asbuffer(self, space):
         bufp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
 
         w_text = space.wrap("text")
         ref = make_ref(space, w_text)
         prev_refcnt = ref.c_ob_refcnt
-        assert api.PyObject_AsCharBuffer(ref, bufp, lenp) == 0
+        assert PyObject_AsCharBuffer(space, ref, bufp, lenp) == 0
         assert ref.c_ob_refcnt == prev_refcnt
         assert lenp[0] == 4
         assert rffi.charp2str(bufp[0]) == 'text'
         lltype.free(bufp, flavor='raw')
         lltype.free(lenp, flavor='raw')
-        api.Py_DecRef(ref)
+        Py_DecRef(space, ref)
 
-    def test_intern(self, space, api):
+    def test_intern(self, space):
         buf = rffi.str2charp("test")
-        w_s1 = api.PyString_InternFromString(buf)
-        w_s2 = api.PyString_InternFromString(buf)
+        w_s1 = PyString_InternFromString(space, buf)
+        w_s2 = PyString_InternFromString(space, buf)
         rffi.free_charp(buf)
         assert w_s1 is w_s2
 
-    def test_AsEncodedObject(self, space, api):
+    def test_AsEncodedObject(self, space):
         ptr = space.wrap('abc')
 
         errors = rffi.str2charp("strict")
 
         encoding = rffi.str2charp("hex")
-        res = api.PyString_AsEncodedObject(
-            ptr, encoding, errors)
+        res = PyString_AsEncodedObject(space, ptr, encoding, errors)
         assert space.unwrap(res) == "616263"
 
-        res = api.PyString_AsEncodedObject(
+        res = PyString_AsEncodedObject(space,
             ptr, encoding, lltype.nullptr(rffi.CCHARP.TO))
         assert space.unwrap(res) == "616263"
         rffi.free_charp(encoding)
 
         encoding = rffi.str2charp("unknown_encoding")
-        self.raises(space, api, LookupError, api.PyString_AsEncodedObject,
-                    ptr, encoding, errors)
+        with raises_w(space, LookupError):
+            PyString_AsEncodedObject(space, ptr, encoding, errors)
         rffi.free_charp(encoding)
 
         rffi.free_charp(errors)
 
-        res = api.PyString_AsEncodedObject(
-            ptr, lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO))
+        NULL = lltype.nullptr(rffi.CCHARP.TO)
+        res = PyString_AsEncodedObject(space, ptr, NULL, NULL)
         assert space.unwrap(res) == "abc"
+        with raises_w(space, TypeError):
+            PyString_AsEncodedObject(space, space.wrap(2), NULL, NULL)
 
-        self.raises(space, api, TypeError, api.PyString_AsEncodedObject,
-            space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
-        )
-
-    def test_AsDecodedObject(self, space, api):
+    def test_AsDecodedObject(self, space):
         w_str = space.wrap('caf\xe9')
         encoding = rffi.str2charp("latin-1")
-        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        w_res = PyString_AsDecodedObject(space, w_str, encoding, None)
         rffi.free_charp(encoding)
         assert space.unwrap(w_res) == u"caf\xe9"
 
-    def test_eq(self, space, api):
-        assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
-        assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
+    def test_eq(self, space):
+        assert 1 == _PyString_Eq(
+            space, space.wrap("hello"), space.wrap("hello"))
+        assert 0 == _PyString_Eq(
+            space, space.wrap("hello"), space.wrap("world"))
 
-    def test_join(self, space, api):
+    def test_join(self, space):
         w_sep = space.wrap('<sep>')
         w_seq = space.wrap(['a', 'b'])
-        w_joined = api._PyString_Join(w_sep, w_seq)
+        w_joined = _PyString_Join(space, w_sep, w_seq)
         assert space.unwrap(w_joined) == 'a<sep>b'
-
diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py
--- a/pypy/module/cpyext/test/test_classobject.py
+++ b/pypy/module/cpyext/test/test_classobject.py
@@ -1,9 +1,13 @@
+from pypy.interpreter.function import Function
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.interpreter.function import Function, Method
+from pypy.module.cpyext.classobject import (
+    PyClass_Check, PyClass_New, PyInstance_Check, PyInstance_New,
+    PyInstance_NewRaw, _PyInstance_Lookup)
+from pypy.module.cpyext.object import PyObject_GetAttr
 
 class TestClassObject(BaseApiTest):
-    def test_newinstance(self, space, api):
+    def test_newinstance(self, space):
         w_class = space.appexec([], """():
             class C:
                 x = None
@@ -14,23 +18,23 @@
             return C
         """)
 
-        assert api.PyClass_Check(w_class)
+        assert PyClass_Check(space, w_class)
 
-        w_instance = api.PyInstance_NewRaw(w_class, None)
-        assert api.PyInstance_Check(w_instance)
+        w_instance = PyInstance_NewRaw(space, w_class, None)
+        assert PyInstance_Check(space, w_instance)
         assert space.getattr(w_instance, space.wrap('x')) is space.w_None
 
-        w_instance = api.PyInstance_NewRaw(w_class, space.wrap(dict(a=3)))
+        w_instance = PyInstance_NewRaw(space, w_class, space.wrap(dict(a=3)))
         assert space.getattr(w_instance, space.wrap('x')) is space.w_None
         assert space.unwrap(space.getattr(w_instance, space.wrap('a'))) == 3
 
-        w_instance = api.PyInstance_New(w_class,
+        w_instance = PyInstance_New(space, w_class,
                                         space.wrap((3,)), space.wrap(dict(y=2)))
         assert space.unwrap(space.getattr(w_instance, space.wrap('x'))) == 1
         assert space.unwrap(space.getattr(w_instance, space.wrap('y'))) == 2
         assert space.unwrap(space.getattr(w_instance, space.wrap('args'))) == (3,)
 
-    def test_lookup(self, space, api):
+    def test_lookup(self, space):
         w_instance = space.appexec([], """():
             class C:
                 def __init__(self):
@@ -39,25 +43,26 @@
             return C()
         """)
 
-        assert api.PyInstance_Check(w_instance)
-        assert api.PyObject_GetAttr(w_instance, space.wrap('x')) is space.w_None
-        assert api._PyInstance_Lookup(w_instance, space.wrap('x')) is space.w_None
-        assert api._PyInstance_Lookup(w_instance, space.wrap('y')) is None
-        assert not api.PyErr_Occurred()
+        assert PyInstance_Check(space, w_instance)
+        assert PyObject_GetAttr(space, w_instance, space.wrap('x')) is space.w_None
+        assert _PyInstance_Lookup(space, w_instance, space.wrap('x')) is space.w_None
+        assert _PyInstance_Lookup(space, w_instance, space.wrap('y')) is None
 
         # getattr returns a bound method
-        assert not isinstance(api.PyObject_GetAttr(w_instance, space.wrap('f')), Function)
+        assert not isinstance(
+            PyObject_GetAttr(space, w_instance, space.wrap('f')), Function)
         # _PyInstance_Lookup returns the raw descriptor
-        assert isinstance(api._PyInstance_Lookup(w_instance, space.wrap('f')), Function)
+        assert isinstance(
+            _PyInstance_Lookup(space, w_instance, space.wrap('f')), Function)
 
-    def test_pyclass_new(self, space, api):
+    def test_pyclass_new(self, space):
         w_bases = space.newtuple([])
         w_dict = space.newdict()
         w_name = space.wrap("C")
-        w_class = api.PyClass_New(w_bases, w_dict, w_name)
+        w_class = PyClass_New(space, w_bases, w_dict, w_name)
         assert not space.isinstance_w(w_class, space.w_type)
         w_instance = space.call_function(w_class)
-        assert api.PyInstance_Check(w_instance)
+        assert PyInstance_Check(space, w_instance)
         assert space.is_true(space.call_method(space.builtin, "isinstance",
                                                w_instance, w_class))
 
@@ -69,5 +74,6 @@
                  Py_INCREF(&PyClass_Type);
                  return (PyObject*)&PyClass_Type;
              """)])
-        class C: pass
+        class C:
+            pass
         assert module.get_classtype() is type(C)
diff --git a/pypy/module/cpyext/test/test_codecs.py b/pypy/module/cpyext/test/test_codecs.py
--- a/pypy/module/cpyext/test/test_codecs.py
+++ b/pypy/module/cpyext/test/test_codecs.py
@@ -1,14 +1,15 @@
 # encoding: iso-8859-15
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.codecs import (
+    PyCodec_IncrementalEncoder, PyCodec_IncrementalDecoder)
 
 class TestCodecs(BaseApiTest):
-    def test_incremental(self, space, api):
+    def test_incremental(self, space):
         utf8 = rffi.str2charp('utf-8')
-        w_encoder = api.PyCodec_IncrementalEncoder(utf8, None)
+        w_encoder = PyCodec_IncrementalEncoder(space, utf8, None)
         w_encoded = space.call_method(w_encoder, 'encode', space.wrap(u'sp&#228;m'))
-        w_decoder = api.PyCodec_IncrementalDecoder(utf8, None)
+        w_decoder = PyCodec_IncrementalDecoder(space, utf8, None)
         w_decoded = space.call_method(w_decoder, 'decode', w_encoded)
         assert space.unwrap(w_decoded) == u'sp&#228;m'
         rffi.free_charp(utf8)
-
diff --git a/pypy/module/cpyext/test/test_complexobject.py b/pypy/module/cpyext/test/test_complexobject.py
--- a/pypy/module/cpyext/test/test_complexobject.py
+++ b/pypy/module/cpyext/test/test_complexobject.py
@@ -1,23 +1,23 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.complexobject import (
+    PyComplex_FromDoubles, PyComplex_RealAsDouble, PyComplex_ImagAsDouble)
 
 class TestComplexObject(BaseApiTest):
-    def test_complexobject(self, space, api):
-        w_value = api.PyComplex_FromDoubles(1.2, 3.4)
+    def test_complexobject(self, space):
+        w_value = PyComplex_FromDoubles(space, 1.2, 3.4)
         assert space.unwrap(w_value) == 1.2+3.4j
-        assert api.PyComplex_RealAsDouble(w_value) == 1.2
-        assert api.PyComplex_ImagAsDouble(w_value) == 3.4
+        assert PyComplex_RealAsDouble(space, w_value) == 1.2
+        assert PyComplex_ImagAsDouble(space, w_value) == 3.4
 
-        assert api.PyComplex_RealAsDouble(space.wrap(42)) == 42
-        assert api.PyComplex_RealAsDouble(space.wrap(1.5)) == 1.5
-        assert api.PyComplex_ImagAsDouble(space.wrap(1.5)) == 0.0
+        assert PyComplex_RealAsDouble(space, space.wrap(42)) == 42
+        assert PyComplex_RealAsDouble(space, space.wrap(1.5)) == 1.5
+        assert PyComplex_ImagAsDouble(space, space.wrap(1.5)) == 0.0
 
         # cpython accepts anything for PyComplex_ImagAsDouble
-        assert api.PyComplex_ImagAsDouble(space.w_None) == 0.0
-        assert not api.PyErr_Occurred()
-        assert api.PyComplex_RealAsDouble(space.w_None) == -1.0
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
+        assert PyComplex_ImagAsDouble(space, space.w_None) == 0.0
+        with raises_w(space, TypeError):
+            PyComplex_RealAsDouble(space, space.w_None)
 
 class AppTestCComplex(AppTestCpythonExtensionBase):
     def test_AsCComplex(self):
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -1,92 +1,96 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.cdatetime import *
+from pypy.module.cpyext.cdatetime import (
+    _PyDateTime_Import, _PyDateTime_FromDateAndTime, _PyDate_FromDate,
+    _PyTime_FromTime, _PyDelta_FromDelta)
 import datetime
 
 class TestDatetime(BaseApiTest):
-    def test_date(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_date = api._PyDate_FromDate(2010, 06, 03, date_api.c_DateType)
+    def test_date(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_date = _PyDate_FromDate(space, 2010, 06, 03, date_api.c_DateType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03'
 
-        assert api.PyDate_Check(w_date)
-        assert api.PyDate_CheckExact(w_date)
+        assert PyDate_Check(space, w_date)
+        assert PyDate_CheckExact(space, w_date)
 
-        assert api.PyDateTime_GET_YEAR(w_date) == 2010
-        assert api.PyDateTime_GET_MONTH(w_date) == 6
-        assert api.PyDateTime_GET_DAY(w_date) == 3
+        assert PyDateTime_GET_YEAR(space, w_date) == 2010
+        assert PyDateTime_GET_MONTH(space, w_date) == 6
+        assert PyDateTime_GET_DAY(space, w_date) == 3
 
-    def test_time(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_time = api._PyTime_FromTime(23, 15, 40, 123456,
-                                      space.w_None, date_api.c_TimeType)
+    def test_time(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_time = _PyTime_FromTime(
+            space, 23, 15, 40, 123456, space.w_None, date_api.c_TimeType)
         assert space.unwrap(space.str(w_time)) == '23:15:40.123456'
 
-        assert api.PyTime_Check(w_time)
-        assert api.PyTime_CheckExact(w_time)
+        assert PyTime_Check(space, w_time)
+        assert PyTime_CheckExact(space, w_time)
 
-        assert api.PyDateTime_TIME_GET_HOUR(w_time) == 23
-        assert api.PyDateTime_TIME_GET_MINUTE(w_time) == 15
-        assert api.PyDateTime_TIME_GET_SECOND(w_time) == 40
-        assert api.PyDateTime_TIME_GET_MICROSECOND(w_time) == 123456
+        assert PyDateTime_TIME_GET_HOUR(space, w_time) == 23
+        assert PyDateTime_TIME_GET_MINUTE(space, w_time) == 15
+        assert PyDateTime_TIME_GET_SECOND(space, w_time) == 40
+        assert PyDateTime_TIME_GET_MICROSECOND(space, w_time) == 123456
 
-    def test_datetime(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_date = api._PyDateTime_FromDateAndTime(
-            2010, 06, 03, 23, 15, 40, 123456,
-            space.w_None, date_api.c_DateTimeType)
+    def test_datetime(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_date = _PyDateTime_FromDateAndTime(
+            space, 2010, 06, 03, 23, 15, 40, 123456, space.w_None,
+            date_api.c_DateTimeType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03 23:15:40.123456'
 
-        assert api.PyDateTime_Check(w_date)
-        assert api.PyDateTime_CheckExact(w_date)
-        assert api.PyDate_Check(w_date)
-        assert not api.PyDate_CheckExact(w_date)
+        assert PyDateTime_Check(space, w_date)
+        assert PyDateTime_CheckExact(space, w_date)
+        assert PyDate_Check(space, w_date)
+        assert not PyDate_CheckExact(space, w_date)
 
-        assert api.PyDateTime_GET_YEAR(w_date) == 2010
-        assert api.PyDateTime_GET_MONTH(w_date) == 6
-        assert api.PyDateTime_GET_DAY(w_date) == 3
-        assert api.PyDateTime_DATE_GET_HOUR(w_date) == 23
-        assert api.PyDateTime_DATE_GET_MINUTE(w_date) == 15
-        assert api.PyDateTime_DATE_GET_SECOND(w_date) == 40
-        assert api.PyDateTime_DATE_GET_MICROSECOND(w_date) == 123456
+        assert PyDateTime_GET_YEAR(space, w_date) == 2010
+        assert PyDateTime_GET_MONTH(space, w_date) == 6
+        assert PyDateTime_GET_DAY(space, w_date) == 3
+        assert PyDateTime_DATE_GET_HOUR(space, w_date) == 23
+        assert PyDateTime_DATE_GET_MINUTE(space, w_date) == 15
+        assert PyDateTime_DATE_GET_SECOND(space, w_date) == 40
+        assert PyDateTime_DATE_GET_MICROSECOND(space, w_date) == 123456
 
-    def test_delta(self, space, api):
-        date_api = api._PyDateTime_Import()
+    def test_delta(self, space):
+        date_api = _PyDateTime_Import(space)
         w_delta = space.appexec(
             [space.wrap(3), space.wrap(15)], """(days, seconds):
             from datetime import timedelta
             return timedelta(days, seconds)
         """)
-        assert api.PyDelta_Check(w_delta)
-        assert api.PyDelta_CheckExact(w_delta)
+        assert PyDelta_Check(space, w_delta)
+        assert PyDelta_CheckExact(space, w_delta)
 
-        w_delta = api._PyDelta_FromDelta(10, 20, 30, True, date_api.c_DeltaType)
-        assert api.PyDelta_Check(w_delta)
-        assert api.PyDelta_CheckExact(w_delta)
+        w_delta = _PyDelta_FromDelta(space, 10, 20, 30, True, date_api.c_DeltaType)
+        assert PyDelta_Check(space, w_delta)
+        assert PyDelta_CheckExact(space, w_delta)
 
-        assert api.PyDateTime_DELTA_GET_DAYS(w_delta) == 10
-        assert api.PyDateTime_DELTA_GET_SECONDS(w_delta) == 20
-        assert api.PyDateTime_DELTA_GET_MICROSECONDS(w_delta) == 30
+        assert PyDateTime_DELTA_GET_DAYS(space, w_delta) == 10
+        assert PyDateTime_DELTA_GET_SECONDS(space, w_delta) == 20
+        assert PyDateTime_DELTA_GET_MICROSECONDS(space, w_delta) == 30
 
-    def test_fromtimestamp(self, space, api):
+    def test_fromtimestamp(self, space):
         w_args = space.wrap((0,))
-        w_date = api.PyDate_FromTimestamp(w_args)
+        w_date = PyDate_FromTimestamp(space, w_args)
         date = datetime.date.fromtimestamp(0)
         assert space.unwrap(space.str(w_date)) == str(date)
 
         w_args = space.wrap((0,))
-        w_date = api.PyDateTime_FromTimestamp(w_args)
+        w_date = PyDateTime_FromTimestamp(space, w_args)
         date = datetime.datetime.fromtimestamp(0)
         assert space.unwrap(space.str(w_date)) == str(date)
 
-    def test_tzinfo(self, space, api):
+    def test_tzinfo(self, space):
         w_tzinfo = space.appexec(
             [], """():
             from datetime import tzinfo
             return tzinfo()
         """)
-        assert api.PyTZInfo_Check(w_tzinfo)
-        assert api.PyTZInfo_CheckExact(w_tzinfo)
-        assert not api.PyTZInfo_Check(space.w_None)
+        assert PyTZInfo_Check(space, w_tzinfo)
+        assert PyTZInfo_CheckExact(space, w_tzinfo)
+        assert not PyTZInfo_Check(space, space.w_None)
 
 class AppTestDatetime(AppTestCpythonExtensionBase):
     def test_CAPI(self):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -1,83 +1,81 @@
 import py
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP, PyTypeObjectPtr
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.dictobject import *
+from pypy.module.cpyext.pyobject import decref
 
 class TestDictObject(BaseApiTest):
-    def test_dict(self, space, api):
-        d = api.PyDict_New()
+    def test_dict(self, space):
+        d = PyDict_New(space)
         assert space.eq_w(d, space.newdict())
 
-        assert space.eq_w(api.PyDict_GetItem(space.wrap({"a": 72}),
+        assert space.eq_w(PyDict_GetItem(space, space.wrap({"a": 72}),
                                              space.wrap("a")),
                           space.wrap(72))
 
-        assert api.PyDict_SetItem(d, space.wrap("c"), space.wrap(42)) >= 0
+        PyDict_SetItem(space, d, space.wrap("c"), space.wrap(42))
         assert space.eq_w(space.getitem(d, space.wrap("c")),
                           space.wrap(42))
 
         space.setitem(d, space.wrap("name"), space.wrap(3))
-        assert space.eq_w(api.PyDict_GetItem(d, space.wrap("name")),
+        assert space.eq_w(PyDict_GetItem(space, d, space.wrap("name")),
                           space.wrap(3))
 
         space.delitem(d, space.wrap("name"))
-        assert not api.PyDict_GetItem(d, space.wrap("name"))
-        assert not api.PyErr_Occurred()
+        assert not PyDict_GetItem(space, d, space.wrap("name"))
 
         buf = rffi.str2charp("name")
-        assert not api.PyDict_GetItemString(d, buf)
+        assert not PyDict_GetItemString(space, d, buf)
         rffi.free_charp(buf)
-        assert not api.PyErr_Occurred()
 
-        assert api.PyDict_Contains(d, space.wrap("c"))
-        assert not api.PyDict_Contains(d, space.wrap("z"))
+        assert PyDict_Contains(space, d, space.wrap("c"))
+        assert not PyDict_Contains(space, d, space.wrap("z"))
 
-        assert api.PyDict_DelItem(d, space.wrap("c")) == 0
-        assert api.PyDict_DelItem(d, space.wrap("name")) < 0
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
-        assert api.PyDict_Size(d) == 0
+        PyDict_DelItem(space, d, space.wrap("c"))
+        with raises_w(space, KeyError):
+            PyDict_DelItem(space, d, space.wrap("name"))
+        assert PyDict_Size(space, d) == 0
 
         space.setitem(d, space.wrap("some_key"), space.wrap(3))
         buf = rffi.str2charp("some_key")
-        assert api.PyDict_DelItemString(d, buf) == 0
-        assert api.PyDict_Size(d) == 0
-        assert api.PyDict_DelItemString(d, buf) < 0
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
+        PyDict_DelItemString(space, d, buf)
+        assert PyDict_Size(space, d) == 0
+        with raises_w(space, KeyError):
+            PyDict_DelItemString(space, d, buf)
         rffi.free_charp(buf)
 
         d = space.wrap({'a': 'b'})
-        api.PyDict_Clear(d)
-        assert api.PyDict_Size(d) == 0
+        PyDict_Clear(space, d)
+        assert PyDict_Size(space, d) == 0
 
-    def test_check(self, space, api):
-        d = api.PyDict_New()
-        assert api.PyDict_Check(d)
-        assert api.PyDict_CheckExact(d)
+    def test_check(self, space):
+        d = PyDict_New(space, )
+        assert PyDict_Check(space, d)
+        assert PyDict_CheckExact(space, d)
         sub = space.appexec([], """():
             class D(dict):
                 pass
             return D""")
         d = space.call_function(sub)
-        assert api.PyDict_Check(d)
-        assert not api.PyDict_CheckExact(d)
+        assert PyDict_Check(space, d)
+        assert not PyDict_CheckExact(space, d)
         i = space.wrap(2)
-        assert not api.PyDict_Check(i)
-        assert not api.PyDict_CheckExact(i)
+        assert not PyDict_Check(space, i)
+        assert not PyDict_CheckExact(space, i)
 
-    def test_keys(self, space, api):
+    def test_keys(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
-        assert space.eq_w(api.PyDict_Keys(w_d), space.wrap(["a"]))
-        assert space.eq_w(api.PyDict_Values(w_d), space.wrap(["b"]))
-        assert space.eq_w(api.PyDict_Items(w_d), space.wrap([("a", "b")]))
+        assert space.eq_w(PyDict_Keys(space, w_d), space.wrap(["a"]))
+        assert space.eq_w(PyDict_Values(space, w_d), space.wrap(["b"]))
+        assert space.eq_w(PyDict_Items(space, w_d), space.wrap([("a", "b")]))
 
-    def test_merge(self, space, api):
+    def test_merge(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
@@ -86,35 +84,34 @@
         space.setitem(w_d2, space.wrap("c"), space.wrap("d"))
         space.setitem(w_d2, space.wrap("e"), space.wrap("f"))
 
-        api.PyDict_Merge(w_d, w_d2, 0)
+        PyDict_Merge(space, w_d, w_d2, 0)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
-        api.PyDict_Merge(w_d, w_d2, 1)
+        PyDict_Merge(space, w_d, w_d2, 1)
         assert space.unwrap(w_d) == dict(a='c', c='d', e='f')
 
-    def test_update(self, space, api):
+    def test_update(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
-        w_d2 = api.PyDict_Copy(w_d)
+        w_d2 = PyDict_Copy(space, w_d)
         assert not space.is_w(w_d2, w_d)
         space.setitem(w_d, space.wrap("c"), space.wrap("d"))
         space.setitem(w_d2, space.wrap("e"), space.wrap("f"))
 
-        api.PyDict_Update(w_d, w_d2)
+        PyDict_Update(space, w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
-    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+    def test_update_doesnt_accept_list_of_tuples(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
         w_d2 = space.wrap([("c", "d"), ("e", "f")])
 
-        api.PyDict_Update(w_d, w_d2)
-        assert api.PyErr_Occurred() is space.w_AttributeError
-        api.PyErr_Clear()
+        with raises_w(space, AttributeError):
+            PyDict_Update(space, w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b') # unchanged
 
-    def test_iter(self, space, api):
+    def test_iter(self, space):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
 
@@ -125,7 +122,7 @@
 
         try:
             w_copy = space.newdict()
-            while api.PyDict_Next(w_dict, ppos, pkey, pvalue):
+            while PyDict_Next(space, w_dict, ppos, pkey, pvalue):
                 w_key = from_ref(space, pkey[0])
                 w_value = from_ref(space, pvalue[0])
                 space.setitem(w_copy, w_key, w_value)
@@ -134,12 +131,12 @@
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
 
-        api.Py_DecRef(py_dict) # release borrowed references
+        decref(space, py_dict) # release borrowed references
 
         assert space.eq_w(space.len(w_copy), space.len(w_dict))
         assert space.eq_w(w_copy, w_dict)
 
-    def test_iterkeys(self, space, api):
+    def test_iterkeys(self, space):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
 
@@ -151,11 +148,11 @@
         values_w = []
         try:
             ppos[0] = 0
-            while api.PyDict_Next(w_dict, ppos, pkey, None):
+            while PyDict_Next(space, w_dict, ppos, pkey, None):
                 w_key = from_ref(space, pkey[0])
                 keys_w.append(w_key)
             ppos[0] = 0
-            while api.PyDict_Next(w_dict, ppos, None, pvalue):
+            while PyDict_Next(space, w_dict, ppos, None, pvalue):
                 w_value = from_ref(space, pvalue[0])
                 values_w.append(w_value)
         finally:
@@ -163,25 +160,25 @@
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
 
-        api.Py_DecRef(py_dict) # release borrowed references
+        decref(space, py_dict) # release borrowed references
 
         assert space.eq_w(space.newlist(keys_w),
                           space.call_method(w_dict, "keys"))
         assert space.eq_w(space.newlist(values_w),
                           space.call_method(w_dict, "values"))
 
-    def test_dictproxy(self, space, api):
+    def test_dictproxy(self, space):
         w_dict = space.sys.get('modules')
-        w_proxy = api.PyDictProxy_New(w_dict)
+        w_proxy = PyDictProxy_New(space, w_dict)
         assert space.contains_w(w_proxy, space.wrap('sys'))
         raises(OperationError, space.setitem,
                w_proxy, space.wrap('sys'), space.w_None)
         raises(OperationError, space.delitem,
                w_proxy, space.wrap('sys'))
         raises(OperationError, space.call_method, w_proxy, 'clear')
-        assert api.PyDictProxy_Check(w_proxy)
+        assert PyDictProxy_Check(space, w_proxy)
 
-    def test_typedict1(self, space, api):
+    def test_typedict1(self, space):
         py_type = make_ref(space, space.w_int)
         py_dict = rffi.cast(PyTypeObjectPtr, py_type).c_tp_dict
         ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
@@ -191,7 +188,7 @@
         pvalue = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         try:
             w_copy = space.newdict()
-            while api.PyDict_Next(py_dict, ppos, pkey, pvalue):
+            while PyDict_Next(space, py_dict, ppos, pkey, pvalue):
                 w_key = from_ref(space, pkey[0])
                 w_value = from_ref(space, pvalue[0])
                 space.setitem(w_copy, w_key, w_value)
@@ -199,7 +196,7 @@
             lltype.free(ppos, flavor='raw')
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
-        api.Py_DecRef(py_type) # release borrowed references
+        decref(space, py_type) # release borrowed references
         # do something with w_copy ?
 
 class AppTestDictObject(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -1,17 +1,26 @@
+import sys
+import os
+import pytest
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.object import PyObject_Size, PyObject_GetItem
+from pypy.module.cpyext.pythonrun import Py_AtExit
 from pypy.module.cpyext.eval import (
-    Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import (c_fopen, c_fclose, c_fileno, 
-                Py_ssize_tP, is_valid_fd)
+    Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags,
+    PyEval_CallObjectWithKeywords, PyObject_CallObject, PyEval_EvalCode,
+    PyRun_SimpleString, PyRun_String, PyRun_StringFlags, PyRun_File,
+    PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals,
+    _PyEval_SliceIndex)
+from pypy.module.cpyext.api import (
+    c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd)
 from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
-import sys, os
 
 class TestEval(BaseApiTest):
-    def test_eval(self, space, api):
+    def test_eval(self, space):
         w_l, w_f = space.fixedview(space.appexec([], """():
         l = []
         def f(arg1, arg2):
@@ -22,7 +31,7 @@
         """))
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, None)
+        w_res = PyEval_CallObjectWithKeywords(space, w_f, w_t, None)
         assert space.int_w(w_res) == 2
         assert space.len_w(w_l) == 2
         w_f = space.appexec([], """():
@@ -35,10 +44,10 @@
         w_t = space.newtuple([space.w_None, space.w_None])
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("xyz"), space.wrap(3))
-        w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, w_d)
+        w_res = PyEval_CallObjectWithKeywords(space, w_f, w_t, w_d)
         assert space.int_w(w_res) == 21
 
-    def test_call_object(self, space, api):
+    def test_call_object(self, space):
         w_l, w_f = space.fixedview(space.appexec([], """():
         l = []
         def f(arg1, arg2):
@@ -49,7 +58,7 @@
         """))
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyObject_CallObject(w_f, w_t)
+        w_res = PyObject_CallObject(space, w_f, w_t)
         assert space.int_w(w_res) == 2
         assert space.len_w(w_l) == 2
 
@@ -61,11 +70,11 @@
             """)
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyObject_CallObject(w_f, w_t)
+        w_res = PyObject_CallObject(space, w_f, w_t)
 
         assert space.int_w(w_res) == 10
 
-    def test_evalcode(self, space, api):
+    def test_evalcode(self, space):
         w_f = space.appexec([], """():
             def f(*args):
                 assert isinstance(args, tuple)
@@ -77,84 +86,78 @@
         w_globals = space.newdict()
         w_locals = space.newdict()
         space.setitem(w_locals, space.wrap("args"), w_t)
-        w_res = api.PyEval_EvalCode(w_f.code, w_globals, w_locals)
+        w_res = PyEval_EvalCode(space, w_f.code, w_globals, w_locals)
 
         assert space.int_w(w_res) == 10
 
-    def test_run_simple_string(self, space, api):
+    def test_run_simple_string(self, space):
         def run(code):
             buf = rffi.str2charp(code)
             try:
-                return api.PyRun_SimpleString(buf)
+                return PyRun_SimpleString(space, buf)
             finally:
                 rffi.free_charp(buf)
 
-        assert 0 == run("42 * 43")
+        assert run("42 * 43") == 0  # no error
+        with pytest.raises(OperationError):
+            run("4..3 * 43")
 
-        assert -1 == run("4..3 * 43")
-
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
-
-    def test_run_string(self, space, api):
+    def test_run_string(self, space):
         def run(code, start, w_globals, w_locals):
             buf = rffi.str2charp(code)
             try:
-                return api.PyRun_String(buf, start, w_globals, w_locals)
+                return PyRun_String(space, buf, start, w_globals, w_locals)
             finally:
                 rffi.free_charp(buf)
 
         w_globals = space.newdict()
         assert 42 * 43 == space.unwrap(
             run("42 * 43", Py_eval_input, w_globals, w_globals))
-        assert api.PyObject_Size(w_globals) == 0
+        assert PyObject_Size(space, w_globals) == 0
 
         assert run("a = 42 * 43", Py_single_input,
                    w_globals, w_globals) == space.w_None
         assert 42 * 43 == space.unwrap(
-            api.PyObject_GetItem(w_globals, space.wrap("a")))
+            PyObject_GetItem(space, w_globals, space.wrap("a")))
 
-    def test_run_string_flags(self, space, api):
+    def test_run_string_flags(self, space):
         flags = lltype.malloc(PyCompilerFlags, flavor='raw')
         flags.c_cf_flags = rffi.cast(rffi.INT, consts.PyCF_SOURCE_IS_UTF8)
         w_globals = space.newdict()
         buf = rffi.str2charp("a = u'caf\xc3\xa9'")
         try:
-            api.PyRun_StringFlags(buf, Py_single_input,
-                                  w_globals, w_globals, flags)
+            PyRun_StringFlags(space, buf, Py_single_input, w_globals,
+                              w_globals, flags)
         finally:
             rffi.free_charp(buf)
         w_a = space.getitem(w_globals, space.wrap("a"))
         assert space.unwrap(w_a) == u'caf\xe9'
         lltype.free(flags, flavor='raw')
 
-    def test_run_file(self, space, api):
+    def test_run_file(self, space):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
         fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
-        api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
+        with raises_w(space, ZeroDivisionError):
+            PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
         c_fclose(fp)
-        assert api.PyErr_Occurred() is space.w_ZeroDivisionError
-        api.PyErr_Clear()
 
         # try again, but with a closed file
         fp = c_fopen(str(filepath), "rb")
         os.close(c_fileno(fp))
-        api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
+        with raises_w(space, IOError):
+            PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
         if is_valid_fd(c_fileno(fp)):
             c_fclose(fp)
-        assert api.PyErr_Occurred() is space.w_IOError
-        api.PyErr_Clear()
-
         rffi.free_charp(filename)
 
-    def test_getbuiltins(self, space, api):
-        assert api.PyEval_GetBuiltins() is space.builtin.w_dict
+    def test_getbuiltins(self, space):
+        assert PyEval_GetBuiltins(space) is space.builtin.w_dict
 
         def cpybuiltins(space):
-            return api.PyEval_GetBuiltins()
+            return PyEval_GetBuiltins(space)
         w_cpybuiltins = space.wrap(interp2app(cpybuiltins))
 
         w_result = space.appexec([w_cpybuiltins], """(cpybuiltins):
@@ -168,13 +171,13 @@
         """)
         assert space.len_w(w_result) == 1
 
-    def test_getglobals(self, space, api):
-        assert api.PyEval_GetLocals() is None
-        assert api.PyEval_GetGlobals() is None
+    def test_getglobals(self, space):
+        assert PyEval_GetLocals(space) is None
+        assert PyEval_GetGlobals(space) is None
 
         def cpyvars(space):
-            return space.newtuple([api.PyEval_GetGlobals(),
-                                   api.PyEval_GetLocals()])
+            return space.newtuple([PyEval_GetGlobals(space),
+                                   PyEval_GetLocals(space)])
         w_cpyvars = space.wrap(interp2app(cpyvars))
 
         w_result = space.appexec([w_cpyvars], """(cpyvars):
@@ -186,26 +189,26 @@
         assert sorted(locals) == ['cpyvars', 'x']
         assert sorted(globals) == ['__builtins__', 'anonymous', 'y']
 
-    def test_sliceindex(self, space, api):
+    def test_sliceindex(self, space):
         pi = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
-        assert api._PyEval_SliceIndex(space.w_None, pi) == 0
-        api.PyErr_Clear()
+        with pytest.raises(OperationError):
+            _PyEval_SliceIndex(space, space.w_None, pi)
 
-        assert api._PyEval_SliceIndex(space.wrap(123), pi) == 1
+        assert _PyEval_SliceIndex(space, space.wrap(123), pi) == 1
         assert pi[0] == 123
 
-        assert api._PyEval_SliceIndex(space.wrap(1 << 66), pi) == 1
+        assert _PyEval_SliceIndex(space, space.wrap(1 << 66), pi) == 1
         assert pi[0] == sys.maxint
 
         lltype.free(pi, flavor='raw')
 
-    def test_atexit(self, space, api):
+    def test_atexit(self, space):
         lst = []
         def func():
             lst.append(42)
-        api.Py_AtExit(func)
+        Py_AtExit(space, func)
         cpyext = space.getbuiltinmodule('cpyext')
-        cpyext.shutdown(space) # simulate shutdown
+        cpyext.shutdown(space)  # simulate shutdown
         assert lst == [42]
 
 class AppTestCall(AppTestCpythonExtensionBase):
@@ -269,6 +272,7 @@
                 return res;
              """),
             ])
+
         def f(*args):
             return args
         assert module.call_func(f) == (None,)
@@ -322,7 +326,7 @@
             ])
         assert module.get_flags() == (0, 0)
 
-        ns = {'module':module}
+        ns = {'module': module}
         exec """from __future__ import division    \nif 1:
                 def nested_flags():
                     return module.get_flags()""" in ns
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ b/pypy/module/cpyext/test/test_floatobject.py
@@ -1,36 +1,40 @@
+import pytest
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.floatobject import (
+    PyFloat_FromDouble, PyFloat_AsDouble, PyFloat_AS_DOUBLE, PyNumber_Float,
+    _PyFloat_Unpack4, _PyFloat_Unpack8)
 
 class TestFloatObject(BaseApiTest):
-    def test_floatobject(self, space, api):
-        assert space.unwrap(api.PyFloat_FromDouble(3.14)) == 3.14
-        assert api.PyFloat_AsDouble(space.wrap(23.45)) == 23.45
-        assert api.PyFloat_AS_DOUBLE(space.wrap(23.45)) == 23.45
+    def test_floatobject(self, space):
+        assert space.unwrap(PyFloat_FromDouble(space, 3.14)) == 3.14
+        assert PyFloat_AsDouble(space, space.wrap(23.45)) == 23.45
+        assert PyFloat_AS_DOUBLE(space, space.wrap(23.45)) == 23.45
+        with pytest.raises(OperationError):
+            PyFloat_AsDouble(space, space.w_None)
 
-        assert api.PyFloat_AsDouble(space.w_None) == -1
-        api.PyErr_Clear()
-
-    def test_coerce(self, space, api):
-        assert space.type(api.PyNumber_Float(space.wrap(3))) is space.w_float
-        assert space.type(api.PyNumber_Float(space.wrap("3"))) is space.w_float
+    def test_coerce(self, space):
+        assert space.type(PyNumber_Float(space, space.wrap(3))) is space.w_float
+        assert space.type(PyNumber_Float(space, space.wrap("3"))) is space.w_float
 
         w_obj = space.appexec([], """():
             class Coerce(object):
                 def __float__(self):
                     return 42.5
             return Coerce()""")
-        assert space.eq_w(api.PyNumber_Float(w_obj), space.wrap(42.5))
+        assert space.eq_w(PyNumber_Float(space, w_obj), space.wrap(42.5))
 
-    def test_unpack(self, space, api):
+    def test_unpack(self, space):
         with rffi.scoped_str2charp("\x9a\x99\x99?") as ptr:
-            assert abs(api._PyFloat_Unpack4(ptr, 1) - 1.2) < 1e-7
+            assert abs(_PyFloat_Unpack4(space, ptr, 1) - 1.2) < 1e-7
         with rffi.scoped_str2charp("?\x99\x99\x9a") as ptr:
-            assert abs(api._PyFloat_Unpack4(ptr, 0) - 1.2) < 1e-7
+            assert abs(_PyFloat_Unpack4(space, ptr, 0) - 1.2) < 1e-7
         with rffi.scoped_str2charp("\x1f\x85\xebQ\xb8\x1e\t@") as ptr:
-            assert abs(api._PyFloat_Unpack8(ptr, 1) - 3.14) < 1e-15
+            assert abs(_PyFloat_Unpack8(space, ptr, 1) - 3.14) < 1e-15
         with rffi.scoped_str2charp("@\t\x1e\xb8Q\xeb\x85\x1f") as ptr:
-            assert abs(api._PyFloat_Unpack8(ptr, 0) - 3.14) < 1e-15
+            assert abs(_PyFloat_Unpack8(space, ptr, 0) - 3.14) < 1e-15
 
 class AppTestFloatObject(AppTestCpythonExtensionBase):
     def test_fromstring(self):
@@ -79,8 +83,6 @@
         assert math.isinf(neginf)
 
     def test_macro_accepts_wrong_pointer_type(self):
-        import math
-
         module = self.import_extension('foo', [
             ("test_macros", "METH_NOARGS",
              """
diff --git a/pypy/module/cpyext/test/test_funcobject.py b/pypy/module/cpyext/test/test_funcobject.py
--- a/pypy/module/cpyext/test/test_funcobject.py
+++ b/pypy/module/cpyext/test/test_funcobject.py
@@ -1,16 +1,18 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
+from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, decref
+from pypy.module.cpyext.methodobject import PyClassMethod_New
 from pypy.module.cpyext.funcobject import (
-    PyFunctionObject, PyCodeObject, CODE_FLAGS)
-from pypy.interpreter.function import Function, Method
+    PyFunctionObject, PyCodeObject, CODE_FLAGS, PyMethod_Function,
+    PyMethod_Self, PyMethod_Class, PyMethod_New, PyFunction_GetCode,
+    PyCode_NewEmpty, PyCode_GetNumFree)
+from pypy.interpreter.function import Function
 from pypy.interpreter.pycode import PyCode
 
 globals().update(CODE_FLAGS)
 
 class TestFunctionObject(BaseApiTest):
-    def test_function(self, space, api):
+    def test_function(self, space):
         w_function = space.appexec([], """():
             def f(): pass
             return f
@@ -19,10 +21,10 @@
         assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is
                 space.gettypeobject(Function.typedef))
         assert "f" == space.unwrap(
-           from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name))
-        api.Py_DecRef(ref)
+            from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name))
+        decref(space, ref)
 
-    def test_method(self, space, api):
+    def test_method(self, space):
         w_method = space.appexec([], """():
             class C(list):
                 def method(self): pass
@@ -33,30 +35,30 @@
         w_self = space.getattr(w_method, space.wrap("im_self"))
         w_class = space.getattr(w_method, space.wrap("im_class"))
 
-        assert space.is_w(api.PyMethod_Function(w_method), w_function)
-        assert space.is_w(api.PyMethod_Self(w_method), w_self)
-        assert space.is_w(api.PyMethod_Class(w_method), w_class)
+        assert space.is_w(PyMethod_Function(space, w_method), w_function)
+        assert space.is_w(PyMethod_Self(space, w_method), w_self)
+        assert space.is_w(PyMethod_Class(space, w_method), w_class)
 
-        w_method2 = api.PyMethod_New(w_function, w_self, w_class)
+        w_method2 = PyMethod_New(space, w_function, w_self, w_class)
         assert space.eq_w(w_method, w_method2)
 
-    def test_getcode(self, space, api):
+    def test_getcode(self, space):
         w_function = space.appexec([], """():
             def func(x, y, z): return x
             return func
         """)
-        w_code = api.PyFunction_GetCode(w_function)
+        w_code = PyFunction_GetCode(space, w_function)
         assert w_code.co_name == "func"
 
         ref = make_ref(space, w_code)
         assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is
                 space.gettypeobject(PyCode.typedef))
         assert "func" == space.unwrap(
-           from_ref(space, rffi.cast(PyCodeObject, ref).c_co_name))
+            from_ref(space, rffi.cast(PyCodeObject, ref).c_co_name))
         assert 3 == rffi.cast(PyCodeObject, ref).c_co_argcount
-        api.Py_DecRef(ref)
+        decref(space, ref)
 
-    def test_co_flags(self, space, api):
+    def test_co_flags(self, space):
         def get_flags(signature, body="pass"):
             w_code = space.appexec([], """():
                 def func(%s): %s
@@ -64,7 +66,7 @@
             """ % (signature, body))
             ref = make_ref(space, w_code)
             co_flags = rffi.cast(PyCodeObject, ref).c_co_flags
-            api.Py_DecRef(ref)
+            decref(space, ref)
             return co_flags
         assert get_flags("x") == CO_NESTED | CO_OPTIMIZED | CO_NEWLOCALS
         assert get_flags("x", "exec x") == CO_NESTED | CO_NEWLOCALS
@@ -72,29 +74,29 @@
         assert get_flags("x, **kw") & CO_VARKEYWORDS
         assert get_flags("x", "yield x") & CO_GENERATOR
 
-    def test_newcode(self, space, api):
+    def test_newcode(self, space):
         filename = rffi.str2charp('filename')
         funcname = rffi.str2charp('funcname')
-        w_code = api.PyCode_NewEmpty(filename, funcname, 3)
+        w_code = PyCode_NewEmpty(space, filename, funcname, 3)
         assert w_code.co_filename == 'filename'
         assert w_code.co_firstlineno == 3
 
         ref = make_ref(space, w_code)
         assert "filename" == space.unwrap(
             from_ref(space, rffi.cast(PyCodeObject, ref).c_co_filename))
-        api.Py_DecRef(ref)
+        decref(space, ref)
         rffi.free_charp(filename)
         rffi.free_charp(funcname)
 
-    def test_getnumfree(self, space, api):
+    def test_getnumfree(self, space):
         w_function = space.appexec([], """():
             a = 5
             def method(x): return a, x
             return method
         """)
-        assert api.PyCode_GetNumFree(w_function.code) == 1
+        assert PyCode_GetNumFree(space, w_function.code) == 1
 
-    def test_classmethod(self, space, api):
+    def test_classmethod(self, space):
         w_function = space.appexec([], """():
             def method(x): return x
             return method
@@ -106,6 +108,7 @@
         space.setattr(w_class, space.wrap("method"), w_function)
         assert space.is_w(space.call_method(w_instance, "method"), w_instance)
         # now a classmethod
-        w_classmethod = api.PyClassMethod_New(w_function)
+        w_classmethod = PyClassMethod_New(space, w_function)
         space.setattr(w_class, space.wrap("classmethod"), w_classmethod)
-        assert space.is_w(space.call_method(w_instance, "classmethod"), w_class)
+        assert space.is_w(
+            space.call_method(w_instance, "classmethod"), w_class)
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -1,48 +1,51 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.import_ import *
+from pypy.module.cpyext.import_ import (
+    _PyImport_AcquireLock, _PyImport_ReleaseLock)
 from rpython.rtyper.lltypesystem import rffi
 
 class TestImport(BaseApiTest):
-    def test_import(self, space, api):
-        stat = api.PyImport_Import(space.wrap("stat"))
+    def test_import(self, space):
+        stat = PyImport_Import(space, space.wrap("stat"))
         assert stat
         assert space.getattr(stat, space.wrap("S_IMODE"))
 
-    def test_addmodule(self, space, api):
+    def test_addmodule(self, space):
         with rffi.scoped_str2charp("sys") as modname:
-            w_sys = api.PyImport_AddModule(modname)
+            w_sys = PyImport_AddModule(space, modname)
         assert w_sys is space.sys
 
         with rffi.scoped_str2charp("foobar") as modname:
-            w_foobar = api.PyImport_AddModule(modname)
+            w_foobar = PyImport_AddModule(space, modname)
         assert space.str_w(space.getattr(w_foobar,
                                          space.wrap('__name__'))) == 'foobar'
 
-    def test_getmoduledict(self, space, api):
+    def test_getmoduledict(self, space):
         testmod = "_functools"
-        w_pre_dict = api.PyImport_GetModuleDict()
+        w_pre_dict = PyImport_GetModuleDict(space, )
         assert not space.contains_w(w_pre_dict, space.wrap(testmod))
 
         with rffi.scoped_str2charp(testmod) as modname:
-            w_module = api.PyImport_ImportModule(modname)
+            w_module = PyImport_ImportModule(space, modname)
             print w_module
             assert w_module
 
-        w_dict = api.PyImport_GetModuleDict()
+        w_dict = PyImport_GetModuleDict(space, )
         assert space.contains_w(w_dict, space.wrap(testmod))
 
-    def test_reload(self, space, api):
-        stat = api.PyImport_Import(space.wrap("stat"))
+    def test_reload(self, space):
+        stat = PyImport_Import(space, space.wrap("stat"))
         space.delattr(stat, space.wrap("S_IMODE"))
-        stat = api.PyImport_ReloadModule(stat)
+        stat = PyImport_ReloadModule(space, stat)
         assert space.getattr(stat, space.wrap("S_IMODE"))
 
-    def test_lock(self, space, api):
+    def test_lock(self, space):
         # "does not crash"
-        api._PyImport_AcquireLock()
-        api._PyImport_AcquireLock()
-        api._PyImport_ReleaseLock()
-        api._PyImport_ReleaseLock()
+        _PyImport_AcquireLock(space, )
+        _PyImport_AcquireLock(space, )
+        _PyImport_ReleaseLock(space, )
+        _PyImport_ReleaseLock(space, )
 
 
 class AppTestImportLogic(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py
--- a/pypy/module/cpyext/test/test_intobject.py
+++ b/pypy/module/cpyext/test/test_intobject.py
@@ -1,51 +1,52 @@
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.intobject import (
+    PyInt_Check, PyInt_AsLong, PyInt_AS_LONG, PyInt_FromLong,
+    PyInt_AsUnsignedLong, PyInt_AsUnsignedLongMask,
+    PyInt_AsUnsignedLongLongMask)
 import sys
 
 class TestIntObject(BaseApiTest):
-    def test_intobject(self, space, api):
-        assert api.PyInt_Check(space.wrap(3))
-        assert api.PyInt_Check(space.w_True)
-        assert not api.PyInt_Check(space.wrap((1, 2, 3)))
+    def test_intobject(self, space):
+        assert PyInt_Check(space, space.wrap(3))
+        assert PyInt_Check(space, space.w_True)
+        assert not PyInt_Check(space, space.wrap((1, 2, 3)))

From pypy.commits at gmail.com  Thu Jan 12 09:35:13 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 12 Jan 2017 06:35:13 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Complete the declaration of
 PyTypeObject
Message-ID: <58779421.85e11c0a.638d0.3314@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89508:5897cf94242d
Date: 2016-12-18 02:07 +0000
http://bitbucket.org/pypy/pypy/changeset/5897cf94242d/

Log:	Complete the declaration of PyTypeObject

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -671,7 +671,7 @@
 	PyObject_VAR_HEAD
 } PyVarObject;
 
-typedef struct _typeobject PyTypeObject;
+struct _typeobject;
 
 typedef void (*freefunc)(void *);
 typedef void (*destructor)(PyObject *);
@@ -751,6 +751,178 @@
 typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
 /* end Py3k buffer interface */
 
+typedef int (*objobjproc)(PyObject *, PyObject *);
+typedef int (*visitproc)(PyObject *, void *);
+typedef int (*traverseproc)(PyObject *, visitproc, void *);
+
+typedef struct {
+	/* For numbers without flag bit Py_TPFLAGS_CHECKTYPES set, all
+	   arguments are guaranteed to be of the object's type (modulo
+	   coercion hacks -- i.e. if the type's coercion function
+	   returns other types, then these are allowed as well).  Numbers that
+	   have the Py_TPFLAGS_CHECKTYPES flag bit set should check *both*
+	   arguments for proper type and implement the necessary conversions
+	   in the slot functions themselves. */
+
+	binaryfunc nb_add;
+	binaryfunc nb_subtract;
+	binaryfunc nb_multiply;
+	binaryfunc nb_divide;
+	binaryfunc nb_remainder;
+	binaryfunc nb_divmod;
+	ternaryfunc nb_power;
+	unaryfunc nb_negative;
+	unaryfunc nb_positive;
+	unaryfunc nb_absolute;
+	inquiry nb_nonzero;
+	unaryfunc nb_invert;
+	binaryfunc nb_lshift;
+	binaryfunc nb_rshift;
+	binaryfunc nb_and;
+	binaryfunc nb_xor;
+	binaryfunc nb_or;
+	coercion nb_coerce;
+	unaryfunc nb_int;
+	unaryfunc nb_long;
+	unaryfunc nb_float;
+	unaryfunc nb_oct;
+	unaryfunc nb_hex;
+	/* Added in release 2.0 */
+	binaryfunc nb_inplace_add;
+	binaryfunc nb_inplace_subtract;
+	binaryfunc nb_inplace_multiply;
+	binaryfunc nb_inplace_divide;
+	binaryfunc nb_inplace_remainder;
+	ternaryfunc nb_inplace_power;
+	binaryfunc nb_inplace_lshift;
+	binaryfunc nb_inplace_rshift;
+	binaryfunc nb_inplace_and;
+	binaryfunc nb_inplace_xor;
+	binaryfunc nb_inplace_or;
+
+	/* Added in release 2.2 */
+	/* The following require the Py_TPFLAGS_HAVE_CLASS flag */
+	binaryfunc nb_floor_divide;
+	binaryfunc nb_true_divide;
+	binaryfunc nb_inplace_floor_divide;
+	binaryfunc nb_inplace_true_divide;
+
+	/* Added in release 2.5 */
+	unaryfunc nb_index;
+} PyNumberMethods;
+
+typedef struct {
+	lenfunc sq_length;
+	binaryfunc sq_concat;
+	ssizeargfunc sq_repeat;
+	ssizeargfunc sq_item;
+	ssizessizeargfunc sq_slice;
+	ssizeobjargproc sq_ass_item;
+	ssizessizeobjargproc sq_ass_slice;
+	objobjproc sq_contains;
+	/* Added in release 2.0 */
+	binaryfunc sq_inplace_concat;
+	ssizeargfunc sq_inplace_repeat;
+} PySequenceMethods;
+
+typedef struct {
+	lenfunc mp_length;
+	binaryfunc mp_subscript;
+	objobjargproc mp_ass_subscript;
+} PyMappingMethods;
+
+typedef struct {
+	readbufferproc bf_getreadbuffer;
+	writebufferproc bf_getwritebuffer;
+	segcountproc bf_getsegcount;
+	charbufferproc bf_getcharbuffer;
+	getbufferproc bf_getbuffer;
+	releasebufferproc bf_releasebuffer;
+} PyBufferProcs;
+
+
+
+typedef struct _typeobject {
+	PyObject_VAR_HEAD
+	const char *tp_name; /* For printing, in format "<module>.<name>" */
+	Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
+
+	/* Methods to implement standard operations */
+
+	destructor tp_dealloc;
+	printfunc tp_print;
+	getattrfunc tp_getattr;
+	setattrfunc tp_setattr;
+	cmpfunc tp_compare;
+	reprfunc tp_repr;
+
+	/* Method suites for standard classes */
+
+	PyNumberMethods *tp_as_number;
+	PySequenceMethods *tp_as_sequence;
+	PyMappingMethods *tp_as_mapping;
+
+	/* More standard operations (here for binary compatibility) */
+
+	hashfunc tp_hash;
+	ternaryfunc tp_call;
+	reprfunc tp_str;
+	getattrofunc tp_getattro;
+	setattrofunc tp_setattro;
+
+	/* Functions to access object as input/output buffer */
+	PyBufferProcs *tp_as_buffer;
+
+	/* Flags to define presence of optional/expanded features */
+	long tp_flags;
+
+	const char *tp_doc; /* Documentation string */
+
+	/* Assigned meaning in release 2.0 */
+	/* call function for all accessible objects */
+	traverseproc tp_traverse;
+
+	/* delete references to contained objects */
+	inquiry tp_clear;
+
+	/* Assigned meaning in release 2.1 */
+	/* rich comparisons */
+	richcmpfunc tp_richcompare;
+
+	/* weak reference enabler */
+	Py_ssize_t tp_weaklistoffset;
+
+	/* Added in release 2.2 */
+	/* Iterators */
+	getiterfunc tp_iter;
+	iternextfunc tp_iternext;
+
+	/* Attribute descriptor and subclassing stuff */
+	struct PyMethodDef *tp_methods;
+	struct PyMemberDef *tp_members;
+	struct PyGetSetDef *tp_getset;
+	struct _typeobject *tp_base;
+	PyObject *tp_dict;
+	descrgetfunc tp_descr_get;
+	descrsetfunc tp_descr_set;
+	Py_ssize_t tp_dictoffset;
+	initproc tp_init;
+	allocfunc tp_alloc;
+	newfunc tp_new;
+	freefunc tp_free; /* Low-level free-memory routine */
+	inquiry tp_is_gc; /* For PyObject_IS_GC */
+	PyObject *tp_bases;
+	PyObject *tp_mro; /* method resolution order */
+	PyObject *tp_cache;
+	PyObject *tp_subclasses;
+	PyObject *tp_weaklist;
+	destructor tp_del;
+
+	/* Type attribute cache version tag. Added in version 2.6 */
+	unsigned int tp_version_tag;
+
+} PyTypeObject;
+
 """)
 
 Py_ssize_t = object_h.gettype('Py_ssize_t')
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -235,4 +235,3 @@
     ("tp_weaklist", PyObject),   #U
     ("tp_del", destructor),      #N
     ])
-cpython_struct("PyTypeObject", PyTypeObjectFields, PyTypeObject)

From pypy.commits at gmail.com  Thu Jan 12 09:41:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 06:41:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Skip a test that cannot pass on PyPy
Message-ID: <58779585.4c9d1c0a.982fa.3757@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89511:b84e248dea00
Date: 2017-01-12 15:26 +0100
http://bitbucket.org/pypy/pypy/changeset/b84e248dea00/

Log:	Skip a test that cannot pass on PyPy

diff --git a/lib-python/3/test/test_module.py b/lib-python/3/test/test_module.py
--- a/lib-python/3/test/test_module.py
+++ b/lib-python/3/test/test_module.py
@@ -2,7 +2,7 @@
 import unittest
 import weakref
 from test.support import gc_collect
-from test.support import check_impl_detail
+from test.support import check_impl_detail, impl_detail
 from test.support.script_helper import assert_python_ok
 
 import sys
@@ -217,6 +217,7 @@
         self.assertEqual(r[-len(ends_with):], ends_with,
                          '{!r} does not end with {!r}'.format(r, ends_with))
 
+    @impl_detail(pypy=False)   # __del__ is typically not called at shutdown
     def test_module_finalization_at_shutdown(self):
         # Module globals and builtins should still be available during shutdown
         rc, out, err = assert_python_ok("-c", "from test import final_a")

From pypy.commits at gmail.com  Thu Jan 12 09:41:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 06:41:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <58779587.54161c0a.4c982.3a46@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89512:833aad96c408
Date: 2017-01-12 15:40 +0100
http://bitbucket.org/pypy/pypy/changeset/833aad96c408/

Log:	fix test

diff --git a/pypy/interpreter/test/test_fsencode.py b/pypy/interpreter/test/test_fsencode.py
--- a/pypy/interpreter/test/test_fsencode.py
+++ b/pypy/interpreter/test/test_fsencode.py
@@ -82,4 +82,8 @@
     def test_null_byte(self):
         space = self.space
         w_u = space.newunicode(u'abc\x00def')
-        space.raises_w(space.w_ValueError, space.fsencode, w_u)
+        # this can behave in two different ways depending on how
+        # much initialized the space is: space.fsencode() can raise
+        # ValueError directly, or return a wrapped bytes with the 0
+        # embedded---and then space.fsencode_w() should raise ValueError.
+        space.raises_w(space.w_ValueError, space.fsencode_w, w_u)

From pypy.commits at gmail.com  Thu Jan 12 09:46:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 06:46:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Delay adding __package__,
 __loader__ and __spec__ in a new module's dict
Message-ID: <587796af.e2acc20a.76375.27a9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89513:47871510f0e3
Date: 2017-01-12 15:45 +0100
http://bitbucket.org/pypy/pypy/changeset/47871510f0e3/

Log:	Delay adding __package__, __loader__ and __spec__ in a new module's
	dict to the actual call of __init__()

diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -14,7 +14,7 @@
 
     _frozen = False
 
-    def __init__(self, space, w_name, w_dict=None, add_package=True):
+    def __init__(self, space, w_name, w_dict=None):
         self.space = space
         if w_dict is None:
             w_dict = space.newdict(module=True)
@@ -22,9 +22,6 @@
         self.w_name = w_name
         if w_name is not None:
             space.setitem(w_dict, space.new_interned_str('__name__'), w_name)
-        # add these three attributes always ('add_package' is no longer used)
-        for extra in ['__package__', '__loader__', '__spec__']:
-            space.setitem(w_dict, space.new_interned_str(extra), space.w_None)
         self.startup_called = False
 
     def _cleanup_(self):
@@ -76,7 +73,7 @@
 
     def descr_module__new__(space, w_subtype, __args__):
         module = space.allocate_instance(Module, w_subtype)
-        Module.__init__(module, space, None, add_package=False)
+        Module.__init__(module, space, None)
         return space.wrap(module)
 
     def descr_module__init__(self, w_name, w_doc=None):
@@ -84,8 +81,10 @@
         self.w_name = w_name
         if w_doc is None:
             w_doc = space.w_None
-        space.setitem(self.w_dict, space.new_interned_str('__name__'), w_name)
-        space.setitem(self.w_dict, space.new_interned_str('__doc__'), w_doc)
+        w_dict = self.w_dict
+        space.setitem(w_dict, space.new_interned_str('__name__'), w_name)
+        space.setitem(w_dict, space.new_interned_str('__doc__'), w_doc)
+        init_extra_module_attrs(space, space.wrap(self))
 
     def descr__reduce__(self, space):
         w_name = space.finditem(self.w_dict, space.wrap('__name__'))
@@ -143,3 +142,12 @@
             raise oefmt(space.w_TypeError, "%N.__dict__ is not a dictionary",
                         self)
         return space.call_function(space.w_list, w_dict)
+
+
+def init_extra_module_attrs(space, w_mod):
+    w_dict = w_mod.getdict(space)
+    if w_dict is None:
+        return
+    for extra in ['__package__', '__loader__', '__spec__']:
+        w_attr = space.new_interned_str(extra)
+        space.call_method(w_dict, 'setdefault', w_attr, space.w_None)
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -215,3 +215,8 @@
                               '__package__': None,
                               '__loader__': None,
                               '__spec__': None}
+
+    def test_module_new_makes_empty_dict(self):
+        import sys
+        m = type(sys).__new__(type(sys))
+        assert not m.__dict__
diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py
--- a/pypy/module/_pickle_support/maker.py
+++ b/pypy/module/_pickle_support/maker.py
@@ -27,7 +27,7 @@
     return space.wrap(fu)
 
 def module_new(space, w_name, w_dict):
-    new_mod = Module(space, w_name, w_dict, add_package=False)
+    new_mod = Module(space, w_name, w_dict)
     return space.wrap(new_mod)
 
 def method_new(space, __args__):
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -4,7 +4,7 @@
 
 import sys, os, stat
 
-from pypy.interpreter.module import Module
+from pypy.interpreter.module import Module, init_extra_module_attrs
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, generic_new_descr
 from pypy.interpreter.error import OperationError, oefmt
@@ -113,11 +113,13 @@
     space.setattr(w_mod, w('__doc__'), space.w_None)
     if pkgdir is not None:
         space.setattr(w_mod, w('__path__'), space.newlist([w(pkgdir)]))
+    init_extra_module_attrs(space, w_mod)
 
 def add_module(space, w_name):
     w_mod = check_sys_modules(space, w_name)
     if w_mod is None:
         w_mod = space.wrap(Module(space, w_name))
+        init_extra_module_attrs(space, w_mod)
         space.sys.setmodule(w_mod)
     return w_mod
 

From pypy.commits at gmail.com  Thu Jan 12 10:08:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 07:08:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: - Skip a test that cannot pass on PyPy
Message-ID: <58779c02.068f1c0a.bc3bc.43af@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89514:c217aa239223
Date: 2017-01-12 16:08 +0100
http://bitbucket.org/pypy/pypy/changeset/c217aa239223/

Log:	- Skip a test that cannot pass on PyPy

	- Update a test with its content in the latest 3.5.x version,
	because neither PyPy nor 3.5.2+ passes it

diff --git a/lib-python/3/test/test_traceback.py b/lib-python/3/test/test_traceback.py
--- a/lib-python/3/test/test_traceback.py
+++ b/lib-python/3/test/test_traceback.py
@@ -8,6 +8,7 @@
 import re
 from test import support
 from test.support import TESTFN, Error, captured_output, unlink, cpython_only
+from test.support import impl_detail
 from test.support.script_helper import assert_python_ok
 import textwrap
 
@@ -179,6 +180,7 @@
         # Issue #18960: coding spec should has no effect
         do_test("0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5)
 
+    @impl_detail(pypy=False)   # __del__ is typically not called at shutdown
     def test_print_traceback_at_exit(self):
         # Issue #22599: Ensure that it is possible to use the traceback module
         # to display an exception at Python exit
@@ -681,8 +683,12 @@
 class TestStack(unittest.TestCase):
 
     def test_walk_stack(self):
-        s = list(traceback.walk_stack(None))
-        self.assertGreater(len(s), 10)
+        def deeper():
+            return list(traceback.walk_stack(None))
+        s1 = list(traceback.walk_stack(None))
+        s2 = deeper()
+        self.assertEqual(len(s2) - len(s1), 1)
+        self.assertEqual(s2[1:], s1)
 
     def test_walk_tb(self):
         try:

From pypy.commits at gmail.com  Thu Jan 12 10:13:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 07:13:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: generalize the expected error messages
Message-ID: <58779d35.cb911c0a.8073d.42a8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89515:511e7a4e706b
Date: 2017-01-12 16:13 +0100
http://bitbucket.org/pypy/pypy/changeset/511e7a4e706b/

Log:	generalize the expected error messages

diff --git a/lib-python/3/test/test_syntax.py b/lib-python/3/test/test_syntax.py
--- a/lib-python/3/test/test_syntax.py
+++ b/lib-python/3/test/test_syntax.py
@@ -31,9 +31,9 @@
 Traceback (most recent call last):
 SyntaxError: invalid syntax
 
->>> None = 1
+>>> None = 1                                 # doctest: +ELLIPSIS
 Traceback (most recent call last):
-SyntaxError: can't assign to keyword
+SyntaxError: can't assign to ...
 
 It's a syntax error to assign to the empty tuple.  Why isn't it an
 error to assign to the empty list?  It will always raise some error at
@@ -234,9 +234,9 @@
 >>> (x for x in x) += 1
 Traceback (most recent call last):
 SyntaxError: can't assign to generator expression
->>> None += 1
+>>> None += 1                              # doctest: +ELLIPSIS
 Traceback (most recent call last):
-SyntaxError: can't assign to keyword
+SyntaxError: can't assign to ...
 >>> f() += 1
 Traceback (most recent call last):
 SyntaxError: can't assign to function call
@@ -501,17 +501,17 @@
 
 Corner-cases that used to fail to raise the correct error:
 
-    >>> def f(*, x=lambda __debug__:0): pass
+    >>> def f(*, x=lambda __debug__:0): pass          # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    SyntaxError: assignment to keyword
+    SyntaxError: ...assign... to ...
 
-    >>> def f(*args:(lambda __debug__:0)): pass
+    >>> def f(*args:(lambda __debug__:0)): pass       # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    SyntaxError: assignment to keyword
+    SyntaxError: ...assign... to ...
 
-    >>> def f(**kwargs:(lambda __debug__:0)): pass
+    >>> def f(**kwargs:(lambda __debug__:0)): pass    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    SyntaxError: assignment to keyword
+    SyntaxError: ...assign... to ...
 
     >>> with (lambda *:0): pass
     Traceback (most recent call last):
@@ -519,13 +519,13 @@
 
 Corner-cases that used to crash:
 
-    >>> def f(**__debug__): pass
+    >>> def f(**__debug__): pass                      # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    SyntaxError: assignment to keyword
+    SyntaxError: ...assign... to ...
 
-    >>> def f(*xx, __debug__): pass
+    >>> def f(*xx, __debug__): pass                   # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    SyntaxError: assignment to keyword
+    SyntaxError: ...assign... to ...
 
 """
 

From pypy.commits at gmail.com  Thu Jan 12 10:27:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 07:27:24 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix the warnings in audioop,
 introduced in a recent cffi version
Message-ID: <5877a05c.54b31c0a.20cb6.48a7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89516:532396b48a7f
Date: 2017-01-12 16:26 +0100
http://bitbucket.org/pypy/pypy/changeset/532396b48a7f/

Log:	Fix the warnings in audioop, introduced in a recent cffi version

diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -374,7 +374,7 @@
 
     sample_count = _sample_count(cp, size)
 
-    rv = ffi.new("unsigned char[]", len(cp) * 2)
+    rv = ffi.new("char[]", len(cp) * 2)
     lib.tostereo(rv, cp, len(cp), size, fac1, fac2)
     return ffi.buffer(rv)[:]
 
@@ -385,7 +385,7 @@
     if len(cp1) != len(cp2):
         raise error("Lengths should be the same")
 
-    rv = ffi.new("unsigned char[]", len(cp1))
+    rv = ffi.new("char[]", len(cp1))
     lib.add(rv, cp1, cp2, len(cp1), size)
     return ffi.buffer(rv)[:]
 
@@ -488,7 +488,7 @@
     ceiling = (q + 1) * outrate
     nbytes = ceiling * bytes_per_frame
 
-    rv = ffi.new("unsigned char[]", nbytes)
+    rv = ffi.new("char[]", nbytes)
     trim_index = lib.ratecv(rv, cp, frame_count, size,
                             nchannels, inrate, outrate,
                             state_d, prev_i, cur_i,

From pypy.commits at gmail.com  Thu Jan 12 10:31:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 07:31:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix the hack: it's possible to call
 subprocess.Popen() with
Message-ID: <5877a134.c9b3c20a.7e7ef.375d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89517:8b095830ab66
Date: 2017-01-12 16:30 +0100
http://bitbucket.org/pypy/pypy/changeset/8b095830ab66/

Log:	Fix the hack: it's possible to call subprocess.Popen() with a
	generator or other lazy object as 'args'

diff --git a/lib-python/3/subprocess.py b/lib-python/3/subprocess.py
--- a/lib-python/3/subprocess.py
+++ b/lib-python/3/subprocess.py
@@ -849,7 +849,7 @@
 
         # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
         # match arguments passed by different versions of virtualenv
-        if args[1:] in (
+        if type(args) is list and args[1:] in (
             ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
             ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
              "if sys.version_info[0] == 3:\n"

From pypy.commits at gmail.com  Thu Jan 12 10:56:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 07:56:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: All the tests give a sensible result on
 PyPy too (tested manually in
Message-ID: <5877a710.d3811c0a.187f3.5ea6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89518:51b0c8f1d834
Date: 2017-01-12 16:55 +0100
http://bitbucket.org/pypy/pypy/changeset/51b0c8f1d834/

Log:	All the tests give a sensible result on PyPy too (tested manually in
	py3.5, revision e0ba73be669b, by setting maxDiff=None in the class).
	The details of the outputs differ too much to make it easy to
	generalize the tests to accept both CPython's and PyPy's style. For
	now let's skip the tests on PyPy.

diff --git a/lib-python/3/test/test_faulthandler.py b/lib-python/3/test/test_faulthandler.py
--- a/lib-python/3/test/test_faulthandler.py
+++ b/lib-python/3/test/test_faulthandler.py
@@ -41,6 +41,12 @@
     finally:
         support.unlink(filename)
 
+# NOTE: all the tests give a sensible result on PyPy too (tested
+# manually in py3.5, revision e0ba73be669b, by setting maxDiff=None in
+# the class).  The details of the outputs differ too much to make it
+# easy to generalize the tests to accept both CPython's and PyPy's
+# style.  For now let's skip the tests on PyPy.
+ at support.cpython_only
 class FaultHandlerTests(unittest.TestCase):
     def get_output(self, code, filename=None, fd=None):
         """

From pypy.commits at gmail.com  Thu Jan 12 11:35:05 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 12 Jan 2017 08:35:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: call int() on ffi.cast("intptr_t",
 ...) to ensure string formating does not throw
Message-ID: <5877b039.ce181c0a.8777e.70cc@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89519:a372a26b6ec0
Date: 2017-01-12 17:33 +0100
http://bitbucket.org/pypy/pypy/changeset/a372a26b6ec0/

Log:	call int() on ffi.cast("intptr_t", ...) to ensure string formating
	does not throw

diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -185,7 +185,7 @@
 
     def __repr__(self):
         return "<%s object at 0x%x>" % (
-            self.typename, tkffi.cast("intptr_t", self._value))
+            self.typename, int(tkffi.cast("intptr_t", self._value)))
 
     def __eq__(self, other):
         if not isinstance(other, Tcl_Obj):

From pypy.commits at gmail.com  Thu Jan 12 11:41:54 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 12 Jan 2017 08:41:54 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Add a way to create API functions
 using C declarations; try to use it
Message-ID: <5877b1d2.8c7e1c0a.b09b3.7290@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89520:05b94a8b7e3d
Date: 2017-01-12 16:40 +0000
http://bitbucket.org/pypy/pypy/changeset/05b94a8b7e3d/

Log:	Add a way to create API functions using C declarations; try to use
	it

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -246,13 +246,15 @@
 
 class ApiFunction(object):
     def __init__(self, argtypes, restype, callable, error=CANNOT_FAIL,
-                 c_name=None, gil=None, result_borrowed=False, result_is_ll=False):
+                 c_name=None, cdecl=None, gil=None,
+                 result_borrowed=False, result_is_ll=False):
         self.argtypes = argtypes
         self.restype = restype
         self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
         self.callable = callable
         self.error_value = error
         self.c_name = c_name
+        self.cdecl = cdecl
 
         # extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
@@ -371,6 +373,8 @@
         return unwrapper
 
     def get_c_restype(self, c_writer):
+        if self.cdecl:
+            return self.cdecl.split(self.c_name)[0].strip()
         return c_writer.gettype(self.restype).replace('@', '').strip()
 
     def get_c_args(self, c_writer):
@@ -470,6 +474,20 @@
         return unwrapper
     return decorate
 
+def api_decl(cdecl, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+    def decorate(func):
+        func._always_inline_ = 'try'
+        name, FUNC = cts.parse_func(cdecl)
+        api_function = ApiFunction(
+            FUNC.ARGS, FUNC.RESULT, func,
+            error=_compute_error(error, FUNC.RESULT), cdecl=cdecl)
+        FUNCTIONS_BY_HEADER[header][name] = api_function
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
+
 def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
     def decorate(func):
         func._always_inline_ = 'try'
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,12 +11,10 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr, slot_function, object_h)
+    PyTypeObjectPtr, slot_function, object_h, api_decl)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
-PyCFunction_typedef = rffi.COpaquePtr(typedef='PyCFunction')
-
 PyMethodDef = object_h.gettype('PyMethodDef')
 PyCFunction = object_h.gettype('PyCFunction')
 PyCFunctionKwArgs = object_h.gettype('PyCFunctionWithKeywords')
@@ -284,7 +282,7 @@
 def PyCFunction_NewEx(space, ml, w_self, w_name):
     return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name))
 
- at cpython_api([PyObject], PyCFunction_typedef)
+ at api_decl("PyCFunction PyCFunction_GetFunction(PyObject *)", object_h)
 def PyCFunction_GetFunction(space, w_obj):
     try:
         cfunction = space.interp_w(W_PyCFunctionObject, w_obj)
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py
--- a/pypy/module/cpyext/test/test_methodobject.py
+++ b/pypy/module/cpyext/test/test_methodobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.api import ApiFunction
 from pypy.module.cpyext.pyobject import PyObject, make_ref, Py_DecRef
 from pypy.module.cpyext.methodobject import (
-    PyDescr_NewMethod, PyCFunction_typedef)
+    PyDescr_NewMethod, PyCFunction)
 from rpython.rtyper.lltypesystem import rffi, lltype
 
 class AppTestMethodObject(AppTestCpythonExtensionBase):
@@ -67,7 +67,7 @@
              '''
              PyCFunction ptr = PyCFunction_GetFunction(args);
              if (!ptr) return NULL;
-             if (ptr == MyModule_getarg_O)
+             if (ptr == (PyCFunction)MyModule_getarg_O)
                  Py_RETURN_TRUE;
              else
                  Py_RETURN_FALSE;
@@ -105,8 +105,7 @@
         ml = lltype.malloc(PyMethodDef, flavor='raw', zero=True)
         namebuf = rffi.cast(rffi.CONST_CCHARP, rffi.str2charp('func'))
         ml.c_ml_name = namebuf
-        ml.c_ml_meth = rffi.cast(PyCFunction_typedef,
-                                 c_func.get_llhelper(space))
+        ml.c_ml_meth = rffi.cast(PyCFunction, c_func.get_llhelper(space))
 
         method = api.PyDescr_NewMethod(space.w_str, ml)
         assert repr(method).startswith(
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -19,7 +19,7 @@
     Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
     Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
-    W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
+    W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
 from pypy.module.cpyext.modsupport import convert_method_defs
 from pypy.module.cpyext.pyobject import (
@@ -386,8 +386,7 @@
 
 def setup_new_method_def(space):
     ptr = get_new_method_def(space)
-    ptr.c_ml_meth = rffi.cast(
-        PyCFunction_typedef, llslot(space, tp_new_wrapper))
+    ptr.c_ml_meth = rffi.cast(PyCFunction, llslot(space, tp_new_wrapper))
 
 def add_tp_new_wrapper(space, dict_w, pto):
     if "__new__" in dict_w:

From pypy.commits at gmail.com  Thu Jan 12 12:13:54 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 12 Jan 2017 09:13:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add failing test from stdlib for
 memoryview
Message-ID: <5877b952.46831c0a.8665b.8a9c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89521:de8975b7947d
Date: 2017-01-12 18:13 +0100
http://bitbucket.org/pypy/pypy/changeset/de8975b7947d/

Log:	add failing test from stdlib for memoryview

diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -193,6 +193,12 @@
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
 
+    def test_hex_long(self):
+        x = b'0' * 200000
+        m1 = memoryview(x)
+        m2 = m1[::-1]
+        assert m2.hex() == '30' * 200000
+
     def test_memoryview_cast(self):
         m1 = memoryview(b'abcdefgh')
         m2 = m1.cast('I')

From pypy.commits at gmail.com  Thu Jan 12 13:26:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 10:26:38 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: obscure fixes for test___all__.py
Message-ID: <5877ca5e.ce841c0a.c6cd9.bd4e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89522:048d4b329858
Date: 2017-01-12 18:07 +0100
http://bitbucket.org/pypy/pypy/changeset/048d4b329858/

Log:	obscure fixes for test___all__.py

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -220,4 +220,9 @@
 for constant in dir(os):
     value = getattr(os, constant)
     if constant.isupper() and type(value) is int:
+        if constant in ['SEEK_SET', 'SEEK_CUR', 'SEEK_END',
+                        'P_NOWAIT', 'P_NOWAITO', 'P_WAIT']:
+            # obscure, but these names are not in CPython's posix module
+            # and if we put it here then they end up twice in 'os.__all__'
+            continue
         Module.interpleveldefs[constant] = "space.wrap(%s)" % value
diff --git a/pypy/module/token/__init__.py b/pypy/module/token/__init__.py
--- a/pypy/module/token/__init__.py
+++ b/pypy/module/token/__init__.py
@@ -22,6 +22,11 @@
     Module.interpleveldefs["tok_name"] = "space.wrap(%r)" % (tok_name,)
     Module.interpleveldefs["N_TOKENS"] = "space.wrap(%d)" % len(tok_name)
     all_names = Module.interpleveldefs.keys()
+    # obscure, but these names are not in CPython's token module
+    # so we remove them from 'token.__all__' otherwise they end up
+    # twice in 'tokenize.__all__'
+    all_names.remove('COMMENT')
+    all_names.remove('NL')
     Module.interpleveldefs["__all__"] = "space.wrap(%r)" % (all_names,)
 
 _init_tokens()

From pypy.commits at gmail.com  Thu Jan 12 13:26:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 10:26:40 -0800 (PST)
Subject: [pypy-commit] pypy default: replace commenting-out-some-lines with
 check_impl_detail()
Message-ID: <5877ca60.12ad1c0a.5434f.c1c8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89523:4b05cf3a2baf
Date: 2017-01-12 18:15 +0100
http://bitbucket.org/pypy/pypy/changeset/4b05cf3a2baf/

Log:	replace commenting-out-some-lines with check_impl_detail()

diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py
--- a/lib-python/2.7/test/test_multiprocessing.py
+++ b/lib-python/2.7/test/test_multiprocessing.py
@@ -1969,9 +1969,10 @@
         if not gc.isenabled():
             gc.enable()
             self.addCleanup(gc.disable)
-        #thresholds = gc.get_threshold()
-        #self.addCleanup(gc.set_threshold, *thresholds)
-        #gc.set_threshold(10)
+        if test_support.check_impl_detail(cpython=True):
+            thresholds = gc.get_threshold()
+            self.addCleanup(gc.set_threshold, *thresholds)
+            gc.set_threshold(10)
 
         # perform numerous block allocations, with cyclic references to make
         # sure objects are collected asynchronously by the gc

From pypy.commits at gmail.com  Thu Jan 12 13:26:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 10:26:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: The T_XXX assignments now do range
 checking and raise a warning
Message-ID: <5877ca65.068f1c0a.bc3bc.bd60@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89525:f7ca99898732
Date: 2017-01-12 19:01 +0100
http://bitbucket.org/pypy/pypy/changeset/f7ca99898732/

Log:	The T_XXX assignments now do range checking and raise a warning

diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py
--- a/pypy/module/cpyext/structmember.py
+++ b/pypy/module/cpyext/structmember.py
@@ -20,21 +20,21 @@
         return True
     raise oefmt(space.w_TypeError, "attribute value type must be bool")
 
-integer_converters = unrolling_iterable([
-    (T_SHORT,  rffi.SHORT,  PyLong_AsLong),
-    (T_INT,    rffi.INT,    PyLong_AsLong),
-    (T_LONG,   rffi.LONG,   PyLong_AsLong),
-    (T_USHORT, rffi.USHORT, PyLong_AsUnsignedLong),
-    (T_UINT,   rffi.UINT,   PyLong_AsUnsignedLong),
-    (T_ULONG,  rffi.ULONG,  PyLong_AsUnsignedLong),
-    (T_BYTE,   rffi.SIGNEDCHAR, PyLong_AsLong),
-    (T_UBYTE,  rffi.UCHAR,  PyLong_AsUnsignedLong),
-    (T_BOOL,   rffi.UCHAR,  convert_bool),
-    (T_FLOAT,  rffi.FLOAT,  PyFloat_AsDouble),
-    (T_DOUBLE, rffi.DOUBLE, PyFloat_AsDouble),
-    (T_LONGLONG,  rffi.LONGLONG,  PyLong_AsLongLong),
-    (T_ULONGLONG, rffi.ULONGLONG, PyLong_AsUnsignedLongLong),
-    (T_PYSSIZET, rffi.SSIZE_T, PyLong_AsSsize_t),
+integer_converters = unrolling_iterable([                     # range checking
+    (T_SHORT,  rffi.SHORT,  PyLong_AsLong,                    True),
+    (T_INT,    rffi.INT,    PyLong_AsLong,                    True),
+    (T_LONG,   rffi.LONG,   PyLong_AsLong,                    False),
+    (T_USHORT, rffi.USHORT, PyLong_AsUnsignedLong,            True),
+    (T_UINT,   rffi.UINT,   PyLong_AsUnsignedLong,            True),
+    (T_ULONG,  rffi.ULONG,  PyLong_AsUnsignedLong,            False),
+    (T_BYTE,   rffi.SIGNEDCHAR, PyLong_AsLong,                True),
+    (T_UBYTE,  rffi.UCHAR,  PyLong_AsUnsignedLong,            True),
+    (T_BOOL,   rffi.UCHAR,  convert_bool,                     False),
+    (T_FLOAT,  rffi.FLOAT,  PyFloat_AsDouble,                 False),
+    (T_DOUBLE, rffi.DOUBLE, PyFloat_AsDouble,                 False),
+    (T_LONGLONG,  rffi.LONGLONG,  PyLong_AsLongLong,          False),
+    (T_ULONGLONG, rffi.ULONGLONG, PyLong_AsUnsignedLongLong,  False),
+    (T_PYSSIZET, rffi.SSIZE_T, PyLong_AsSsize_t,              False),
     ])
 
 _HEADER = 'pypy_structmember_decl.h'
@@ -47,7 +47,7 @@
 
     member_type = rffi.cast(lltype.Signed, w_member.c_type)
     for converter in integer_converters:
-        typ, lltyp, _ = converter
+        typ, lltyp, _, _ = converter
         if typ == member_type:
             result = rffi.cast(rffi.CArrayPtr(lltyp), addr)
             if lltyp is rffi.FLOAT:
@@ -112,11 +112,16 @@
                         "can't delete numeric/char attribute")
 
     for converter in integer_converters:
-        typ, lltyp, getter = converter
+        typ, lltyp, getter, range_checking = converter
         if typ == member_type:
             value = getter(space, w_value)
             array = rffi.cast(rffi.CArrayPtr(lltyp), addr)
-            array[0] = rffi.cast(lltyp, value)
+            casted = rffi.cast(lltyp, value)
+            if range_checking:
+                if rffi.cast(lltype.typeOf(value), casted) != value:
+                    space.warn(space.wrap("structmember: truncation of value"),
+                               space.w_RuntimeWarning)
+            array[0] = casted
             return 0
 
     if member_type == T_CHAR:

From pypy.commits at gmail.com  Thu Jan 12 13:26:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 10:26:47 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Update the version numbers to 3.5.2,
 after 215771f42f81
Message-ID: <5877ca67.0d1a1c0a.a3449.c25c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89526:dbe5313227fc
Date: 2017-01-12 19:19 +0100
http://bitbucket.org/pypy/pypy/changeset/dbe5313227fc/

Log:	Update the version numbers to 3.5.2, after 215771f42f81

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -21,12 +21,12 @@
 /* Version parsed out into numeric values */
 #define PY_MAJOR_VERSION	3
 #define PY_MINOR_VERSION	5
-#define PY_MICRO_VERSION	1
+#define PY_MICRO_VERSION	2
 #define PY_RELEASE_LEVEL	PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL	0
 
 /* Version as a string */
-#define PY_VERSION		"3.5.1"
+#define PY_VERSION		"3.5.2"
 
 /* PyPy version as a string */
 #define PYPY_VERSION "5.6.0-alpha0"
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -6,7 +6,7 @@
 from pypy.interpreter import gateway
 
 #XXX # the release serial 42 is not in range(16)
-CPYTHON_VERSION            = (3, 5, 1, "final", 0)
+CPYTHON_VERSION            = (3, 5, 2, "final", 0)
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 

From pypy.commits at gmail.com  Thu Jan 12 13:26:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 10:26:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <5877ca69.92ae1c0a.f8bcd.c00f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89527:daf8de0e7dab
Date: 2017-01-12 19:25 +0100
http://bitbucket.org/pypy/pypy/changeset/daf8de0e7dab/

Log:	merge heads

diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -185,7 +185,7 @@
 
     def __repr__(self):
         return "<%s object at 0x%x>" % (
-            self.typename, tkffi.cast("intptr_t", self._value))
+            self.typename, int(tkffi.cast("intptr_t", self._value)))
 
     def __eq__(self, other):
         if not isinstance(other, Tcl_Obj):
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -193,6 +193,12 @@
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
 
+    def test_hex_long(self):
+        x = b'0' * 200000
+        m1 = memoryview(x)
+        m2 = m1[::-1]
+        assert m2.hex() == '30' * 200000
+
     def test_memoryview_cast(self):
         m1 = memoryview(b'abcdefgh')
         m2 = m1.cast('I')

From pypy.commits at gmail.com  Thu Jan 12 13:26:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 10:26:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix (without test,
 because there doesn't seem to be any test for the
Message-ID: <5877ca63.07941c0a.591bd.b957@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89524:24048a09826f
Date: 2017-01-12 18:26 +0100
http://bitbucket.org/pypy/pypy/changeset/24048a09826f/

Log:	Fix (without test, because there doesn't seem to be any test for the
	whole module; relying here on lib-
	python/3/test/test_structmembers.py)

diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py
--- a/pypy/module/cpyext/structmember.py
+++ b/pypy/module/cpyext/structmember.py
@@ -13,6 +13,13 @@
 from pypy.module.cpyext.typeobjectdefs import PyMemberDef
 from rpython.rlib.unroll import unrolling_iterable
 
+def convert_bool(space, w_obj):
+    if space.is_w(w_obj, space.w_False):
+        return False
+    if space.is_w(w_obj, space.w_True):
+        return True
+    raise oefmt(space.w_TypeError, "attribute value type must be bool")
+
 integer_converters = unrolling_iterable([
     (T_SHORT,  rffi.SHORT,  PyLong_AsLong),
     (T_INT,    rffi.INT,    PyLong_AsLong),
@@ -22,7 +29,7 @@
     (T_ULONG,  rffi.ULONG,  PyLong_AsUnsignedLong),
     (T_BYTE,   rffi.SIGNEDCHAR, PyLong_AsLong),
     (T_UBYTE,  rffi.UCHAR,  PyLong_AsUnsignedLong),
-    (T_BOOL,   rffi.UCHAR,  PyLong_AsLong),
+    (T_BOOL,   rffi.UCHAR,  convert_bool),
     (T_FLOAT,  rffi.FLOAT,  PyFloat_AsDouble),
     (T_DOUBLE, rffi.DOUBLE, PyFloat_AsDouble),
     (T_LONGLONG,  rffi.LONGLONG,  PyLong_AsLongLong),

From pypy.commits at gmail.com  Thu Jan 12 14:11:52 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 12 Jan 2017 11:11:52 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Skip part of the test when
 untranslated (because something something ctypes).
Message-ID: <5877d4f8.05a81c0a.d79d0.dc06@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89528:ccc4bbdd1a66
Date: 2017-01-12 19:10 +0000
http://bitbucket.org/pypy/pypy/changeset/ccc4bbdd1a66/

Log:	Skip part of the test when untranslated (because something something
	ctypes).

diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -67,7 +67,7 @@
             raise oefmt(space.w_TypeError,
                         "%s() takes no keyword arguments", self.name)
 
-        func = rffi.cast(PyCFunction, self.ml.c_ml_meth)
+        func = self.ml.c_ml_meth
         length = space.int_w(space.len(w_args))
         if flags & METH_KEYWORDS:
             func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth)
@@ -337,4 +337,3 @@
     if name == "__methods__":
         return space.newlist(method_list_w)
     raise OperationError(space.w_AttributeError, space.wrap(name))
-
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py
--- a/pypy/module/cpyext/test/test_methodobject.py
+++ b/pypy/module/cpyext/test/test_methodobject.py
@@ -89,7 +89,8 @@
 
         assert mod.isCFunction(mod.getarg_O) == "getarg_O"
         assert mod.getModule(mod.getarg_O) == 'MyModule'
-        assert mod.isSameFunction(mod.getarg_O)
+        if self.runappdirect:  # XXX: fails untranslated
+            assert mod.isSameFunction(mod.getarg_O)
         raises(SystemError, mod.isSameFunction, 1)
 
 class TestPyCMethodObject(BaseApiTest):

From pypy.commits at gmail.com  Thu Jan 12 16:27:12 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 12 Jan 2017 13:27:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Skip this cElementTree test like all
 others in this file.
Message-ID: <5877f4b0.c4811c0a.8a9b8.39d1@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r89529:056552e32143
Date: 2016-12-07 16:36 +0000
http://bitbucket.org/pypy/pypy/changeset/056552e32143/

Log:	Skip this cElementTree test like all others in this file.

diff --git a/lib-python/3/test/test_xml_etree_c.py b/lib-python/3/test/test_xml_etree_c.py
--- a/lib-python/3/test/test_xml_etree_c.py
+++ b/lib-python/3/test/test_xml_etree_c.py
@@ -11,6 +11,7 @@
                                 fresh=['_elementtree', 'xml.etree'])
 
 
+ at unittest.skipUnless(cET, 'requires _elementtree')
 class MiscTests(unittest.TestCase):
     # Issue #8651.
     @support.bigmemtest(size=support._2G + 100, memuse=1, dry_run=False)

From pypy.commits at gmail.com  Thu Jan 12 16:27:14 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 12 Jan 2017 13:27:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: This is how CPython reports all
 assignments to reserved names.
Message-ID: <5877f4b2.d4051c0a.c77d6.4101@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r89530:a1c35f4c589a
Date: 2017-01-12 22:16 +0100
http://bitbucket.org/pypy/pypy/changeset/a1c35f4c589a/

Log:	This is how CPython reports all assignments to reserved names. (see
	test_syntax.py)

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -126,7 +126,7 @@
         try:
             misc.check_forbidden_name(name)
         except misc.ForbiddenNameAssignment as e:
-            self.error("cannot assign to %s" % (e.name,), node)
+            self.error("assignment to keyword", node)
 
     def new_identifier(self, name):
         return misc.new_identifier(self.space, name)
@@ -138,7 +138,7 @@
         except ast.UnacceptableExpressionContext as e:
             self.error_ast(e.msg, e.node)
         except misc.ForbiddenNameAssignment as e:
-            self.error_ast("cannot assign to %s" % (e.name,), e.node)
+            self.error_ast("assignment to keyword", e.node)
 
     def handle_del_stmt(self, del_node):
         targets = self.handle_exprlist(del_node.get_child(1), ast.Del)
diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py
--- a/pypy/interpreter/astcompiler/asthelpers.py
+++ b/pypy/interpreter/astcompiler/asthelpers.py
@@ -182,5 +182,5 @@
 
 class __extend__(ast.NameConstant):
 
-    _description = "name constant"
+    _description = "keyword"
     constant = True
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -812,7 +812,7 @@
             for template in invalid:
                 input = template % (name,)
                 exc = py.test.raises(SyntaxError, self.get_ast, input).value
-                assert exc.msg == "cannot assign to %s" % (name,)
+                assert exc.msg == "assignment to keyword"
 
     def test_lambda(self):
         lam = self.get_first_expr("lambda x: expr")

From pypy.commits at gmail.com  Thu Jan 12 16:27:16 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 12 Jan 2017 13:27:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: pyclbr analyses the source code,
 and does not return aliased and renamed items.
Message-ID: <5877f4b4.09bb1c0a.9b3e7.33a0@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r89531:da0283b19e53
Date: 2017-01-12 22:06 +0100
http://bitbucket.org/pypy/pypy/changeset/da0283b19e53/

Log:	pyclbr analyses the source code, and does not return aliased and
	renamed items. This happens for pickle.py, when the _pickle module
	is not available.

diff --git a/lib-python/3/test/test_pyclbr.py b/lib-python/3/test/test_pyclbr.py
--- a/lib-python/3/test/test_pyclbr.py
+++ b/lib-python/3/test/test_pyclbr.py
@@ -125,7 +125,10 @@
                     raise
 
         # Now check for missing stuff.
-        def defined_in(item, module):
+        def defined_in(item, module, name):
+            if item.__name__ != name:
+                # Item was defined with another name
+                return False
             if isinstance(item, type):
                 return item.__module__ == module.__name__
             if isinstance(item, FunctionType):
@@ -134,7 +137,7 @@
         for name in dir(module):
             item = getattr(module, name)
             if isinstance(item,  (type, FunctionType)):
-                if defined_in(item, module):
+                if defined_in(item, module, name):
                     self.assertHaskey(dict, name, ignore)
 
     def test_easy(self):

From pypy.commits at gmail.com  Thu Jan 12 17:04:45 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 12 Jan 2017 14:04:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg backout a1c35f4c589a
Message-ID: <5877fd7d.52301c0a.eb11b.4fa8@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r89532:056335ce40d8
Date: 2017-01-12 23:03 +0100
http://bitbucket.org/pypy/pypy/changeset/056335ce40d8/

Log:	hg backout a1c35f4c589a

	The test was fixed in 511e7a4e706b

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -126,7 +126,7 @@
         try:
             misc.check_forbidden_name(name)
         except misc.ForbiddenNameAssignment as e:
-            self.error("assignment to keyword", node)
+            self.error("cannot assign to %s" % (e.name,), node)
 
     def new_identifier(self, name):
         return misc.new_identifier(self.space, name)
@@ -138,7 +138,7 @@
         except ast.UnacceptableExpressionContext as e:
             self.error_ast(e.msg, e.node)
         except misc.ForbiddenNameAssignment as e:
-            self.error_ast("assignment to keyword", e.node)
+            self.error_ast("cannot assign to %s" % (e.name,), e.node)
 
     def handle_del_stmt(self, del_node):
         targets = self.handle_exprlist(del_node.get_child(1), ast.Del)
diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py
--- a/pypy/interpreter/astcompiler/asthelpers.py
+++ b/pypy/interpreter/astcompiler/asthelpers.py
@@ -182,5 +182,5 @@
 
 class __extend__(ast.NameConstant):
 
-    _description = "keyword"
+    _description = "name constant"
     constant = True
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -812,7 +812,7 @@
             for template in invalid:
                 input = template % (name,)
                 exc = py.test.raises(SyntaxError, self.get_ast, input).value
-                assert exc.msg == "assignment to keyword"
+                assert exc.msg == "cannot assign to %s" % (name,)
 
     def test_lambda(self):
         lam = self.get_first_expr("lambda x: expr")

From pypy.commits at gmail.com  Thu Jan 12 17:13:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 14:13:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Test and fix for an issue with braces
 inside brackets
Message-ID: <5877ff9a.810b1c0a.e7074.5c49@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89533:ec2d5317659d
Date: 2017-01-12 23:13 +0100
http://bitbucket.org/pypy/pypy/changeset/ec2d5317659d/

Log:	Test and fix for an issue with braces inside brackets

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -122,6 +122,11 @@
                             nested -= 1
                             if not nested:
                                 break
+                        elif c == "[":
+                            i += 1
+                            while i < end and s[i] != "]":
+                                i += 1
+                            continue
                         i += 1
                     if nested:
                         raise oefmt(space.w_ValueError, "Unmatched '{'")
@@ -214,7 +219,10 @@
                 try:
                     w_arg = self.args[index]
                 except IndexError:
-                    raise oefmt(space.w_IndexError, "out of range")
+                    raise oefmt(space.w_IndexError,
+                                "out of range: index %d but only %d argument%s",
+                                index, len(self.args),
+                                "s" if len(self.args) != 1 else "")
             return self._resolve_lookups(w_arg, name, i, end)
 
         @jit.unroll_safe
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -191,6 +191,11 @@
         assert self.s('{0:\x00<12}').format(3+2.0j) == '(3+2j)' + '\x00' * 6
         assert self.s('{0:\x01<12}').format(3+2.0j) == '(3+2j)' + '\x01' * 6
 
+    def test_more_indexing_cases(self):
+        assert self.s('x{[3]}y').format(['a', 'b', 'c', 'd', 'e']) == 'xdy'
+        assert self.s('x{[[]}y').format({'[': 'a'}) == 'xay'
+        assert self.s('x{[{]}y').format({'{': 'a'}) == 'xay'
+
 
 class AppTestUnicodeFormat(BaseStringFormatTests):
     def setup_class(cls):

From pypy.commits at gmail.com  Thu Jan 12 17:52:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 14:52:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: More tests and fixes
Message-ID: <587808ad.cb911c0a.8073d.752f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89534:ce9b5b7a5f2d
Date: 2017-01-12 23:51 +0100
http://bitbucket.org/pypy/pypy/changeset/ce9b5b7a5f2d/

Log:	More tests and fixes

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -165,6 +165,12 @@
                         conversion = None
                         i += 1
                     return s[start:end_name], conversion, i
+                elif c == "[":
+                    while i + 1 < end and s[i + 1] != "]":
+                        i += 1
+                elif c == "{":
+                    raise oefmt(self.space.w_ValueError,
+                                "unexpected '{' in field name")
                 i += 1
             return s[start:end], None, end
 
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -195,6 +195,11 @@
         assert self.s('x{[3]}y').format(['a', 'b', 'c', 'd', 'e']) == 'xdy'
         assert self.s('x{[[]}y').format({'[': 'a'}) == 'xay'
         assert self.s('x{[{]}y').format({'{': 'a'}) == 'xay'
+        assert self.s("x{[:]}y").format({":" : "a"}) == "xay"
+        assert self.s("x{[!]}y").format({"!" : "a"}) == "xay"
+        raises(ValueError, self.s("{a{}b}").format, 42)
+        raises(ValueError, self.s("{a{b}").format, 42)
+        raises(ValueError, self.s("{[}").format, 42)
 
 
 class AppTestUnicodeFormat(BaseStringFormatTests):

From pypy.commits at gmail.com  Thu Jan 12 18:27:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 12 Jan 2017 15:27:14 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix doc
Message-ID: <587810d2.810b1c0a.e7074.8c0d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89535:da438adf8c4f
Date: 2017-01-13 00:26 +0100
http://bitbucket.org/pypy/pypy/changeset/da438adf8c4f/

Log:	Fix doc

diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -336,6 +336,6 @@
    that fixes some bugs and is translatable.
 
 *  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the files ``xxxtype.py`` and
-   ``xxxobject.py`` contain respectively the definition of the type and its
-   (default) implementation.
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)

From pypy.commits at gmail.com  Fri Jan 13 03:27:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 00:27:52 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58788f88.05a81c0a.fba8d.2582@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r844:00513eb0a6f7
Date: 2017-01-13 09:27 +0100
http://bitbucket.org/pypy/pypy.org/changeset/00513eb0a6f7/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66521 of $105000 (63.4%)
+   $66541 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2357 left
+    we have $2373 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $59030 of $80000 (73.8%)
+   $59040 of $80000 (73.8%)
    <div id="progressbar">
    </div>
 
@@ -29,7 +29,7 @@
   
   <!--
   Current status:
-    we have $48221 left
+    we have $48229 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Fri Jan 13 03:47:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 00:47:47 -0800 (PST)
Subject: [pypy-commit] pypy default: Revert the change in the docstrings
 (tests fail, and anyway it's better after all)
Message-ID: <58789433.878f1c0a.ab837.36cd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89536:eb42c935cc8d
Date: 2017-01-13 09:47 +0100
http://bitbucket.org/pypy/pypy/changeset/eb42c935cc8d/

Log:	Revert the change in the docstrings (tests fail, and anyway it's
	better after all)

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -483,7 +483,7 @@
     return lifeline.get_any_weakref(space)
 
 dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
-                            doc="dictionary for instance variables")
+                            doc="dictionary for instance variables (if defined)")
 dict_descr.name = '__dict__'
 
 
@@ -515,7 +515,7 @@
     return space.newtuple([w_docstring])
 
 weakref_descr = GetSetProperty(descr_get_weakref,
-                               doc="list of weak references to the object")
+                    doc="list of weak references to the object (if defined)")
 weakref_descr.name = '__weakref__'
 
 def make_weakref_descr(cls):

From pypy.commits at gmail.com  Fri Jan 13 05:02:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 02:02:55 -0800 (PST)
Subject: [pypy-commit] cffi default: Use __sync_synchronize() only if
 setup.py detects we have it. Fix for
Message-ID: <5878a5cf.09bb1c0a.27a9b.5b69@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2857:018427a42764
Date: 2017-01-13 11:02 +0100
http://bitbucket.org/cffi/cffi/changeset/018427a42764/

Log:	Use __sync_synchronize() only if setup.py detects we have it. Fix
	for people using an incredibly old gcc.

diff --git a/c/call_python.c b/c/call_python.c
--- a/c/call_python.c
+++ b/c/call_python.c
@@ -177,7 +177,7 @@
 #if (defined(WITH_THREAD) && !defined(_MSC_VER) &&   \
      !defined(__amd64__) && !defined(__x86_64__) &&   \
      !defined(__i386__) && !defined(__i386))
-# if defined(__GNUC__)
+# if defined(HAVE_SYNC_SYNCHRONIZE)
 #   define read_barrier()  __sync_synchronize()
 # elif defined(_AIX)
 #   define read_barrier()  __lwsync()
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,22 @@
         if not ok1:
             no_working_compiler_found()
         sys.stderr.write("Note: will not use '__thread' in the C code\n")
-        sys.stderr.write("The above error message can be safely ignored\n")
+        _safe_to_ignore()
+
+def ask_supports_sync_synchronize():
+    if sys.platform == 'win32':
+        return
+    config = get_config()
+    ok = config.try_link('int main(void) { __sync_synchronize(); return 0; }')
+    if ok:
+        define_macros.append(('HAVE_SYNC_SYNCHRONIZE', None))
+    else:
+        sys.stderr.write("Note: will not use '__sync_synchronize()'"
+                         " in the C code\n")
+        _safe_to_ignore()
+
+def _safe_to_ignore():
+    sys.stderr.write("***** The above error message can be safely ignored.\n\n")
 
 def uses_msvc():
     config = get_config()
@@ -118,6 +133,7 @@
 else:
     use_pkg_config()
     ask_supports_thread()
+    ask_supports_sync_synchronize()
 
 if 'freebsd' in sys.platform:
     include_dirs.append('/usr/local/include')

From pypy.commits at gmail.com  Fri Jan 13 06:10:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 03:10:56 -0800 (PST)
Subject: [pypy-commit] pypy default: Found another way to fix
 http://bugs.python.org/issue29006 in PyPy:
Message-ID: <5878b5c0.068f1c0a.5f5f0.8557@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89537:235e8a388979
Date: 2017-01-13 12:10 +0100
http://bitbucket.org/pypy/pypy/changeset/235e8a388979/

Log:	Found another way to fix http://bugs.python.org/issue29006 in PyPy:
	remember which exact statements were open during the previous
	commit, and if we get SQLITE_LOCKED we close these ones

diff --git a/lib-python/2.7/sqlite3/test/regression.py b/lib-python/2.7/sqlite3/test/regression.py
--- a/lib-python/2.7/sqlite3/test/regression.py
+++ b/lib-python/2.7/sqlite3/test/regression.py
@@ -351,10 +351,7 @@
         self.assertRaises(ValueError, cur.execute, " \0select 2")
         self.assertRaises(ValueError, cur.execute, "select 2\0")
 
-    @test_support.impl_detail(pypy=False)
     def CheckCommitCursorReset(self):
-        # This test is for logic added in 2.7.13 which PyPy doesn't
-        # implement.  See http://bugs.python.org/issue29006
         """
         Connection.commit() did reset cursors, which made sqlite3
         to return rows multiple times when fetched from cursors
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -220,6 +220,7 @@
         self.__statements_counter = 0
         self.__rawstatements = set()
         self._statement_cache = _StatementCache(self, cached_statements)
+        self.__statements_already_committed = weakref.WeakSet()
 
         self.__func_cache = {}
         self.__aggregates = {}
@@ -363,6 +364,12 @@
                 if cursor is not None:
                     cursor._reset = True
 
+    def _reset_already_committed_statements(self):
+        lst = list(self.__statements_already_committed)
+        self.__statements_already_committed.clear()
+        for statement in lst:
+            statement._reset()
+
     @_check_thread_wrap
     @_check_closed_wrap
     def __call__(self, sql):
@@ -418,15 +425,21 @@
         if not self._in_transaction:
             return
 
-        # The following line is a KNOWN DIFFERENCE with CPython 2.7.13.
-        # More precisely, the corresponding line was removed in the
-        # version 2.7.13 of CPython, but this is causing troubles for
-        # PyPy (and potentially for CPython too):
-        #
-        #     http://bugs.python.org/issue29006
-        #
-        # So for now, we keep this line.
-        self.__do_all_statements(Statement._reset, False)
+        # PyPy fix for non-refcounting semantics: since 2.7.13 (and in
+        # <= 2.6.x), the statements are not automatically reset upon
+        # commit.  However, if this is followed by some specific SQL
+        # operations like "drop table", these open statements come in
+        # the way and cause the "drop table" to fail.  On CPython the
+        # problem is much less important because typically all the old
+        # statements are freed already by reference counting.  So here,
+        # we add all the still-alive statements to a WeakSet which is
+        # usually ignored, except if we get SQLITE_LOCKED
+        # afterwards---at which point we reset all statements in this
+        # WeakSet.
+        for weakref in self.__statements:
+            statement = weakref()
+            if statement is not None:
+                self.__statements_already_committed.add(statement)
 
         statement_star = _ffi.new('sqlite3_stmt **')
         ret = _lib.sqlite3_prepare_v2(self._db, b"COMMIT", -1,
@@ -827,8 +840,18 @@
                 self.__statement._set_params(params)
 
                 # Actually execute the SQL statement
+
                 ret = _lib.sqlite3_step(self.__statement._statement)
 
+                # PyPy: if we get SQLITE_LOCKED, it's probably because
+                # one of the cursors created previously is still alive
+                # and not reset and the operation we're trying to do
+                # makes Sqlite unhappy about that.  In that case, we
+                # automatically reset all old cursors and try again.
+                if ret == _lib.SQLITE_LOCKED:
+                    self.__connection._reset_already_committed_statements()
+                    ret = _lib.sqlite3_step(self.__statement._statement)
+
                 if ret == _lib.SQLITE_ROW:
                     if multiple:
                         raise ProgrammingError("executemany() can only execute DML statements.")
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -483,11 +483,6 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* The ``sqlite`` module was updated on 2.7.13 to no longer reset all
-  cursors when there is a commit.  This causes troubles for PyPy (and
-  potentially for CPython too), and so for now we didn't port this change:
-  see http://bugs.python.org/issue29006.
-
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/

From pypy.commits at gmail.com  Fri Jan 13 06:55:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 03:55:49 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix for rgc.move_out_of_nursery() in
 very old GCs like "semispace"
Message-ID: <5878c045.145e1c0a.eb01a.a01c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89538:df8618e832de
Date: 2017-01-13 12:55 +0100
http://bitbucket.org/pypy/pypy/changeset/df8618e832de/

Log:	Fix for rgc.move_out_of_nursery() in very old GCs like "semispace"

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1601,6 +1601,8 @@
                                       resulttype=llmemory.Address)
             hop.genop("cast_adr_to_ptr", [v_ret],
                       resultvar = hop.spaceop.result)
+        else:
+            hop.rename("same_as")
 
 
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -112,6 +112,7 @@
 
     def get_raw_address(self):
         from rpython.rtyper.lltypesystem import rffi
+        # may still raise ValueError on some GCs
         return rffi.get_raw_address_of_string(self.value)
 
 class SubBuffer(Buffer):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -542,8 +542,13 @@
         become identical to the one returned.
 
         NOTE: Only use for immutable objects!
+
+        NOTE: Might fail on some GCs!  You have to check again
+        can_move() afterwards.  It should always work with the default
+        GC.  With Boehm, can_move() is always False so
+        move_out_of_nursery() should never be called in the first place.
     """
-    pass
+    return obj
 
 class MoveOutOfNurseryEntry(ExtRegistryEntry):
     _about_ = move_out_of_nursery
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1336,6 +1336,8 @@
     as key is alive. If key goes out of scope, the buffer will eventually
     be freed. `string` cannot go out of scope until the RawBytes object
     referencing it goes out of scope.
+
+    NOTE: may raise ValueError on some GCs, but not the default one.
     """
     assert isinstance(string, str)
     from rpython.rtyper.annlowlevel import llstr
@@ -1346,6 +1348,8 @@
     if we_are_translated():
         if rgc.can_move(string):
             string = rgc.move_out_of_nursery(string)
+            if rgc.can_move(string):
+                raise ValueError("cannot make string immovable")
 
         # string cannot move now! return the address
         lldata = llstr(string)
diff --git a/rpython/rtyper/lltypesystem/test/test_ztranslated.py b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
--- a/rpython/rtyper/lltypesystem/test/test_ztranslated.py
+++ b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
@@ -37,7 +37,10 @@
     return ptr
 
 def main(argv=[]):
-    use_str()
+    try:
+        use_str()
+    except ValueError:
+        return 42
     gc.collect()
     mystr = b"12341234aa"*4096*10
     #debug_assert(not rgc.can_move(mystr), "long string can move... why?")
@@ -56,4 +59,15 @@
 
 def test_compiled_incminimark():
     fn = compile(main, [], gcpolicy="incminimark")
-    fn()
+    res = fn()
+    assert res == 0
+
+def test_compiled_semispace():
+    fn = compile(main, [], gcpolicy="semispace")
+    res = fn()
+    assert res == 42    # get_raw_address_of_string() raises ValueError
+
+def test_compiled_boehm():
+    fn = compile(main, [], gcpolicy="boehm")
+    res = fn()
+    assert res == 0

From pypy.commits at gmail.com  Fri Jan 13 07:12:23 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 13 Jan 2017 04:12:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: enhance _array_to_hexstring,
 it takes now the iteration direction into account
Message-ID: <5878c427.92ae1c0a.f389e.9fc8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89539:f293437a43a6
Date: 2017-01-13 11:32 +0100
http://bitbucket.org/pypy/pypy/changeset/f293437a43a6/

Log:	enhance _array_to_hexstring, it takes now the iteration direction
	into account

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -579,17 +579,15 @@
 PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8-1)-1)
 
 @specialize.arg(3) # raw access
-def _array_to_hexstring(space, buf, len=0, rawaccess=False):
-    if rawaccess:
-        length = len
-    else:
-        length = buf.getlength()
+def _array_to_hexstring(space, buf, start, step, length, rawaccess=False):
     hexstring = StringBuilder(length*2)
 
     if length > PY_SIZE_T_MAX/2:
         raise OperationError(space.w_MemoryError, space.w_None)
 
-    for i in range(length):
+    stepped = 0
+    i = start
+    while stepped < length:
         if rawaccess:
             byte = ord(buf[i])
         else:
@@ -598,6 +596,8 @@
         hexstring.append(HEXDIGITS[c])
         c = (byte & 0xf)
         hexstring.append(HEXDIGITS[c])
+        i += step
+        stepped += 1
 
     return space.wrap(hexstring.build())
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -675,7 +675,13 @@
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         self._check_released(space)
-        return _array_to_hexstring(space, self.buf)
+        if isinstance(self.buf, SubBuffer):
+            step = self.strides[0]
+            return _array_to_hexstring(space, self.buf.buffer,
+                                       self.buf.offset, step,
+                                       self.getlength())
+        else:
+            return _array_to_hexstring(space, self.buf, 0, 1, self.getlength())
 
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -194,10 +194,10 @@
         assert memoryview(b"abc").hex() == u'616263'
 
     def test_hex_long(self):
-        x = b'0' * 200000
+        x = b'01' * 100000
         m1 = memoryview(x)
         m2 = m1[::-1]
-        assert m2.hex() == '30' * 200000
+        assert m2.hex() == '3130' * 100000
 
     def test_memoryview_cast(self):
         m1 = memoryview(b'abcdefgh')

From pypy.commits at gmail.com  Fri Jan 13 07:12:26 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 13 Jan 2017 04:12:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: new test case (reverse buffer type i),
Message-ID: <5878c42a.0d1a1c0a.2fc08.b0ec@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89540:4cada5dbc4cd
Date: 2017-01-13 13:10 +0100
http://bitbucket.org/pypy/pypy/changeset/4cada5dbc4cd/

Log:	new test case (reverse buffer type i), copy over flag checking from
	cpyext and adapt the code there. changes needed to properly setup
	the memoryview flags

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -8,7 +8,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.interpreter.error import oefmt
-from pypy.objspace.std.memoryobject import W_MemoryView
+from pypy.objspace.std.memoryobject import W_MemoryView, _pybuffer_iscontiguous
 from pypy.module.cpyext.object import _dealloc
 from pypy.module.cpyext.import_ import PyImport_Import
 
@@ -164,41 +164,8 @@
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
     return 0
 
-def _IsFortranContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(view.c_ndim):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-
-def _IsCContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(ndim - 1, -1, -1):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
+# If you are looking for _IsFortran/CContiguous look in 
+# pypy/module/cpyext/memoryobject.py
 
 @cpython_api([Py_bufferP, lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
 def PyBuffer_IsContiguous(space, view, fort):
@@ -207,15 +174,13 @@
     (fort is 'A').  Return 0 otherwise."""
     # traverse the strides, checking for consistent stride increases from
     # right-to-left (c) or left-to-right (fortran). Copied from cpython
-    if view.c_suboffsets:
-        return 0
-    if (fort == 'C'):
-        return _IsCContiguous(view)
-    elif (fort == 'F'):
-        return _IsFortranContiguous(view)
-    elif (fort == 'A'):
-        return (_IsCContiguous(view) or _IsFortranContiguous(view))
-    return 0
+
+    # plan_rich: I wanted to reuse this check in pypy/objspace/std/...
+    # memoryview.py, thus I have rewritten it, maybe it is time to 
+    # find a way to share this code?
+    return _pybuffer_iscontiguous(view.c_suboffsets,
+              widen(view.c_ndim), view.c_shape, view.c_strides,
+              view.c_itemsize, fort)
 
 @cpython_api([PyObject], PyObject, result_is_ll=True)
 def PyMemoryView_FromObject(space, w_obj):
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -13,6 +13,7 @@
 from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
 from pypy.objspace.std.bytesobject import getbytevalue
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import always_inline
 
 MEMORYVIEW_MAX_DIM = 64
 MEMORYVIEW_SCALAR   = 0x0001
@@ -571,19 +572,26 @@
         flags = 0
         if ndim == 0:
             flags |= MEMORYVIEW_SCALAR | MEMORYVIEW_C | MEMORYVIEW_FORTRAN
-        if ndim == 1:
-            shape = buf.getshape()
-            strides = buf.getstrides()
-            if len(shape) > 0 and shape[0] == 1 and \
-               len(strides) > 0 and strides[0] == buf.getitemsize():
+        elif ndim == 1:
+            shape = self.getshape()
+            strides = self.getstrides()
+            if shape[0] == 1 or strides[0] == self.getitemsize():
                 flags |= MEMORYVIEW_C | MEMORYVIEW_SCALAR
-        # TODO for now?
-        flags |= MEMORYVIEW_C
-        # TODO if buf.is_contiguous('C'):
-        # TODO     flags |= MEMORYVIEW_C
-        # TODO elif buf.is_contiguous('F'):
-        # TODO     flags |= MEMORYVIEW_FORTRAN
+        else:
+            ndim = self.getndim()
+            shape = self.getshape()
+            strides = self.getstrides()
+            itemsize = self.getitemsize()
+            if _pybuffer_iscontiguous(None, ndim, shape, strides,
+                                      itemsize, 'C'):
+                flags |= MEMORYVIEW_C
+            if _pybuffer_iscontiguous(None, ndim, shape, strides,
+                                      itemsize, 'F'):
+                flags |= MEMORYVIEW_C
 
+        if self.suboffsets:
+            flags |= MEMORYVIEW_PIL
+            flags &= ~(MEMORYVIEW_C|MEMORYVIEW_FORTRAN)
         # TODO missing suboffsets
 
         self.flags = flags
@@ -675,19 +683,17 @@
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         self._check_released(space)
-        if isinstance(self.buf, SubBuffer):
-            step = self.strides[0]
-            return _array_to_hexstring(space, self.buf.buffer,
-                                       self.buf.offset, step,
-                                       self.getlength())
+        if memory_view_c_contiguous(space, self.flags):
+            return _array_to_hexstring(space, self.buf, 0, 1, self.getlength())
         else:
-            return _array_to_hexstring(space, self.buf, 0, 1, self.getlength())
+            bytes = self.as_str()
+            return _array_to_hexstring(space, bytes, 0, 1, len(bytes), True)
 
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'
 
 def memory_view_c_contiguous(space, flags):
-    return flags & (space.BUF_CONTIG_RO|MEMORYVIEW_C) != 0
+    return flags & (MEMORYVIEW_SCALAR|MEMORYVIEW_C)
 
 W_MemoryView.typedef = TypeDef(
     "memoryview",
@@ -721,3 +727,51 @@
     _pypy_raw_address = interp2app(W_MemoryView.descr_pypy_raw_address),
     )
 W_MemoryView.typedef.acceptable_as_base_class = False
+
+def _IsFortranContiguous(ndim, shape, strides, itemsize):
+    if ndim == 0:
+        return 1
+    if not strides:
+        return ndim == 1
+    sd = itemsize
+    if ndim == 1:
+        return shape[0] == 1 or sd == strides[0]
+    for i in range(ndim):
+        dim = shape[i]
+        if dim == 0:
+            return 1
+        if strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(ndim, shape, strides, itemsize):
+    if ndim == 0:
+        return 1
+    if not strides:
+        return ndim == 1
+    sd = itemsize
+    if ndim == 1:
+        return shape[0] == 1 or sd == strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = shape[i]
+        if dim == 0:
+            return 1
+        if strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+ at always_inline
+def _pybuffer_iscontiguous(suboffsets, ndim, shape, strides, itemsize, fort):
+    if suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(ndim, shape, strides, itemsize)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(ndim, shape, strides, itemsize)
+    elif (fort == 'A'):
+        return (_IsCContiguous(ndim, shape, strides, itemsize) or \
+                _IsFortranContiguous(ndim, shape, strides, itemsize))
+    return 0
+
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -8,7 +8,7 @@
 from pypy.conftest import option
 
 class AppTestMemoryView:
-    spaceconfig = dict(usemodules=['array'])
+    spaceconfig = dict(usemodules=['array', 'sys'])
 
     def test_basic(self):
         v = memoryview(b"abc")
@@ -199,6 +199,16 @@
         m2 = m1[::-1]
         assert m2.hex() == '3130' * 100000
 
+    def test_hex_2(self):
+        import array
+        import sys
+        m1 = memoryview(array.array('i', [1,2,3,4]))
+        m2 = m1[::-1]
+        if sys.byteorder == 'little':
+            assert m2.hex() == "04000000030000000200000001000000"
+        else:
+            assert m2.hex() == "00000004000000030000000200000001"
+
     def test_memoryview_cast(self):
         m1 = memoryview(b'abcdefgh')
         m2 = m1.cast('I')
@@ -391,7 +401,10 @@
         assert view.format == 'b'
         assert cview.format == 'i'
         #
-        assert cview.cast('b').cast('q').cast('b').tolist() == []
+        a = cview.cast('b')
+        b = a.cast('q')
+        c = b.cast('b')
+        assert c.tolist() == []
         #
         assert cview.format == 'i'
         raises(TypeError, "cview.cast('i')")
@@ -399,7 +412,7 @@
     def test_cast_with_shape(self):
         empty = self.MockArray([1,0,2,0,3,0],
                     dim=1, fmt='h', size=2,
-                    strides=[8], shape=[6])
+                    strides=[2], shape=[6])
         view = memoryview(empty)
         byteview = view.cast('b')
         assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]

From pypy.commits at gmail.com  Fri Jan 13 07:16:20 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 13 Jan 2017 04:16:20 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: change parameters of _array_to_hexstring
 for bytearray(...).hex()
Message-ID: <5878c514.52301c0a.536b6.a5bb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89541:6b60487f8de0
Date: 2017-01-13 13:13 +0100
http://bitbucket.org/pypy/pypy/changeset/6b60487f8de0/

Log:	change parameters of _array_to_hexstring for bytearray(...).hex()

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -483,7 +483,7 @@
 
     def descr_hex(self, space):
         data = self.getdata()
-        return _array_to_hexstring(space, data, len(data), True)
+        return _array_to_hexstring(space, data, 0, 1, len(data), True)
 
     def descr_mod(self, space, w_values):
         return mod_format(space, self, w_values, fmt_type=FORMAT_BYTEARRAY)

From pypy.commits at gmail.com  Fri Jan 13 07:16:22 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 13 Jan 2017 04:16:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: same fix for bytes objects
Message-ID: <5878c516.d5091c0a.81af2.ab3e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89542:4da8c14ed0f0
Date: 2017-01-13 13:15 +0100
http://bitbucket.org/pypy/pypy/changeset/4da8c14ed0f0/

Log:	same fix for bytes objects

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -678,7 +678,7 @@
 
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
-        return _array_to_hexstring(space, StringBuffer(self._value))
+        return _array_to_hexstring(space, self._value, 0, 1, len(self._value), True)
 
     def descr_mod(self, space, w_values):
         return mod_format(space, self, w_values, fmt_type=FORMAT_BYTES)

From pypy.commits at gmail.com  Fri Jan 13 11:07:53 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 13 Jan 2017 08:07:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: parameter moved,
 calling StringBuffer before parameter is passed to
 _array_to_hexstring, seems to translate now
Message-ID: <5878fb59.973f1c0a.773ad.2fd0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89543:08b6498f72ef
Date: 2017-01-13 17:07 +0100
http://bitbucket.org/pypy/pypy/changeset/08b6498f72ef/

Log:	parameter moved, calling StringBuffer before parameter is passed to
	_array_to_hexstring, seems to translate now

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -578,7 +578,7 @@
 HEXDIGITS = "0123456789abcdef"
 PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8-1)-1)
 
- at specialize.arg(3) # raw access
+ at specialize.arg(5) # raw access
 def _array_to_hexstring(space, buf, start, step, length, rawaccess=False):
     hexstring = StringBuilder(length*2)
 
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -678,7 +678,7 @@
 
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
-        return _array_to_hexstring(space, self._value, 0, 1, len(self._value), True)
+        return _array_to_hexstring(space, StringBuffer(self._value), 0, 1, len(self._value))
 
     def descr_mod(self, space, w_values):
         return mod_format(space, self, w_values, fmt_type=FORMAT_BYTES)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -3,7 +3,7 @@
 """
 import operator
 
-from rpython.rlib.buffer import Buffer, SubBuffer
+from rpython.rlib.buffer import Buffer, SubBuffer, StringBuffer
 from rpython.rlib.objectmodel import compute_hash
 from rpython.rlib.rstruct.error import StructError
 from pypy.interpreter.baseobjspace import W_Root
@@ -687,7 +687,7 @@
             return _array_to_hexstring(space, self.buf, 0, 1, self.getlength())
         else:
             bytes = self.as_str()
-            return _array_to_hexstring(space, bytes, 0, 1, len(bytes), True)
+            return _array_to_hexstring(space, StringBuffer(bytes), 0, 1, len(bytes))
 
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'

From pypy.commits at gmail.com  Fri Jan 13 11:13:52 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 13 Jan 2017 08:13:52 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Configure types using only the C
 source passed to parse_source()
Message-ID: <5878fcc0.896f1c0a.1c9ae.f7bc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89544:bdd3e262b996
Date: 2017-01-13 16:13 +0000
http://bitbucket.org/pypy/pypy/changeset/bdd3e262b996/

Log:	Configure types using only the C source passed to parse_source()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -974,7 +974,7 @@
 
 } PyTypeObject;
 
-""", configure_now=True)
+""", headers=['sys/types.h', 'stdarg.h', 'stdio.h'], configure_now=True)
 
 Py_ssize_t = object_h.gettype('Py_ssize_t')
 Py_ssize_tP = object_h.gettype('Py_ssize_t *')
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -6,6 +6,7 @@
 except ImportError:
     import pycparser
 import weakref, re
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rfile import FILEP
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rfficache, rffi_platform
@@ -675,17 +676,15 @@
 
 
 class ParsedSource(object):
-    def __init__(self, source, parser, definitions=None, macros=None, eci=None):
+    def __init__(self, source, parser, definitions=None, macros=None, headers=None):
         from pypy.module.cpyext.api import configure_eci
         self.source = source
         self.definitions = definitions if definitions is not None else {}
         self.macros = macros if macros is not None else {}
         self.structs = {}
         self.ctx = parser
-        if eci is None:
-            eci = configure_eci
-        self._Config = type(
-            'Config', (object,), {'_compilation_info_': eci})
+        self.headers = headers if headers is not None else ['sys/types.h']
+        self._Config = type('Config', (object,), {})
         self._TYPES = {}
         self.includes = []
 
@@ -728,6 +727,16 @@
             cpython_struct(type_name, struct.fields, forward=struct.TYPE)
         return struct.TYPE
 
+    def build_eci(self):
+        all_sources = [x.source for x in self.includes] + [self.source]
+        all_headers = self.headers
+        for x in self.includes:
+            for hdr in x.headers:
+                if hdr not in all_headers:
+                    all_headers.append(hdr)
+        return ExternalCompilationInfo(
+            post_include_bits=all_sources, includes=all_headers)
+
     def configure_types(self, configure_now=False):
         for name, (obj, quals) in self.ctx._declarations.iteritems():
             if obj in self.ctx._included_declarations:
@@ -738,6 +747,7 @@
             elif name.startswith('macro '):
                 name = name[6:]
                 self.add_macro(name, obj)
+        self._Config._compilation_info_ = self.build_eci()
         for name, TYPE in rffi_platform.configure(self._Config).iteritems():
             if name in self._TYPES:
                 self._TYPES[name].become(TYPE)
@@ -794,9 +804,9 @@
         return decl.name, FUNCP.TO
 
 
-def parse_source(source, includes=None, eci=None, configure_now=False):
+def parse_source(source, includes=None, headers=None, configure_now=False):
     ctx = Parser()
-    src = ParsedSource(source, ctx, eci=eci)
+    src = ParsedSource(source, ctx, headers=headers)
     if includes is not None:
         for header in includes:
             src.include(header)
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -1,8 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.module.cpyext.cparser import parse_source
 
-def test_configure(tmpdir):
+def test_configure():
     decl = """
     typedef ssize_t Py_ssize_t;
 
@@ -12,11 +11,7 @@
         double ob_fval;
     } TestFloatObject;
     """
-    hdr = tmpdir / 'header.h'
-    hdr.write(decl)
-    eci = ExternalCompilationInfo(
-        include_dirs=[str(tmpdir)], includes=['sys/types.h', 'header.h'])
-    res = parse_source(decl, eci=eci, configure_now=True)
+    res = parse_source(decl, configure_now=True)
     TestFloatObject = res.definitions['TestFloatObject']
     assert isinstance(TestFloatObject, lltype.Struct)
     assert TestFloatObject.c_ob_refcnt == rffi.SSIZE_T
@@ -45,7 +40,7 @@
     assert 'PyFloatObject' in hdr.definitions
     assert 'PyObject_HEAD' in hdr.macros
 
-def test_include(tmpdir):
+def test_include():
     cdef1 = """
     typedef ssize_t Py_ssize_t;
 
@@ -57,8 +52,6 @@
         char *name;
     } Type;
     """
-    base_name = tmpdir / 'base.h'
-    base_name.write(cdef1)
     cdef2 = """
     typedef struct {
         PyObject_HEAD
@@ -66,19 +59,15 @@
         Type *type;
     } Object;
     """
-    (tmpdir / 'object.h').write(cdef2)
-    eci = ExternalCompilationInfo(
-        include_dirs=[str(tmpdir)],
-        includes=['sys/types.h', 'base.h', 'object.h'])
-    hdr1 = parse_source(cdef1, eci=eci, configure_now=True)
+    hdr1 = parse_source(cdef1, configure_now=True)
     Type = hdr1.definitions['Type']
     assert isinstance(Type, lltype.Struct)
-    hdr2 = parse_source(cdef2, includes=[hdr1], eci=eci, configure_now=True)
+    hdr2 = parse_source(cdef2, includes=[hdr1], configure_now=True)
     assert 'Type' not in hdr2.definitions
     Object = hdr2.definitions['Object']
     assert Object.c_type.TO is Type
 
-def test_incomplete(tmpdir):
+def test_incomplete():
     cdef = """
     typedef ssize_t Py_ssize_t;
 
@@ -94,15 +83,11 @@
     } Buffer;
 
     """
-    (tmpdir / 'foo.h').write(cdef)
-    eci = ExternalCompilationInfo(
-        include_dirs=[str(tmpdir)],
-        includes=['sys/types.h', 'foo.h'])
-    foo_h = parse_source(cdef, eci=eci)
+    foo_h = parse_source(cdef)
     Object = foo_h.gettype('Object')
     assert isinstance(Object, lltype.ForwardReference)
 
-def test_recursive(tmpdir):
+def test_recursive():
     cdef = """
     typedef ssize_t Py_ssize_t;
 
@@ -121,29 +106,21 @@
         Object *obj;
     } Type;
     """
-    (tmpdir / 'foo.h').write(cdef)
-    eci = ExternalCompilationInfo(
-        include_dirs=[str(tmpdir)],
-        includes=['sys/types.h', 'foo.h'])
-    foo_h = parse_source(cdef, eci=eci, configure_now=True)
+    foo_h = parse_source(cdef, configure_now=True)
     Object = foo_h.definitions['Object']
     assert isinstance(Object, lltype.Struct)
     hash(Object)
 
-def test_const(tmpdir):
+def test_const():
     cdef = """
     typedef struct {
         const char * const foo;
     } bar;
     """
-    (tmpdir / 'foo.h').write(cdef)
-    eci = ExternalCompilationInfo(
-        include_dirs=[str(tmpdir)],
-        includes=['sys/types.h', 'foo.h'])
-    hdr = parse_source(cdef, eci=eci, configure_now=True)
+    hdr = parse_source(cdef, configure_now=True)
     assert hdr.definitions['bar'].c_foo == rffi.CONST_CCHARP != rffi.CCHARP
 
-def test_gettype(tmpdir):
+def test_gettype():
     decl = """
     typedef ssize_t Py_ssize_t;
 
@@ -156,15 +133,11 @@
         double ob_fval;
     } TestFloatObject;
     """
-    hdr = tmpdir / 'header.h'
-    hdr.write(decl)
-    eci = ExternalCompilationInfo(
-        include_dirs=[str(tmpdir)], includes=['sys/types.h', 'header.h'])
-    res = parse_source(decl, eci=eci, configure_now=True)
+    res = parse_source(decl, configure_now=True)
     assert res.gettype('Py_ssize_t') == rffi.SSIZE_T
     assert res.gettype('TestFloatObject *').TO.c_ob_refcnt == rffi.SSIZE_T
 
-def test_parse_funcdecl(tmpdir):
+def test_parse_funcdecl():
     decl = """
     typedef ssize_t Py_ssize_t;
 
@@ -179,11 +152,7 @@
 
     typedef TestFloatObject* (*func_t)(int, int);
     """
-    hdr = tmpdir / 'header.h'
-    hdr.write(decl)
-    eci = ExternalCompilationInfo(
-        include_dirs=[str(tmpdir)], includes=['sys/types.h', 'header.h'])
-    res = parse_source(decl, eci=eci, configure_now=True)
+    res = parse_source(decl, configure_now=True)
     name, FUNC = res.parse_func("func_t some_func(TestFloatObject*)")
     assert name == 'some_func'
     assert FUNC.RESULT == res.gettype('func_t')

From pypy.commits at gmail.com  Fri Jan 13 11:24:13 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 13 Jan 2017 08:24:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: duplicate code, translation issue
Message-ID: <5878ff2d.810b1c0a.48125.3c95@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89545:92c913693d64
Date: 2017-01-13 17:20 +0100
http://bitbucket.org/pypy/pypy/changeset/92c913693d64/

Log:	duplicate code, translation issue

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -8,7 +8,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.interpreter.error import oefmt
-from pypy.objspace.std.memoryobject import W_MemoryView, _pybuffer_iscontiguous
+from pypy.objspace.std.memoryobject import W_MemoryView
 from pypy.module.cpyext.object import _dealloc
 from pypy.module.cpyext.import_ import PyImport_Import
 
@@ -164,8 +164,41 @@
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
     return 0
 
-# If you are looking for _IsFortran/CContiguous look in 
-# pypy/module/cpyext/memoryobject.py
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
 
 @cpython_api([Py_bufferP, lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
 def PyBuffer_IsContiguous(space, view, fort):
@@ -175,12 +208,15 @@
     # traverse the strides, checking for consistent stride increases from
     # right-to-left (c) or left-to-right (fortran). Copied from cpython
 
-    # plan_rich: I wanted to reuse this check in pypy/objspace/std/...
-    # memoryview.py, thus I have rewritten it, maybe it is time to 
-    # find a way to share this code?
-    return _pybuffer_iscontiguous(view.c_suboffsets,
-              widen(view.c_ndim), view.c_shape, view.c_strides,
-              view.c_itemsize, fort)
+    if view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
 
 @cpython_api([PyObject], PyObject, result_is_ll=True)
 def PyMemoryView_FromObject(space, w_obj):
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -582,10 +582,10 @@
             shape = self.getshape()
             strides = self.getstrides()
             itemsize = self.getitemsize()
-            if _pybuffer_iscontiguous(None, ndim, shape, strides,
+            if PyBuffer_isContiguous(None, ndim, shape, strides,
                                       itemsize, 'C'):
                 flags |= MEMORYVIEW_C
-            if _pybuffer_iscontiguous(None, ndim, shape, strides,
+            if PyBuffer_isContiguous(None, ndim, shape, strides,
                                       itemsize, 'F'):
                 flags |= MEMORYVIEW_C
 
@@ -762,8 +762,7 @@
         sd *= dim
     return 1
 
- at always_inline
-def _pybuffer_iscontiguous(suboffsets, ndim, shape, strides, itemsize, fort):
+def PyBuffer_isContiguous(suboffsets, ndim, shape, strides, itemsize, fort):
     if suboffsets:
         return 0
     if (fort == 'C'):

From pypy.commits at gmail.com  Fri Jan 13 11:30:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 08:30:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: tcl fix
Message-ID: <5879009e.42061c0a.2db68.4171@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89546:1545e485e3c1
Date: 2017-01-13 17:29 +0100
http://bitbucket.org/pypy/pypy/changeset/1545e485e3c1/

Log:	tcl fix

diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -31,7 +31,7 @@
 
 def FromTclString(s):
     try:
-        return s.decode('utf8')
+        return s.decode('utf-8')
     except UnicodeDecodeError:
         # Tcl encodes null character as \xc0\x80
         try:
@@ -194,7 +194,7 @@
 
     @property
     def typename(self):
-        return tkffi.string(self._value.typePtr.name)
+        return FromTclString(tkffi.string(self._value.typePtr.name))
 
     @property
     def string(self):
@@ -203,6 +203,6 @@
             length = tkffi.new("int*")
             s = tklib.Tcl_GetStringFromObj(self._value, length)
             value = tkffi.buffer(s, length[0])[:]
-            value = value.decode('utf8')
+            value = value.decode('utf-8')
             self._string = value
         return self._string

From pypy.commits at gmail.com  Fri Jan 13 11:44:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 08:44:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Apply the test fix in CPython 3.5.trunk
Message-ID: <58790409.44641c0a.ad077.e2d9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89547:03c02638a180
Date: 2017-01-13 17:44 +0100
http://bitbucket.org/pypy/pypy/changeset/03c02638a180/

Log:	Apply the test fix in CPython 3.5.trunk

diff --git a/lib-python/3/sqlite3/test/factory.py b/lib-python/3/sqlite3/test/factory.py
--- a/lib-python/3/sqlite3/test/factory.py
+++ b/lib-python/3/sqlite3/test/factory.py
@@ -183,10 +183,12 @@
     def CheckFakeCursorClass(self):
         # Issue #24257: Incorrect use of PyObject_IsInstance() caused
         # segmentation fault.
+        # Issue #27861: Also applies for cursor factory.
         class FakeCursor(str):
             __class__ = sqlite.Cursor
-        cur = self.con.cursor(factory=FakeCursor)
-        self.assertRaises(TypeError, sqlite.Row, cur, ())
+        self.con.row_factory = sqlite.Row
+        self.assertRaises(TypeError, self.con.cursor, FakeCursor)
+        self.assertRaises(TypeError, sqlite.Row, FakeCursor(), ())
 
     def tearDown(self):
         self.con.close()

From pypy.commits at gmail.com  Fri Jan 13 12:06:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 09:06:47 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add an XXX hack
Message-ID: <58790927.43e61c0a.451d3.5830@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89548:c7e665a4d094
Date: 2017-01-13 18:05 +0100
http://bitbucket.org/pypy/pypy/changeset/c7e665a4d094/

Log:	add an XXX hack

diff --git a/lib-python/3/inspect.py b/lib-python/3/inspect.py
--- a/lib-python/3/inspect.py
+++ b/lib-python/3/inspect.py
@@ -2077,6 +2077,8 @@
 
     s = getattr(func, "__text_signature__", None)
     if not s:
+        if func is object:  # XXX PyPy hack until we support __text_signature__
+            return '()'     # in the same cases as CPython
         raise ValueError("no signature found for builtin {!r}".format(func))
 
     return _signature_fromstr(cls, func, s, skip_bound_arg)

From pypy.commits at gmail.com  Fri Jan 13 12:13:14 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 13 Jan 2017 09:13:14 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Move pseudo-header to an actual
 header file p/m/c/parse/cpyext_object.h
Message-ID: <58790aaa.878f1c0a.ab837.5f64@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89549:1becaa3af977
Date: 2017-01-13 17:12 +0000
http://bitbucket.org/pypy/pypy/changeset/1becaa3af977/

Log:	Move pseudo-header to an actual header file
	p/m/c/parse/cpyext_object.h

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -46,6 +46,7 @@
 
 pypydir = py.path.local(pypydir)
 include_dir = pypydir / 'module' / 'cpyext' / 'include'
+parse_dir = pypydir / 'module' / 'cpyext' / 'parse'
 source_dir = pypydir / 'module' / 'cpyext' / 'src'
 translator_c_dir = py.path.local(cdir)
 include_dirs = [
@@ -666,315 +667,9 @@
                              % (cpyname, ))
 build_exported_objects()
 
-object_h = parse_source("""
-typedef ssize_t Py_ssize_t;
-
-#define PyObject_HEAD  \
-    Py_ssize_t ob_refcnt;        \
-    Py_ssize_t ob_pypy_link;     \
-    struct _typeobject *ob_type;
-
-#define PyObject_VAR_HEAD		\
-	PyObject_HEAD			\
-	Py_ssize_t ob_size; /* Number of items in variable part */
-
-typedef struct _object {
-    PyObject_HEAD
-} PyObject;
-
-typedef struct {
-	PyObject_VAR_HEAD
-} PyVarObject;
-
-struct _typeobject;
-
-typedef void (*freefunc)(void *);
-typedef void (*destructor)(PyObject *);
-typedef int (*printfunc)(PyObject *, FILE *, int);
-typedef PyObject *(*getattrfunc)(PyObject *, char *);
-typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
-typedef int (*setattrfunc)(PyObject *, char *, PyObject *);
-typedef int (*setattrofunc)(PyObject *, PyObject *, PyObject *);
-typedef int (*cmpfunc)(PyObject *, PyObject *);
-typedef PyObject *(*reprfunc)(PyObject *);
-typedef long (*hashfunc)(PyObject *);
-typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
-typedef PyObject *(*getiterfunc) (PyObject *);
-typedef PyObject *(*iternextfunc) (PyObject *);
-typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);
-typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *);
-typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
-typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
-typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t);
-
-typedef PyObject * (*unaryfunc)(PyObject *);
-typedef PyObject * (*binaryfunc)(PyObject *, PyObject *);
-typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
-typedef int (*inquiry)(PyObject *);
-typedef Py_ssize_t (*lenfunc)(PyObject *);
-typedef int (*coercion)(PyObject **, PyObject **);
-typedef PyObject *(*intargfunc)(PyObject *, int);
-typedef PyObject *(*intintargfunc)(PyObject *, int, int);
-typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
-typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
-typedef int(*intobjargproc)(PyObject *, int, PyObject *);
-typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *);
-typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
-typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
-typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
-
-
-/* int-based buffer interface */
-typedef int (*getreadbufferproc)(PyObject *, int, void **);
-typedef int (*getwritebufferproc)(PyObject *, int, void **);
-typedef int (*getsegcountproc)(PyObject *, int *);
-typedef int (*getcharbufferproc)(PyObject *, int, char **);
-/* ssize_t-based buffer interface */
-typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
-typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
-typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
-typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
-
-/* Py3k buffer interface, adapted for PyPy */
-#define Py_MAX_NDIMS 32
-#define Py_MAX_FMT 128
-typedef struct bufferinfo {
-    void *buf;
-    PyObject *obj;        /* owned reference */
-    Py_ssize_t len;
-
-    /* This is Py_ssize_t so it can be
-       pointed to by strides in simple case.*/
-    Py_ssize_t itemsize;
-    int readonly;
-    int ndim;
-    char *format;
-    Py_ssize_t *shape;
-    Py_ssize_t *strides;
-    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
-    unsigned char _format[Py_MAX_FMT];
-    Py_ssize_t _strides[Py_MAX_NDIMS];
-    Py_ssize_t _shape[Py_MAX_NDIMS];
-    /* static store for shape and strides of
-       mono-dimensional buffers. */
-    /* Py_ssize_t smalltable[2]; */
-    void *internal; /* always NULL for app-level objects */
-} Py_buffer;
-
-
-typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
-typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
-/* end Py3k buffer interface */
-
-typedef int (*objobjproc)(PyObject *, PyObject *);
-typedef int (*visitproc)(PyObject *, void *);
-typedef int (*traverseproc)(PyObject *, visitproc, void *);
-
-typedef struct {
-	/* For numbers without flag bit Py_TPFLAGS_CHECKTYPES set, all
-	   arguments are guaranteed to be of the object's type (modulo
-	   coercion hacks -- i.e. if the type's coercion function
-	   returns other types, then these are allowed as well).  Numbers that
-	   have the Py_TPFLAGS_CHECKTYPES flag bit set should check *both*
-	   arguments for proper type and implement the necessary conversions
-	   in the slot functions themselves. */
-
-	binaryfunc nb_add;
-	binaryfunc nb_subtract;
-	binaryfunc nb_multiply;
-	binaryfunc nb_divide;
-	binaryfunc nb_remainder;
-	binaryfunc nb_divmod;
-	ternaryfunc nb_power;
-	unaryfunc nb_negative;
-	unaryfunc nb_positive;
-	unaryfunc nb_absolute;
-	inquiry nb_nonzero;
-	unaryfunc nb_invert;
-	binaryfunc nb_lshift;
-	binaryfunc nb_rshift;
-	binaryfunc nb_and;
-	binaryfunc nb_xor;
-	binaryfunc nb_or;
-	coercion nb_coerce;
-	unaryfunc nb_int;
-	unaryfunc nb_long;
-	unaryfunc nb_float;
-	unaryfunc nb_oct;
-	unaryfunc nb_hex;
-	/* Added in release 2.0 */
-	binaryfunc nb_inplace_add;
-	binaryfunc nb_inplace_subtract;
-	binaryfunc nb_inplace_multiply;
-	binaryfunc nb_inplace_divide;
-	binaryfunc nb_inplace_remainder;
-	ternaryfunc nb_inplace_power;
-	binaryfunc nb_inplace_lshift;
-	binaryfunc nb_inplace_rshift;
-	binaryfunc nb_inplace_and;
-	binaryfunc nb_inplace_xor;
-	binaryfunc nb_inplace_or;
-
-	/* Added in release 2.2 */
-	/* The following require the Py_TPFLAGS_HAVE_CLASS flag */
-	binaryfunc nb_floor_divide;
-	binaryfunc nb_true_divide;
-	binaryfunc nb_inplace_floor_divide;
-	binaryfunc nb_inplace_true_divide;
-
-	/* Added in release 2.5 */
-	unaryfunc nb_index;
-} PyNumberMethods;
-
-typedef struct {
-	lenfunc sq_length;
-	binaryfunc sq_concat;
-	ssizeargfunc sq_repeat;
-	ssizeargfunc sq_item;
-	ssizessizeargfunc sq_slice;
-	ssizeobjargproc sq_ass_item;
-	ssizessizeobjargproc sq_ass_slice;
-	objobjproc sq_contains;
-	/* Added in release 2.0 */
-	binaryfunc sq_inplace_concat;
-	ssizeargfunc sq_inplace_repeat;
-} PySequenceMethods;
-
-typedef struct {
-	lenfunc mp_length;
-	binaryfunc mp_subscript;
-	objobjargproc mp_ass_subscript;
-} PyMappingMethods;
-
-typedef struct {
-	readbufferproc bf_getreadbuffer;
-	writebufferproc bf_getwritebuffer;
-	segcountproc bf_getsegcount;
-	charbufferproc bf_getcharbuffer;
-	getbufferproc bf_getbuffer;
-	releasebufferproc bf_releasebuffer;
-} PyBufferProcs;
-
-/* from descrobject.h */
-typedef PyObject *(*getter)(PyObject *, void *);
-typedef int (*setter)(PyObject *, PyObject *, void *);
-
-typedef struct PyGetSetDef {
-	char *name;
-	getter get;
-	setter set;
-	char *doc;
-	void *closure;
-} PyGetSetDef;
-
-/* from methodobject.h */
-typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
-typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
-                                             PyObject *);
-typedef PyObject *(*PyNoArgsFunction)(PyObject *);
-
-struct PyMethodDef {
-    const char  *ml_name;   /* The name of the built-in function/method */
-    PyCFunction  ml_meth;   /* The C function that implements it */
-    int          ml_flags;  /* Combination of METH_xxx flags, which mostly
-                               describe the args expected by the C func */
-    const char  *ml_doc;    /* The __doc__ attribute, or NULL */
-};
-typedef struct PyMethodDef PyMethodDef;
-
-/* from structmember.h */
-typedef struct PyMemberDef {
-    /* Current version, use this */
-    char *name;
-    int type;
-    Py_ssize_t offset;
-    int flags;
-    char *doc;
-} PyMemberDef;
-
-
-typedef struct _typeobject {
-	PyObject_VAR_HEAD
-	/* const */ char *tp_name; /* For printing, in format "<module>.<name>" */
-	Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
-
-	/* Methods to implement standard operations */
-
-	destructor tp_dealloc;
-	printfunc tp_print;
-	getattrfunc tp_getattr;
-	setattrfunc tp_setattr;
-	cmpfunc tp_compare;
-	reprfunc tp_repr;
-
-	/* Method suites for standard classes */
-
-	PyNumberMethods *tp_as_number;
-	PySequenceMethods *tp_as_sequence;
-	PyMappingMethods *tp_as_mapping;
-
-	/* More standard operations (here for binary compatibility) */
-
-	hashfunc tp_hash;
-	ternaryfunc tp_call;
-	reprfunc tp_str;
-	getattrofunc tp_getattro;
-	setattrofunc tp_setattro;
-
-	/* Functions to access object as input/output buffer */
-	PyBufferProcs *tp_as_buffer;
-
-	/* Flags to define presence of optional/expanded features */
-	long tp_flags;
-
-	/*const*/ char *tp_doc; /* Documentation string */
-
-	/* Assigned meaning in release 2.0 */
-	/* call function for all accessible objects */
-	traverseproc tp_traverse;
-
-	/* delete references to contained objects */
-	inquiry tp_clear;
-
-	/* Assigned meaning in release 2.1 */
-	/* rich comparisons */
-	richcmpfunc tp_richcompare;
-
-	/* weak reference enabler */
-	Py_ssize_t tp_weaklistoffset;
-
-	/* Added in release 2.2 */
-	/* Iterators */
-	getiterfunc tp_iter;
-	iternextfunc tp_iternext;
-
-	/* Attribute descriptor and subclassing stuff */
-	struct PyMethodDef *tp_methods;
-	struct PyMemberDef *tp_members;
-	struct PyGetSetDef *tp_getset;
-	struct _typeobject *tp_base;
-	PyObject *tp_dict;
-	descrgetfunc tp_descr_get;
-	descrsetfunc tp_descr_set;
-	Py_ssize_t tp_dictoffset;
-	initproc tp_init;
-	allocfunc tp_alloc;
-	newfunc tp_new;
-	freefunc tp_free; /* Low-level free-memory routine */
-	inquiry tp_is_gc; /* For PyObject_IS_GC */
-	PyObject *tp_bases;
-	PyObject *tp_mro; /* method resolution order */
-	PyObject *tp_cache;
-	PyObject *tp_subclasses;
-	PyObject *tp_weaklist;
-	destructor tp_del;
-
-	/* Type attribute cache version tag. Added in version 2.6 */
-	unsigned int tp_version_tag;
-
-} PyTypeObject;
-
-""", headers=['sys/types.h', 'stdarg.h', 'stdio.h'], configure_now=True)
+object_cdef = (parse_dir / 'cpyext_object.h').read()
+object_h = parse_source(object_cdef,
+    headers=['sys/types.h', 'stdarg.h', 'stdio.h'], configure_now=True)
 
 Py_ssize_t = object_h.gettype('Py_ssize_t')
 Py_ssize_tP = object_h.gettype('Py_ssize_t *')
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -0,0 +1,308 @@
+
+typedef ssize_t Py_ssize_t;
+
+#define PyObject_HEAD  \
+    Py_ssize_t ob_refcnt;        \
+    Py_ssize_t ob_pypy_link;     \
+    struct _typeobject *ob_type;
+
+#define PyObject_VAR_HEAD		\
+	PyObject_HEAD			\
+	Py_ssize_t ob_size; /* Number of items in variable part */
+
+typedef struct _object {
+    PyObject_HEAD
+} PyObject;
+
+typedef struct {
+	PyObject_VAR_HEAD
+} PyVarObject;
+
+struct _typeobject;
+
+typedef void (*freefunc)(void *);
+typedef void (*destructor)(PyObject *);
+typedef int (*printfunc)(PyObject *, FILE *, int);
+typedef PyObject *(*getattrfunc)(PyObject *, char *);
+typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
+typedef int (*setattrfunc)(PyObject *, char *, PyObject *);
+typedef int (*setattrofunc)(PyObject *, PyObject *, PyObject *);
+typedef int (*cmpfunc)(PyObject *, PyObject *);
+typedef PyObject *(*reprfunc)(PyObject *);
+typedef long (*hashfunc)(PyObject *);
+typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
+typedef PyObject *(*getiterfunc) (PyObject *);
+typedef PyObject *(*iternextfunc) (PyObject *);
+typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);
+typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *);
+typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
+typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
+typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t);
+
+typedef PyObject * (*unaryfunc)(PyObject *);
+typedef PyObject * (*binaryfunc)(PyObject *, PyObject *);
+typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
+typedef int (*inquiry)(PyObject *);
+typedef Py_ssize_t (*lenfunc)(PyObject *);
+typedef int (*coercion)(PyObject **, PyObject **);
+typedef PyObject *(*intargfunc)(PyObject *, int);
+typedef PyObject *(*intintargfunc)(PyObject *, int, int);
+typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
+typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
+typedef int(*intobjargproc)(PyObject *, int, PyObject *);
+typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *);
+typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
+typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
+typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
+
+
+/* int-based buffer interface */
+typedef int (*getreadbufferproc)(PyObject *, int, void **);
+typedef int (*getwritebufferproc)(PyObject *, int, void **);
+typedef int (*getsegcountproc)(PyObject *, int *);
+typedef int (*getcharbufferproc)(PyObject *, int, char **);
+/* ssize_t-based buffer interface */
+typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
+typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
+typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
+typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
+
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 128
+typedef struct bufferinfo {
+    void *buf;
+    PyObject *obj;        /* owned reference */
+    Py_ssize_t len;
+
+    /* This is Py_ssize_t so it can be
+       pointed to by strides in simple case.*/
+    Py_ssize_t itemsize;
+    int readonly;
+    int ndim;
+    char *format;
+    Py_ssize_t *shape;
+    Py_ssize_t *strides;
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format[Py_MAX_FMT];
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
+    /* static store for shape and strides of
+       mono-dimensional buffers. */
+    /* Py_ssize_t smalltable[2]; */
+    void *internal; /* always NULL for app-level objects */
+} Py_buffer;
+
+
+typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
+typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
+/* end Py3k buffer interface */
+
+typedef int (*objobjproc)(PyObject *, PyObject *);
+typedef int (*visitproc)(PyObject *, void *);
+typedef int (*traverseproc)(PyObject *, visitproc, void *);
+
+typedef struct {
+	/* For numbers without flag bit Py_TPFLAGS_CHECKTYPES set, all
+	   arguments are guaranteed to be of the object's type (modulo
+	   coercion hacks -- i.e. if the type's coercion function
+	   returns other types, then these are allowed as well).  Numbers that
+	   have the Py_TPFLAGS_CHECKTYPES flag bit set should check *both*
+	   arguments for proper type and implement the necessary conversions
+	   in the slot functions themselves. */
+
+	binaryfunc nb_add;
+	binaryfunc nb_subtract;
+	binaryfunc nb_multiply;
+	binaryfunc nb_divide;
+	binaryfunc nb_remainder;
+	binaryfunc nb_divmod;
+	ternaryfunc nb_power;
+	unaryfunc nb_negative;
+	unaryfunc nb_positive;
+	unaryfunc nb_absolute;
+	inquiry nb_nonzero;
+	unaryfunc nb_invert;
+	binaryfunc nb_lshift;
+	binaryfunc nb_rshift;
+	binaryfunc nb_and;
+	binaryfunc nb_xor;
+	binaryfunc nb_or;
+	coercion nb_coerce;
+	unaryfunc nb_int;
+	unaryfunc nb_long;
+	unaryfunc nb_float;
+	unaryfunc nb_oct;
+	unaryfunc nb_hex;
+	/* Added in release 2.0 */
+	binaryfunc nb_inplace_add;
+	binaryfunc nb_inplace_subtract;
+	binaryfunc nb_inplace_multiply;
+	binaryfunc nb_inplace_divide;
+	binaryfunc nb_inplace_remainder;
+	ternaryfunc nb_inplace_power;
+	binaryfunc nb_inplace_lshift;
+	binaryfunc nb_inplace_rshift;
+	binaryfunc nb_inplace_and;
+	binaryfunc nb_inplace_xor;
+	binaryfunc nb_inplace_or;
+
+	/* Added in release 2.2 */
+	/* The following require the Py_TPFLAGS_HAVE_CLASS flag */
+	binaryfunc nb_floor_divide;
+	binaryfunc nb_true_divide;
+	binaryfunc nb_inplace_floor_divide;
+	binaryfunc nb_inplace_true_divide;
+
+	/* Added in release 2.5 */
+	unaryfunc nb_index;
+} PyNumberMethods;
+
+typedef struct {
+	lenfunc sq_length;
+	binaryfunc sq_concat;
+	ssizeargfunc sq_repeat;
+	ssizeargfunc sq_item;
+	ssizessizeargfunc sq_slice;
+	ssizeobjargproc sq_ass_item;
+	ssizessizeobjargproc sq_ass_slice;
+	objobjproc sq_contains;
+	/* Added in release 2.0 */
+	binaryfunc sq_inplace_concat;
+	ssizeargfunc sq_inplace_repeat;
+} PySequenceMethods;
+
+typedef struct {
+	lenfunc mp_length;
+	binaryfunc mp_subscript;
+	objobjargproc mp_ass_subscript;
+} PyMappingMethods;
+
+typedef struct {
+	readbufferproc bf_getreadbuffer;
+	writebufferproc bf_getwritebuffer;
+	segcountproc bf_getsegcount;
+	charbufferproc bf_getcharbuffer;
+	getbufferproc bf_getbuffer;
+	releasebufferproc bf_releasebuffer;
+} PyBufferProcs;
+
+/* from descrobject.h */
+typedef PyObject *(*getter)(PyObject *, void *);
+typedef int (*setter)(PyObject *, PyObject *, void *);
+
+typedef struct PyGetSetDef {
+	char *name;
+	getter get;
+	setter set;
+	char *doc;
+	void *closure;
+} PyGetSetDef;
+
+/* from methodobject.h */
+typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
+typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
+                                             PyObject *);
+typedef PyObject *(*PyNoArgsFunction)(PyObject *);
+
+struct PyMethodDef {
+    const char  *ml_name;   /* The name of the built-in function/method */
+    PyCFunction  ml_meth;   /* The C function that implements it */
+    int          ml_flags;  /* Combination of METH_xxx flags, which mostly
+                               describe the args expected by the C func */
+    const char  *ml_doc;    /* The __doc__ attribute, or NULL */
+};
+typedef struct PyMethodDef PyMethodDef;
+
+/* from structmember.h */
+typedef struct PyMemberDef {
+    /* Current version, use this */
+    char *name;
+    int type;
+    Py_ssize_t offset;
+    int flags;
+    char *doc;
+} PyMemberDef;
+
+
+typedef struct _typeobject {
+	PyObject_VAR_HEAD
+	/* const */ char *tp_name; /* For printing, in format "<module>.<name>" */
+	Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
+
+	/* Methods to implement standard operations */
+
+	destructor tp_dealloc;
+	printfunc tp_print;
+	getattrfunc tp_getattr;
+	setattrfunc tp_setattr;
+	cmpfunc tp_compare;
+	reprfunc tp_repr;
+
+	/* Method suites for standard classes */
+
+	PyNumberMethods *tp_as_number;
+	PySequenceMethods *tp_as_sequence;
+	PyMappingMethods *tp_as_mapping;
+
+	/* More standard operations (here for binary compatibility) */
+
+	hashfunc tp_hash;
+	ternaryfunc tp_call;
+	reprfunc tp_str;
+	getattrofunc tp_getattro;
+	setattrofunc tp_setattro;
+
+	/* Functions to access object as input/output buffer */
+	PyBufferProcs *tp_as_buffer;
+
+	/* Flags to define presence of optional/expanded features */
+	long tp_flags;
+
+	/*const*/ char *tp_doc; /* Documentation string */
+
+	/* Assigned meaning in release 2.0 */
+	/* call function for all accessible objects */
+	traverseproc tp_traverse;
+
+	/* delete references to contained objects */
+	inquiry tp_clear;
+
+	/* Assigned meaning in release 2.1 */
+	/* rich comparisons */
+	richcmpfunc tp_richcompare;
+
+	/* weak reference enabler */
+	Py_ssize_t tp_weaklistoffset;
+
+	/* Added in release 2.2 */
+	/* Iterators */
+	getiterfunc tp_iter;
+	iternextfunc tp_iternext;
+
+	/* Attribute descriptor and subclassing stuff */
+	struct PyMethodDef *tp_methods;
+	struct PyMemberDef *tp_members;
+	struct PyGetSetDef *tp_getset;
+	struct _typeobject *tp_base;
+	PyObject *tp_dict;
+	descrgetfunc tp_descr_get;
+	descrsetfunc tp_descr_set;
+	Py_ssize_t tp_dictoffset;
+	initproc tp_init;
+	allocfunc tp_alloc;
+	newfunc tp_new;
+	freefunc tp_free; /* Low-level free-memory routine */
+	inquiry tp_is_gc; /* For PyObject_IS_GC */
+	PyObject *tp_bases;
+	PyObject *tp_mro; /* method resolution order */
+	PyObject *tp_cache;
+	PyObject *tp_subclasses;
+	PyObject *tp_weaklist;
+	destructor tp_del;
+
+	/* Type attribute cache version tag. Added in version 2.6 */
+	unsigned int tp_version_tag;
+
+} PyTypeObject;
+

From pypy.commits at gmail.com  Fri Jan 13 12:16:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 09:16:35 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix the deadlock in one test
Message-ID: <58790b73.43e61c0a.451d3.5f1c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89550:d0f316bdcdd6
Date: 2017-01-13 18:15 +0100
http://bitbucket.org/pypy/pypy/changeset/d0f316bdcdd6/

Log:	Fix the deadlock in one test

diff --git a/lib-python/3/test/test_cmd_line_script.py b/lib-python/3/test/test_cmd_line_script.py
--- a/lib-python/3/test/test_cmd_line_script.py
+++ b/lib-python/3/test/test_cmd_line_script.py
@@ -189,8 +189,8 @@
         try:
             # Drain stderr until prompt
             while True:
-                data = stderr.read(4)
-                if data == b">>> ":
+                data = stderr.read(len(sys.ps1))
+                if data == sys.ps1.encode('utf-8'):
                     break
                 stderr.readline()
             yield p
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -138,7 +138,7 @@
     RegrTest('test_cmath.py', core=True),
     RegrTest('test_cmd.py'),
     RegrTest('test_cmd_line.py'),
-    RegrTest('test_cmd_line_script.py', skip="XXX: deadlocks?"),
+    RegrTest('test_cmd_line_script.py'),
     RegrTest('test_code.py', core=True),
     RegrTest('test_code_module.py'),
     RegrTest('test_codeccallbacks.py', core=True),

From pypy.commits at gmail.com  Fri Jan 13 12:42:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 13 Jan 2017 09:42:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Really fix the test. Skip it in case
 there is a pypy-only mixture of stdout and stderr
Message-ID: <5879119c.43e61c0a.451d3.713d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89551:5614377fb5fc
Date: 2017-01-13 18:42 +0100
http://bitbucket.org/pypy/pypy/changeset/5614377fb5fc/

Log:	Really fix the test. Skip it in case there is a pypy-only mixture of
	stdout and stderr

diff --git a/lib-python/3/test/test_cmd_line_script.py b/lib-python/3/test/test_cmd_line_script.py
--- a/lib-python/3/test/test_cmd_line_script.py
+++ b/lib-python/3/test/test_cmd_line_script.py
@@ -188,11 +188,21 @@
             stderr = p.stdout
         try:
             # Drain stderr until prompt
-            while True:
-                data = stderr.read(len(sys.ps1))
-                if data == sys.ps1.encode('utf-8'):
-                    break
-                stderr.readline()
+            if support.check_impl_detail(pypy=True):
+                ps1 = b">>>> "
+                # PyPy: the prompt is still printed to stdout, like it
+                # is in CPython 2.7.  This messes up the logic below
+                # if stdout and stderr are different.  Skip for now.
+                if separate_stderr:
+                    self.skipTest("the prompt is still written to "
+                                  "stdout in pypy")
+            else:
+                ps1 = b">>> "
+            # logic fixed to support the case of lines that are shorter
+            # than len(ps1) characters
+            got = b""
+            while not got.endswith(ps1):
+                got += stderr.read(1)
             yield p
         finally:
             kill_python(p)

From pypy.commits at gmail.com  Fri Jan 13 13:13:42 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 13 Jan 2017 10:13:42 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Remove duplication between
 cpyext_object.h and the regular headers
Message-ID: <587918d6.0e821c0a.63665.8387@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89552:7d32ade95762
Date: 2017-01-13 18:12 +0000
http://bitbucket.org/pypy/pypy/changeset/7d32ade95762/

Log:	Remove duplication between cpyext_object.h and the regular headers

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -51,6 +51,7 @@
 translator_c_dir = py.path.local(cdir)
 include_dirs = [
     include_dir,
+    parse_dir,
     translator_c_dir,
     udir,
     ]
diff --git a/pypy/module/cpyext/include/descrobject.h b/pypy/module/cpyext/include/descrobject.h
--- a/pypy/module/cpyext/include/descrobject.h
+++ b/pypy/module/cpyext/include/descrobject.h
@@ -1,16 +1,5 @@
 #ifndef Py_DESCROBJECT_H
 #define Py_DESCROBJECT_H
-typedef PyObject *(*getter)(PyObject *, void *);
-typedef int (*setter)(PyObject *, PyObject *, void *);
-
-typedef struct PyGetSetDef {
-	char *name;
-	getter get;
-	setter set;
-	char *doc;
-	void *closure;
-} PyGetSetDef;
-
 
 #define PyDescr_COMMON \
     PyObject_HEAD \
diff --git a/pypy/module/cpyext/include/methodobject.h b/pypy/module/cpyext/include/methodobject.h
--- a/pypy/module/cpyext/include/methodobject.h
+++ b/pypy/module/cpyext/include/methodobject.h
@@ -7,20 +7,6 @@
 extern "C" {
 #endif
 
-typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
-typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
-                                             PyObject *);
-typedef PyObject *(*PyNoArgsFunction)(PyObject *);
-
-struct PyMethodDef {
-    const char  *ml_name;   /* The name of the built-in function/method */
-    PyCFunction  ml_meth;   /* The C function that implements it */
-    int          ml_flags;  /* Combination of METH_xxx flags, which mostly
-                               describe the args expected by the C func */
-    const char  *ml_doc;    /* The __doc__ attribute, or NULL */
-};
-typedef struct PyMethodDef PyMethodDef;
-
 typedef struct
 {
     PyObject_HEAD
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -7,6 +7,9 @@
 extern "C" {
 #endif
 
+
+#include <cpyext_object.h>
+
 #define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
 
 
@@ -16,29 +19,12 @@
 */
 #define staticforward static
 
-#define PyObject_HEAD  \
-    Py_ssize_t ob_refcnt;        \
-    Py_ssize_t ob_pypy_link;     \
-    struct _typeobject *ob_type;
-
-#define PyObject_VAR_HEAD		\
-	PyObject_HEAD			\
-	Py_ssize_t ob_size; /* Number of items in variable part */
-
 #define PyObject_HEAD_INIT(type)	\
 	1, 0, type,
 
 #define PyVarObject_HEAD_INIT(type, size)	\
 	PyObject_HEAD_INIT(type) size,
 
-typedef struct _object {
-    PyObject_HEAD
-} PyObject;
-
-typedef struct {
-	PyObject_VAR_HEAD
-} PyVarObject;
-
 #ifdef PYPY_DEBUG_REFCOUNT
 /* Slow version, but useful for debugging */
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)(ob)))
@@ -94,84 +80,7 @@
 #define Py_GT 4
 #define Py_GE 5
 
-struct _typeobject;
-typedef void (*freefunc)(void *);
-typedef void (*destructor)(PyObject *);
-typedef int (*printfunc)(PyObject *, FILE *, int);
-typedef PyObject *(*getattrfunc)(PyObject *, char *);
-typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
-typedef int (*setattrfunc)(PyObject *, char *, PyObject *);
-typedef int (*setattrofunc)(PyObject *, PyObject *, PyObject *);
-typedef int (*cmpfunc)(PyObject *, PyObject *);
-typedef PyObject *(*reprfunc)(PyObject *);
-typedef long (*hashfunc)(PyObject *);
-typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
-typedef PyObject *(*getiterfunc) (PyObject *);
-typedef PyObject *(*iternextfunc) (PyObject *);
-typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);
-typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *);
-typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
-typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
-typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t);
-
-typedef PyObject * (*unaryfunc)(PyObject *);
-typedef PyObject * (*binaryfunc)(PyObject *, PyObject *);
-typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
-typedef int (*inquiry)(PyObject *);
-typedef Py_ssize_t (*lenfunc)(PyObject *);
-typedef int (*coercion)(PyObject **, PyObject **);
-typedef PyObject *(*intargfunc)(PyObject *, int) Py_DEPRECATED(2.5);
-typedef PyObject *(*intintargfunc)(PyObject *, int, int) Py_DEPRECATED(2.5);
-typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
-typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
-typedef int(*intobjargproc)(PyObject *, int, PyObject *);
-typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *);
-typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
-typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
-typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
-
-
-/* int-based buffer interface */
-typedef int (*getreadbufferproc)(PyObject *, int, void **);
-typedef int (*getwritebufferproc)(PyObject *, int, void **);
-typedef int (*getsegcountproc)(PyObject *, int *);
-typedef int (*getcharbufferproc)(PyObject *, int, char **);
-/* ssize_t-based buffer interface */
-typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
-typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
-typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
-typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
-
 /* Py3k buffer interface, adapted for PyPy */
-#define Py_MAX_NDIMS 32
-#define Py_MAX_FMT 128
-typedef struct bufferinfo {
-    void *buf;
-    PyObject *obj;        /* owned reference */
-    Py_ssize_t len;
-
-    /* This is Py_ssize_t so it can be
-       pointed to by strides in simple case.*/
-    Py_ssize_t itemsize;
-    int readonly;
-    int ndim;
-    char *format;
-    Py_ssize_t *shape;
-    Py_ssize_t *strides;
-    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
-    unsigned char _format[Py_MAX_FMT];
-    Py_ssize_t _strides[Py_MAX_NDIMS];
-    Py_ssize_t _shape[Py_MAX_NDIMS];
-    /* static store for shape and strides of
-       mono-dimensional buffers. */
-    /* Py_ssize_t smalltable[2]; */
-    void *internal; /* always NULL for app-level objects */
-} Py_buffer;
-
-
-typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
-typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
-
     /* Flags for getting buffers */
 #define PyBUF_SIMPLE 0
 #define PyBUF_WRITABLE 0x0001
@@ -203,178 +112,6 @@
 #define PyBUF_SHADOW 0x400
 /* end Py3k buffer interface */
 
-typedef int (*objobjproc)(PyObject *, PyObject *);
-typedef int (*visitproc)(PyObject *, void *);
-typedef int (*traverseproc)(PyObject *, visitproc, void *);
-
-typedef struct {
-	/* For numbers without flag bit Py_TPFLAGS_CHECKTYPES set, all
-	   arguments are guaranteed to be of the object's type (modulo
-	   coercion hacks -- i.e. if the type's coercion function
-	   returns other types, then these are allowed as well).  Numbers that
-	   have the Py_TPFLAGS_CHECKTYPES flag bit set should check *both*
-	   arguments for proper type and implement the necessary conversions
-	   in the slot functions themselves. */
-
-	binaryfunc nb_add;
-	binaryfunc nb_subtract;
-	binaryfunc nb_multiply;
-	binaryfunc nb_divide;
-	binaryfunc nb_remainder;
-	binaryfunc nb_divmod;
-	ternaryfunc nb_power;
-	unaryfunc nb_negative;
-	unaryfunc nb_positive;
-	unaryfunc nb_absolute;
-	inquiry nb_nonzero;
-	unaryfunc nb_invert;
-	binaryfunc nb_lshift;
-	binaryfunc nb_rshift;
-	binaryfunc nb_and;
-	binaryfunc nb_xor;
-	binaryfunc nb_or;
-	coercion nb_coerce;
-	unaryfunc nb_int;
-	unaryfunc nb_long;
-	unaryfunc nb_float;
-	unaryfunc nb_oct;
-	unaryfunc nb_hex;
-	/* Added in release 2.0 */
-	binaryfunc nb_inplace_add;
-	binaryfunc nb_inplace_subtract;
-	binaryfunc nb_inplace_multiply;
-	binaryfunc nb_inplace_divide;
-	binaryfunc nb_inplace_remainder;
-	ternaryfunc nb_inplace_power;
-	binaryfunc nb_inplace_lshift;
-	binaryfunc nb_inplace_rshift;
-	binaryfunc nb_inplace_and;
-	binaryfunc nb_inplace_xor;
-	binaryfunc nb_inplace_or;
-
-	/* Added in release 2.2 */
-	/* The following require the Py_TPFLAGS_HAVE_CLASS flag */
-	binaryfunc nb_floor_divide;
-	binaryfunc nb_true_divide;
-	binaryfunc nb_inplace_floor_divide;
-	binaryfunc nb_inplace_true_divide;
-
-	/* Added in release 2.5 */
-	unaryfunc nb_index;
-} PyNumberMethods;
-
-typedef struct {
-	lenfunc sq_length;
-	binaryfunc sq_concat;
-	ssizeargfunc sq_repeat;
-	ssizeargfunc sq_item;
-	ssizessizeargfunc sq_slice;
-	ssizeobjargproc sq_ass_item;
-	ssizessizeobjargproc sq_ass_slice;
-	objobjproc sq_contains;
-	/* Added in release 2.0 */
-	binaryfunc sq_inplace_concat;
-	ssizeargfunc sq_inplace_repeat;
-} PySequenceMethods;
-
-typedef struct {
-	lenfunc mp_length;
-	binaryfunc mp_subscript;
-	objobjargproc mp_ass_subscript;
-} PyMappingMethods;
-
-typedef struct {
-	readbufferproc bf_getreadbuffer;
-	writebufferproc bf_getwritebuffer;
-	segcountproc bf_getsegcount;
-	charbufferproc bf_getcharbuffer;
-	getbufferproc bf_getbuffer;
-	releasebufferproc bf_releasebuffer;
-} PyBufferProcs;
-
-
-
-typedef struct _typeobject {
-	PyObject_VAR_HEAD
-	const char *tp_name; /* For printing, in format "<module>.<name>" */
-	Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
-
-	/* Methods to implement standard operations */
-
-	destructor tp_dealloc;
-	printfunc tp_print;
-	getattrfunc tp_getattr;
-	setattrfunc tp_setattr;
-	cmpfunc tp_compare;
-	reprfunc tp_repr;
-
-	/* Method suites for standard classes */
-
-	PyNumberMethods *tp_as_number;
-	PySequenceMethods *tp_as_sequence;
-	PyMappingMethods *tp_as_mapping;
-
-	/* More standard operations (here for binary compatibility) */
-
-	hashfunc tp_hash;
-	ternaryfunc tp_call;
-	reprfunc tp_str;
-	getattrofunc tp_getattro;
-	setattrofunc tp_setattro;
-
-	/* Functions to access object as input/output buffer */
-	PyBufferProcs *tp_as_buffer;
-
-	/* Flags to define presence of optional/expanded features */
-	long tp_flags;
-
-	const char *tp_doc; /* Documentation string */
-
-	/* Assigned meaning in release 2.0 */
-	/* call function for all accessible objects */
-	traverseproc tp_traverse;
-
-	/* delete references to contained objects */
-	inquiry tp_clear;
-
-	/* Assigned meaning in release 2.1 */
-	/* rich comparisons */
-	richcmpfunc tp_richcompare;
-
-	/* weak reference enabler */
-	Py_ssize_t tp_weaklistoffset;
-
-	/* Added in release 2.2 */
-	/* Iterators */
-	getiterfunc tp_iter;
-	iternextfunc tp_iternext;
-
-	/* Attribute descriptor and subclassing stuff */
-	struct PyMethodDef *tp_methods;
-	struct PyMemberDef *tp_members;
-	struct PyGetSetDef *tp_getset;
-	struct _typeobject *tp_base;
-	PyObject *tp_dict;
-	descrgetfunc tp_descr_get;
-	descrsetfunc tp_descr_set;
-	Py_ssize_t tp_dictoffset;
-	initproc tp_init;
-	allocfunc tp_alloc;
-	newfunc tp_new;
-	freefunc tp_free; /* Low-level free-memory routine */
-	inquiry tp_is_gc; /* For PyObject_IS_GC */
-	PyObject *tp_bases;
-	PyObject *tp_mro; /* method resolution order */
-	PyObject *tp_cache;
-	PyObject *tp_subclasses;
-	PyObject *tp_weaklist;
-	destructor tp_del;
-
-	/* Type attribute cache version tag. Added in version 2.6 */
-	unsigned int tp_version_tag;
-
-} PyTypeObject;
-
 typedef struct {
     PyTypeObject ht_type;
     PyNumberMethods as_number;
diff --git a/pypy/module/cpyext/include/structmember.h b/pypy/module/cpyext/include/structmember.h
--- a/pypy/module/cpyext/include/structmember.h
+++ b/pypy/module/cpyext/include/structmember.h
@@ -19,22 +19,6 @@
 #define offsetof(type, member) ( (int) & ((type*)0) -> member )
 #endif
 
-/* An array of memberlist structures defines the name, type and offset
-   of selected members of a C structure.  These can be read by
-   PyMember_Get() and set by PyMember_Set() (except if their READONLY flag
-   is set).  The array must be terminated with an entry whose name
-   pointer is NULL. */
-
-
-
-typedef struct PyMemberDef {
-    /* Current version, use this */
-    char *name;
-    int type;
-    Py_ssize_t offset;
-    int flags;
-    char *doc;
-} PyMemberDef;
 
 /* Types */
 #define T_SHORT         0
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -19,7 +19,6 @@
 } PyVarObject;
 
 struct _typeobject;
-
 typedef void (*freefunc)(void *);
 typedef void (*destructor)(PyObject *);
 typedef int (*printfunc)(PyObject *, FILE *, int);

From pypy.commits at gmail.com  Sat Jan 14 03:39:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 00:39:21 -0800 (PST)
Subject: [pypy-commit] pypy issue2464: try a fix for issue 2464
Message-ID: <5879e3b9.cb911c0a.619ed.8118@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: issue2464
Changeset: r89553:000177660dd7
Date: 2017-01-14 09:18 +0100
http://bitbucket.org/pypy/pypy/changeset/000177660dd7/

Log:	try a fix for issue 2464


From pypy.commits at gmail.com  Sat Jan 14 03:39:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 00:39:23 -0800 (PST)
Subject: [pypy-commit] pypy issue2464: Found a generic way to solve the
 __objclass__ problem
Message-ID: <5879e3bb.8a9a1c0a.46f5d.f4f1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: issue2464
Changeset: r89554:2d4f56fdba06
Date: 2017-01-14 09:38 +0100
http://bitbucket.org/pypy/pypy/changeset/2d4f56fdba06/

Log:	Found a generic way to solve the __objclass__ problem

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -276,12 +276,15 @@
         self.use_closure = use_closure
 
     def copy_for_type(self, w_objclass):
-        new = instantiate(GetSetProperty)
-        new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
-                  None, self.use_closure)
-        new.name = self.name
-        new.w_objclass = w_objclass
-        return new
+        if self.objclass_getter is None:
+            new = instantiate(GetSetProperty)
+            new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
+                      None, self.use_closure)
+            new.name = self.name
+            new.w_objclass = w_objclass
+            return new
+        else:
+            return self
 
     @unwrap_spec(w_cls = WrappedDefault(None))
     def descr_property_get(self, space, w_obj, w_cls=None):
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1329,3 +1329,6 @@
             pass
         assert X.__dict__['__dict__'].__objclass__ is X
         assert X.__dict__['__weakref__'].__objclass__ is X
+        assert object.__dict__['__class__'].__objclass__ is object
+        assert int.__dict__['imag'].__objclass__ is int
+        assert file.closed.__objclass__ is file
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1312,10 +1312,13 @@
     def build(self, typedef):
         "NOT_RPYTHON: initialization-time only."
         from pypy.objspace.std.objectobject import W_ObjectObject
+        from pypy.interpreter.typedef import GetSetProperty
+        from rpython.rlib.objectmodel import instantiate
 
         space = self.space
         rawdict = typedef.rawdict
         lazyloaders = {}
+        w_type = instantiate(W_TypeObject)
 
         # compute the bases
         if typedef is W_ObjectObject.typedef:
@@ -1327,13 +1330,16 @@
         # wrap everything
         dict_w = {}
         for descrname, descrvalue in rawdict.items():
+            # special case for GetSetProperties' __objclass__:
+            if isinstance(descrvalue, GetSetProperty):
+                descrvalue = descrvalue.copy_for_type(w_type)
             dict_w[descrname] = space.wrap(descrvalue)
 
         if typedef.applevel_subclasses_base is not None:
             overridetypedef = typedef.applevel_subclasses_base.typedef
         else:
             overridetypedef = typedef
-        w_type = W_TypeObject(space, typedef.name, bases_w, dict_w,
+        w_type.__init__(space, typedef.name, bases_w, dict_w,
                               overridetypedef=overridetypedef,
                               is_heaptype=overridetypedef.heaptype)
         if typedef is not overridetypedef:

From pypy.commits at gmail.com  Sat Jan 14 04:08:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 01:08:00 -0800 (PST)
Subject: [pypy-commit] pypy default: A WeakSet is not necessary: just copy
 the list of weakrefs. We don't
Message-ID: <5879ea70.849c1c0a.dd8a.6571@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89555:15abf75346b2
Date: 2017-01-14 10:07 +0100
http://bitbucket.org/pypy/pypy/changeset/15abf75346b2/

Log:	A WeakSet is not necessary: just copy the list of weakrefs. We don't
	need extra logic to remember statements in previous commits, because
	they are still in self.__statements now unless their weakref died.

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -220,7 +220,7 @@
         self.__statements_counter = 0
         self.__rawstatements = set()
         self._statement_cache = _StatementCache(self, cached_statements)
-        self.__statements_already_committed = weakref.WeakSet()
+        self.__statements_already_committed = []
 
         self.__func_cache = {}
         self.__aggregates = {}
@@ -365,10 +365,12 @@
                     cursor._reset = True
 
     def _reset_already_committed_statements(self):
-        lst = list(self.__statements_already_committed)
-        self.__statements_already_committed.clear()
-        for statement in lst:
-            statement._reset()
+        lst = self.__statements_already_committed
+        self.__statements_already_committed = []
+        for weakref in lst:
+            statement = weakref()
+            if statement is not None:
+                statement._reset()
 
     @_check_thread_wrap
     @_check_closed_wrap
@@ -432,14 +434,11 @@
         # the way and cause the "drop table" to fail.  On CPython the
         # problem is much less important because typically all the old
         # statements are freed already by reference counting.  So here,
-        # we add all the still-alive statements to a WeakSet which is
-        # usually ignored, except if we get SQLITE_LOCKED
+        # we copy all the still-alive statements to another list which
+        # is usually ignored, except if we get SQLITE_LOCKED
         # afterwards---at which point we reset all statements in this
-        # WeakSet.
-        for weakref in self.__statements:
-            statement = weakref()
-            if statement is not None:
-                self.__statements_already_committed.add(statement)
+        # list.
+        self.__statements_already_committed = self.__statements[:]
 
         statement_star = _ffi.new('sqlite3_stmt **')
         ret = _lib.sqlite3_prepare_v2(self._db, b"COMMIT", -1,

From pypy.commits at gmail.com  Sat Jan 14 08:28:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 05:28:09 -0800 (PST)
Subject: [pypy-commit] pypy issue2464: fix test
Message-ID: <587a2769.c4811c0a.28521.1327@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: issue2464
Changeset: r89556:899294ce2166
Date: 2017-01-14 14:26 +0100
http://bitbucket.org/pypy/pypy/changeset/899294ce2166/

Log:	fix test

diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -140,7 +140,11 @@
             pass
         def fget(self, space, w_self):
             assert self is prop
-        prop = typedef.GetSetProperty(fget, use_closure=True)
+        # NB. this GetSetProperty is not copied when creating the
+        # W_TypeObject because of 'cls'.  Without it, a duplicate of the
+        # GetSetProperty is taken and it is given the w_objclass that is
+        # the W_TypeObject
+        prop = typedef.GetSetProperty(fget, use_closure=True, cls=W_SomeType)
         W_SomeType.typedef = typedef.TypeDef(
             'some_type',
             x=prop)

From pypy.commits at gmail.com  Sat Jan 14 08:28:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 05:28:16 -0800 (PST)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <587a2770.85e11c0a.9bfc0.17de@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89559:9971839b05e4
Date: 2017-01-14 14:27 +0100
http://bitbucket.org/pypy/pypy/changeset/9971839b05e4/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -101,3 +101,7 @@
 
 Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to
 the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak.
+
+.. branch: issue2464
+
+Give (almost?) all GetSetProperties a valid __objclass__.

From pypy.commits at gmail.com  Sat Jan 14 08:28:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 05:28:11 -0800 (PST)
Subject: [pypy-commit] pypy issue2464: ready to merge
Message-ID: <587a276b.92ae1c0a.f389e.1817@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: issue2464
Changeset: r89557:0d931f4a7ea0
Date: 2017-01-14 14:26 +0100
http://bitbucket.org/pypy/pypy/changeset/0d931f4a7ea0/

Log:	ready to merge


From pypy.commits at gmail.com  Sat Jan 14 08:28:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 05:28:13 -0800 (PST)
Subject: [pypy-commit] pypy default: hg merge issue2464
Message-ID: <587a276d.9d711c0a.65f4a.1bbc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89558:c7cd66a19b5e
Date: 2017-01-14 14:26 +0100
http://bitbucket.org/pypy/pypy/changeset/c7cd66a19b5e/

Log:	hg merge issue2464

	Give (almost?) all GetSetProperties a valid __objclass__.

diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -140,7 +140,11 @@
             pass
         def fget(self, space, w_self):
             assert self is prop
-        prop = typedef.GetSetProperty(fget, use_closure=True)
+        # NB. this GetSetProperty is not copied when creating the
+        # W_TypeObject because of 'cls'.  Without it, a duplicate of the
+        # GetSetProperty is taken and it is given the w_objclass that is
+        # the W_TypeObject
+        prop = typedef.GetSetProperty(fget, use_closure=True, cls=W_SomeType)
         W_SomeType.typedef = typedef.TypeDef(
             'some_type',
             x=prop)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -276,12 +276,15 @@
         self.use_closure = use_closure
 
     def copy_for_type(self, w_objclass):
-        new = instantiate(GetSetProperty)
-        new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
-                  None, self.use_closure)
-        new.name = self.name
-        new.w_objclass = w_objclass
-        return new
+        if self.objclass_getter is None:
+            new = instantiate(GetSetProperty)
+            new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
+                      None, self.use_closure)
+            new.name = self.name
+            new.w_objclass = w_objclass
+            return new
+        else:
+            return self
 
     @unwrap_spec(w_cls = WrappedDefault(None))
     def descr_property_get(self, space, w_obj, w_cls=None):
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1329,3 +1329,6 @@
             pass
         assert X.__dict__['__dict__'].__objclass__ is X
         assert X.__dict__['__weakref__'].__objclass__ is X
+        assert object.__dict__['__class__'].__objclass__ is object
+        assert int.__dict__['imag'].__objclass__ is int
+        assert file.closed.__objclass__ is file
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1312,10 +1312,13 @@
     def build(self, typedef):
         "NOT_RPYTHON: initialization-time only."
         from pypy.objspace.std.objectobject import W_ObjectObject
+        from pypy.interpreter.typedef import GetSetProperty
+        from rpython.rlib.objectmodel import instantiate
 
         space = self.space
         rawdict = typedef.rawdict
         lazyloaders = {}
+        w_type = instantiate(W_TypeObject)
 
         # compute the bases
         if typedef is W_ObjectObject.typedef:
@@ -1327,13 +1330,16 @@
         # wrap everything
         dict_w = {}
         for descrname, descrvalue in rawdict.items():
+            # special case for GetSetProperties' __objclass__:
+            if isinstance(descrvalue, GetSetProperty):
+                descrvalue = descrvalue.copy_for_type(w_type)
             dict_w[descrname] = space.wrap(descrvalue)
 
         if typedef.applevel_subclasses_base is not None:
             overridetypedef = typedef.applevel_subclasses_base.typedef
         else:
             overridetypedef = typedef
-        w_type = W_TypeObject(space, typedef.name, bases_w, dict_w,
+        w_type.__init__(space, typedef.name, bases_w, dict_w,
                               overridetypedef=overridetypedef,
                               is_heaptype=overridetypedef.heaptype)
         if typedef is not overridetypedef:

From pypy.commits at gmail.com  Sat Jan 14 10:58:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 07:58:41 -0800 (PST)
Subject: [pypy-commit] pypy default: issue 2465: failing test
Message-ID: <587a4ab1.0b561c0a.ea73d.79af@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89560:3fa70d621b54
Date: 2017-01-14 15:50 +0100
http://bitbucket.org/pypy/pypy/changeset/3fa70d621b54/

Log:	issue 2465: failing test

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4620,3 +4620,19 @@
             time.clock()
             return 0
         self.interp_operations(g, [])
+
+    def test_issue2465(self):
+        driver = JitDriver(greens=[], reds=['i', 'a', 'b'])
+        class F(object):
+            def __init__(self, floatval):
+                self.floatval = floatval
+        def f(i):
+            a = F(0.0)
+            b = None
+            while i > 0:
+                driver.jit_merge_point(i=i, a=a, b=b)
+                b = F(a.floatval / 1.)
+                i -= 1
+            return i
+
+        self.meta_interp(f, [10])

From pypy.commits at gmail.com  Sat Jan 14 10:58:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 07:58:44 -0800 (PST)
Subject: [pypy-commit] pypy reverse-debugger: Need to force disabled modules
 to be false, else they may still be
Message-ID: <587a4ab4.85e11c0a.9bfc0.78dc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r89561:e90317857d27
Date: 2017-01-14 16:56 +0100
http://bitbucket.org/pypy/pypy/changeset/e90317857d27/

Log:	Need to force disabled modules to be false, else they may still be
	enabled by suggestions (and _vmprof is always suggested)

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -296,7 +296,8 @@
     if config.translation.sandbox:
         modules = default_modules
     if config.translation.reverse_debugger:
-        modules -= reverse_debugger_disable_modules
+        for mod in reverse_debugger_disable_modules:
+            setattr(config.objspace.usemodules, mod, False)
     # ignore names from 'essential_modules', notably 'exceptions', which
     # may not be present in config.objspace.usemodules at all
     modules = [name for name in modules if name not in essential_modules]

From pypy.commits at gmail.com  Sat Jan 14 11:21:40 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 14 Jan 2017 08:21:40 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: fix translation
Message-ID: <587a5014.09bb1c0a.27a9b.8ad5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89562:58a2143f6b9d
Date: 2017-01-14 16:20 +0000
http://bitbucket.org/pypy/pypy/changeset/58a2143f6b9d/

Log:	fix translation

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -148,6 +148,7 @@
     headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl')
     for name in ["pypy_macros.h"] + FUNCTIONS_BY_HEADER.keys():
         headers.append(udir.join(name))
+    headers.append(parse_dir / 'cpyext_object.h')
     _copy_header_files(headers, dstdir)
 
     if copy_numpy_headers:
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -749,6 +749,8 @@
                 self.add_macro(name, obj)
         self._Config._compilation_info_ = self.build_eci()
         for name, TYPE in rffi_platform.configure(self._Config).iteritems():
+            # hack: prevent the source from being pasted into common_header.h
+            del TYPE._hints['eci']
             if name in self._TYPES:
                 self._TYPES[name].become(TYPE)
 

From pypy.commits at gmail.com  Sat Jan 14 11:27:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 08:27:40 -0800 (PST)
Subject: [pypy-commit] pypy default: Hack to "fix" test_issue2465
Message-ID: <587a517c.4f831c0a.33253.93fe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89563:99a37d78d644
Date: 2017-01-14 17:27 +0100
http://bitbucket.org/pypy/pypy/changeset/99a37d78d644/

Log:	Hack to "fix" test_issue2465

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -350,6 +350,14 @@
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
                                       args=args + extra,
                                       descr=target_token))
+
+            # XXX see test_issue2465 for a case where the following occurs
+            # XXX (it would be better to really understand the problem
+            # XXX and fix it, but I am failing so far)
+            if label_op is not None and len(args + extra) != label_op.numargs():
+                del self.optimizer._newoperations[-1]    # remove the JUMP
+                raise InvalidLoop
+
             return None # explicit because the return can be non-None
         return virtual_state
 

From pypy.commits at gmail.com  Sat Jan 14 11:31:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jan 2017 08:31:43 -0800 (PST)
Subject: [pypy-commit] pypy reverse-debugger: Added tag RevDB-pypy2.7-v5.6.1
 for changeset e90317857d27
Message-ID: <587a526f.0d1a1c0a.2fc08.a24f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r89564:98de56f32171
Date: 2017-01-14 17:31 +0100
http://bitbucket.org/pypy/pypy/changeset/98de56f32171/

Log:	Added tag RevDB-pypy2.7-v5.6.1 for changeset e90317857d27

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -37,3 +37,4 @@
 4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
 d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1

From pypy.commits at gmail.com  Sat Jan 14 13:53:26 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 14 Jan 2017 10:53:26 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <587a73a6.92ae1c0a.f389e.04d6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89565:9c9b7e4614da
Date: 2017-01-14 20:24 +0200
http://bitbucket.org/pypy/pypy/changeset/9c9b7e4614da/

Log:	merge default into branch

diff too long, truncating to 2000 out of 2762 lines

diff --git a/lib-python/2.7/sqlite3/test/regression.py b/lib-python/2.7/sqlite3/test/regression.py
--- a/lib-python/2.7/sqlite3/test/regression.py
+++ b/lib-python/2.7/sqlite3/test/regression.py
@@ -351,10 +351,7 @@
         self.assertRaises(ValueError, cur.execute, " \0select 2")
         self.assertRaises(ValueError, cur.execute, "select 2\0")
 
-    @test_support.impl_detail(pypy=False)
     def CheckCommitCursorReset(self):
-        # This test is for logic added in 2.7.13 which PyPy doesn't
-        # implement.  See http://bugs.python.org/issue29006
         """
         Connection.commit() did reset cursors, which made sqlite3
         to return rows multiple times when fetched from cursors
diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py
--- a/lib-python/2.7/test/test_multiprocessing.py
+++ b/lib-python/2.7/test/test_multiprocessing.py
@@ -1969,9 +1969,10 @@
         if not gc.isenabled():
             gc.enable()
             self.addCleanup(gc.disable)
-        #thresholds = gc.get_threshold()
-        #self.addCleanup(gc.set_threshold, *thresholds)
-        #gc.set_threshold(10)
+        if test_support.check_impl_detail(cpython=True):
+            thresholds = gc.get_threshold()
+            self.addCleanup(gc.set_threshold, *thresholds)
+            gc.set_threshold(10)
 
         # perform numerous block allocations, with cyclic references to make
         # sure objects are collected asynchronously by the gc
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -220,6 +220,7 @@
         self.__statements_counter = 0
         self.__rawstatements = set()
         self._statement_cache = _StatementCache(self, cached_statements)
+        self.__statements_already_committed = weakref.WeakSet()
 
         self.__func_cache = {}
         self.__aggregates = {}
@@ -363,6 +364,12 @@
                 if cursor is not None:
                     cursor._reset = True
 
+    def _reset_already_committed_statements(self):
+        lst = list(self.__statements_already_committed)
+        self.__statements_already_committed.clear()
+        for statement in lst:
+            statement._reset()
+
     @_check_thread_wrap
     @_check_closed_wrap
     def __call__(self, sql):
@@ -418,15 +425,21 @@
         if not self._in_transaction:
             return
 
-        # The following line is a KNOWN DIFFERENCE with CPython 2.7.13.
-        # More precisely, the corresponding line was removed in the
-        # version 2.7.13 of CPython, but this is causing troubles for
-        # PyPy (and potentially for CPython too):
-        #
-        #     http://bugs.python.org/issue29006
-        #
-        # So for now, we keep this line.
-        self.__do_all_statements(Statement._reset, False)
+        # PyPy fix for non-refcounting semantics: since 2.7.13 (and in
+        # <= 2.6.x), the statements are not automatically reset upon
+        # commit.  However, if this is followed by some specific SQL
+        # operations like "drop table", these open statements come in
+        # the way and cause the "drop table" to fail.  On CPython the
+        # problem is much less important because typically all the old
+        # statements are freed already by reference counting.  So here,
+        # we add all the still-alive statements to a WeakSet which is
+        # usually ignored, except if we get SQLITE_LOCKED
+        # afterwards---at which point we reset all statements in this
+        # WeakSet.
+        for weakref in self.__statements:
+            statement = weakref()
+            if statement is not None:
+                self.__statements_already_committed.add(statement)
 
         statement_star = _ffi.new('sqlite3_stmt **')
         ret = _lib.sqlite3_prepare_v2(self._db, b"COMMIT", -1,
@@ -827,8 +840,18 @@
                 self.__statement._set_params(params)
 
                 # Actually execute the SQL statement
+
                 ret = _lib.sqlite3_step(self.__statement._statement)
 
+                # PyPy: if we get SQLITE_LOCKED, it's probably because
+                # one of the cursors created previously is still alive
+                # and not reset and the operation we're trying to do
+                # makes Sqlite unhappy about that.  In that case, we
+                # automatically reset all old cursors and try again.
+                if ret == _lib.SQLITE_LOCKED:
+                    self.__connection._reset_already_committed_statements()
+                    ret = _lib.sqlite3_step(self.__statement._statement)
+
                 if ret == _lib.SQLITE_ROW:
                     if multiple:
                         raise ProgrammingError("executemany() can only execute DML statements.")
diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -374,7 +374,7 @@
 
     sample_count = _sample_count(cp, size)
 
-    rv = ffi.new("unsigned char[]", len(cp) * 2)
+    rv = ffi.new("char[]", len(cp) * 2)
     lib.tostereo(rv, cp, len(cp), size, fac1, fac2)
     return ffi.buffer(rv)[:]
 
@@ -385,7 +385,7 @@
     if len(cp1) != len(cp2):
         raise error("Lengths should be the same")
 
-    rv = ffi.new("unsigned char[]", len(cp1))
+    rv = ffi.new("char[]", len(cp1))
     lib.add(rv, cp1, cp2, len(cp1), size)
     return ffi.buffer(rv)[:]
 
@@ -488,7 +488,7 @@
     ceiling = (q + 1) * outrate
     nbytes = ceiling * bytes_per_frame
 
-    rv = ffi.new("unsigned char[]", nbytes)
+    rv = ffi.new("char[]", nbytes)
     trim_index = lib.ratecv(rv, cp, frame_count, size,
                             nchannels, inrate, outrate,
                             state_d, prev_i, cur_i,
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -483,11 +483,6 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* The ``sqlite`` module was updated on 2.7.13 to no longer reset all
-  cursors when there is a commit.  This causes troubles for PyPy (and
-  potentially for CPython too), and so for now we didn't port this change:
-  see http://bugs.python.org/issue29006.
-
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -336,6 +336,6 @@
    that fixes some bugs and is translatable.
 
 *  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the files ``xxxtype.py`` and
-   ``xxxobject.py`` contain respectively the definition of the type and its
-   (default) implementation.
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -8,6 +8,7 @@
 
 from rpython.rlib.jit import promote
 from rpython.rlib.objectmodel import compute_identity_hash, specialize
+from rpython.rlib.objectmodel import instantiate
 from rpython.tool.sourcetools import compile2, func_with_new_name
 
 
@@ -221,10 +222,6 @@
     exec source.compile() in miniglobals
     return miniglobals['descr_typecheck_%s' % func.__name__]
 
-def unknown_objclass_getter(space):
-    # NB. this is an AttributeError to make inspect.py happy
-    raise oefmt(space.w_AttributeError, "generic property has no __objclass__")
-
 @specialize.arg(0)
 def make_objclass_getter(tag, func, cls):
     if func and hasattr(func, 'im_func'):
@@ -235,7 +232,7 @@
 @specialize.memo()
 def _make_objclass_getter(cls):
     if not cls:
-        return unknown_objclass_getter, cls
+        return None, cls
     miniglobals = {}
     if isinstance(cls, str):
         assert cls.startswith('<'), "pythontype typecheck should begin with <"
@@ -254,6 +251,8 @@
 
 class GetSetProperty(W_Root):
     _immutable_fields_ = ["fget", "fset", "fdel"]
+    name = '<generic property>'
+    w_objclass = None
 
     @specialize.arg(7)
     def __init__(self, fget, fset=None, fdel=None, doc=None,
@@ -265,15 +264,25 @@
                                             cls=cls, use_closure=use_closure)
         fdel = make_descr_typecheck_wrapper((tag, 2), fdel,
                                             cls=cls, use_closure=use_closure)
+        self._init(fget, fset, fdel, doc, cls, objclass_getter, use_closure)
+
+    def _init(self, fget, fset, fdel, doc, cls, objclass_getter, use_closure):
         self.fget = fget
         self.fset = fset
         self.fdel = fdel
         self.doc = doc
         self.reqcls = cls
-        self.name = '<generic property>'
         self.objclass_getter = objclass_getter
         self.use_closure = use_closure
 
+    def copy_for_type(self, w_objclass):
+        new = instantiate(GetSetProperty)
+        new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
+                  None, self.use_closure)
+        new.name = self.name
+        new.w_objclass = w_objclass
+        return new
+
     @unwrap_spec(w_cls = WrappedDefault(None))
     def descr_property_get(self, space, w_obj, w_cls=None):
         """property.__get__(obj[, type]) -> value
@@ -322,7 +331,14 @@
                                                space.wrap(self.name)]))
 
     def descr_get_objclass(space, property):
-        return property.objclass_getter(space)
+        if property.w_objclass is not None:
+            return property.w_objclass
+        if property.objclass_getter is not None:
+            return property.objclass_getter(space)
+        # NB. this is an AttributeError to make inspect.py happy
+        raise oefmt(space.w_AttributeError,
+                    "generic property has no __objclass__")
+
 
 def interp_attrproperty(name, cls, doc=None):
     "NOT_RPYTHON: initialization-time only"
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -23,6 +23,10 @@
 def pytest_ignore_collect(path, config):
     if py.path.local.sysfind('genreflex') is None and config.option.runappdirect:
         return True          # "can't run dummy tests in -A"
+    if disabled:
+        return True
+
+disabled = None
 
 def pytest_configure(config):
     if py.path.local.sysfind('genreflex') is None:
@@ -37,7 +41,7 @@
             # build dummy backend (which has reflex info and calls hard-wired)
             import os
             from rpython.translator.tool.cbuild import ExternalCompilationInfo
-            from rpython.translator.platform import platform
+            from rpython.translator.platform import platform, CompilationError
             from rpython.translator import cdir
 
             from rpython.rtyper.lltypesystem import rffi
@@ -55,9 +59,16 @@
                 use_cpp_linker=True,
             )
 
-            soname = platform.compile(
-                [], eci,
-                outputfilename='libcppyy_dummy_backend',
-                standalone=False)
+            try:
+                soname = platform.compile(
+                    [], eci,
+                    outputfilename='libcppyy_dummy_backend',
+                    standalone=False)
+            except CompilationError as e:
+                if '-std=c++11' in str(e):
+                    global disabled
+                    disabled = str(e)
+                    return
+                raise
 
             lcapi.reflection_library = str(soname)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -250,14 +250,13 @@
 cpyext_namespace = NameManager('cpyext_')
 
 class ApiFunction(object):
-    def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED,
+    def __init__(self, argtypes, restype, callable, error=CANNOT_FAIL,
                  c_name=None, gil=None, result_borrowed=False, result_is_ll=False):
         self.argtypes = argtypes
         self.restype = restype
         self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
         self.callable = callable
-        if error is not _NOT_SPECIFIED:
-            self.error_value = error
+        self.error_value = error
         self.c_name = c_name
 
         # extract the signature from the (CPython-level) code object
@@ -299,7 +298,7 @@
 
         argtypesw = zip(self.argtypes,
                         [_name.startswith("w_") for _name in self.argnames])
-        error_value = getattr(self, "error_value", CANNOT_FAIL)
+        error_value = self.error_value
         if (isinstance(self.restype, lltype.Ptr)
                 and error_value is not CANNOT_FAIL):
             assert lltype.typeOf(error_value) == self.restype
@@ -437,12 +436,12 @@
     def decorate(func):
         if func.__name__ in FUNCTIONS_BY_HEADER[header]:
             raise ValueError("%s already registered" % func.__name__)
-        api_function = _create_api_func(
-            func, argtypes, restype, error, gil=gil,
+        func._always_inline_ = 'try'
+        api_function = ApiFunction(
+            argtypes, restype, func,
+            error=_compute_error(error, restype), gil=gil,
             result_borrowed=result_borrowed, result_is_ll=result_is_ll)
-        unwrapper = api_function.get_unwrapper()
-        unwrapper.func = func
-        unwrapper.api_func = api_function
+        FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
 
         # ZZZ is this whole logic really needed???  It seems to be only
         # for RPython code calling PyXxx() functions directly.  I would
@@ -470,32 +469,33 @@
             assert got_integer == expect_integer, (
                 'got %r not integer' % (res,))
             return res
+        INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
 
-        if header is not None:
-            FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
-            INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
-        return unwrapper
-    return decorate
-
-def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
-    def decorate(func):
-        c_name = func.__name__
-        api_function = _create_api_func(func, argtypes, restype, error, c_name)
         unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
         return unwrapper
     return decorate
 
+def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
+    def decorate(func):
+        func._always_inline_ = 'try'
+        api_function = ApiFunction(
+            argtypes, restype, func,
+            error=_compute_error(error, restype),
+            c_name=func.__name__)
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
 
-def _create_api_func(
-        func, argtypes, restype, error=_NOT_SPECIFIED, c_name=None,
-        gil=None, result_borrowed=False, result_is_ll=False):
+def _compute_error(error, restype):
+    """Convert error specification to actual error value of type restype."""
     if isinstance(restype, lltype.Typedef):
         real_restype = restype.OF
     else:
         real_restype = restype
-
     if error is _NOT_SPECIFIED:
         if isinstance(real_restype, lltype.Ptr):
             error = lltype.nullptr(real_restype.TO)
@@ -503,11 +503,7 @@
             error = CANNOT_FAIL
     if type(error) is int:
         error = rffi.cast(real_restype, error)
-
-    func._always_inline_ = 'try'
-    return ApiFunction(
-        argtypes, restype, func, error, c_name=c_name, gil=gil,
-        result_borrowed=result_borrowed, result_is_ll=result_is_ll)
+    return error
 
 
 def cpython_struct(name, fields, forward=None, level=1):
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -118,12 +118,16 @@
                     """ % (type_name,)))
         except OperationError:
             return 0
+    return check, check_exact
 
-make_check_function("PyDateTime_Check", "datetime")
-make_check_function("PyDate_Check", "date")
-make_check_function("PyTime_Check", "time")
-make_check_function("PyDelta_Check", "timedelta")
-make_check_function("PyTZInfo_Check", "tzinfo")
+PyDateTime_Check, PyDateTime_CheckExact = make_check_function(
+    "PyDateTime_Check", "datetime")
+PyDate_Check, PyDate_CheckExact = make_check_function("PyDate_Check", "date")
+PyTime_Check, PyTime_CheckExact = make_check_function("PyTime_Check", "time")
+PyDelta_Check, PyDelta_CheckExact = make_check_function(
+    "PyDelta_Check", "timedelta")
+PyTZInfo_Check, PyTZInfo_CheckExact = make_check_function(
+    "PyTZInfo_Check", "tzinfo")
 
 # Constructors. They are better used as macros.
 
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,20 +1,22 @@
 import py, pytest
+import contextlib
 from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
-from pypy.module.cpyext import api
+from pypy.module.cpyext.api import (
+    slot_function, cpython_api, copy_header_files, INTERPLEVEL_API,
+    Py_ssize_t, Py_ssize_tP, PyObject)
 from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest
-PyObject = api.PyObject
 from pypy.interpreter.error import OperationError
 from rpython.rlib import rawrefcount
 import os
 
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetWrapped(space, w_arg):
-    assert isinstance(w_arg, W_Root)
- at api.cpython_api([PyObject], lltype.Void)
-def PyPy_GetReference(space, arg):
-    assert lltype.typeOf(arg) ==  PyObject
+ at contextlib.contextmanager
+def raises_w(space, expected_exc):
+    with pytest.raises(OperationError) as excinfo:
+        yield
+    operror = excinfo.value
+    assert operror.w_type is getattr(space, 'w_' + expected_exc.__name__)
 
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
@@ -35,7 +37,7 @@
             def __getattr__(self, name):
                 return getattr(cls.space, name)
         cls.api = CAPI()
-        CAPI.__dict__.update(api.INTERPLEVEL_API)
+        CAPI.__dict__.update(INTERPLEVEL_API)
 
         print 'DONT_FREE_ANY_MORE'
         rawrefcount._dont_free_any_more()
@@ -71,20 +73,28 @@
         if self.check_and_print_leaks():
             assert False, "Test leaks or loses object(s)."
 
- at api.cpython_api([api.Py_ssize_t], api.Py_ssize_t, error=-1)
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetWrapped(space, w_arg):
+    assert isinstance(w_arg, W_Root)
+
+ at slot_function([PyObject], lltype.Void)
+def PyPy_GetReference(space, arg):
+    assert lltype.typeOf(arg) ==  PyObject
+
+ at cpython_api([Py_ssize_t], Py_ssize_t, error=-1)
 def PyPy_TypedefTest1(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_t
+    assert lltype.typeOf(arg) == Py_ssize_t
     return 0
 
- at api.cpython_api([api.Py_ssize_tP], api.Py_ssize_tP)
+ at cpython_api([Py_ssize_tP], Py_ssize_tP)
 def PyPy_TypedefTest2(space, arg):
-    assert lltype.typeOf(arg) == api.Py_ssize_tP
+    assert lltype.typeOf(arg) == Py_ssize_tP
     return None
 
 class TestConversion(BaseApiTest):
-    def test_conversions(self, space, api):
-        api.PyPy_GetWrapped(space.w_None)
-        api.PyPy_GetReference(space.w_None)
+    def test_conversions(self, space):
+        PyPy_GetWrapped(space, space.w_None)
+        PyPy_GetReference(space, space.w_None)
 
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
@@ -95,7 +105,7 @@
         assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Py_ssize_t *arg0'
 
         PyPy_TypedefTest1(space, 0)
-        ppos = lltype.malloc(api.Py_ssize_tP.TO, 1, flavor='raw')
+        ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
         ppos[0] = 0
         PyPy_TypedefTest2(space, ppos)
         lltype.free(ppos, flavor='raw')
@@ -103,7 +113,7 @@
 @pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
-    api.copy_header_files(tmpdir, True)
+    copy_header_files(tmpdir, True)
     def check(name):
         f = tmpdir.join(name)
         assert f.check(file=True)
diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py
--- a/pypy/module/cpyext/test/test_boolobject.py
+++ b/pypy/module/cpyext/test/test_boolobject.py
@@ -1,20 +1,22 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong
+from pypy.module.cpyext.floatobject import PyFloat_FromDouble
 
 class TestBoolObject(BaseApiTest):
-    def test_fromlong(self, space, api):
+    def test_fromlong(self, space):
         for i in range(-3, 3):
-            obj = api.PyBool_FromLong(i)
+            obj = PyBool_FromLong(space, i)
             if i:
                 assert obj is space.w_True
             else:
                 assert obj is space.w_False
 
-    def test_check(self, space, api):
-        assert api.PyBool_Check(space.w_True)
-        assert api.PyBool_Check(space.w_False)
-        assert not api.PyBool_Check(space.w_None)
-        assert not api.PyBool_Check(api.PyFloat_FromDouble(1.0))
+    def test_check(self, space):
+        assert PyBool_Check(space, space.w_True)
+        assert PyBool_Check(space, space.w_False)
+        assert not PyBool_Check(space, space.w_None)
+        assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0))
 
 class AppTestBoolMacros(AppTestCpythonExtensionBase):
     def test_macros(self):
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -1,14 +1,18 @@
 # encoding: utf-8
+import pytest
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.interpreter.error import OperationError
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject
+from pypy.module.cpyext.bytesobject import (
+    new_empty_str, PyBytesObject, _PyString_Resize, PyString_Concat,
+    PyString_ConcatAndDel, PyString_Format, PyString_InternFromString,
+    PyString_AsEncodedObject, PyString_AsDecodedObject, _PyString_Eq,
+    _PyString_Join)
 from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref
-from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
+from pypy.module.cpyext.object import PyObject_AsCharBuffer
 
-import py
-import sys
 
 class AppTestBytesObject(AppTestCpythonExtensionBase):
     def test_bytesobject(self):
@@ -449,21 +453,21 @@
         assert 0 == module.get_len(b)
 
 class TestBytes(BaseApiTest):
-    def test_bytes_resize(self, space, api):
+    def test_bytes_resize(self, space):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         py_str.c_ob_sval[0] = 'a'
         py_str.c_ob_sval[1] = 'b'
         py_str.c_ob_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
-        api._PyString_Resize(ar, 3)
+        _PyString_Resize(space, ar, 3)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 3
         assert py_str.c_ob_sval[1] == 'b'
         assert py_str.c_ob_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
-        api._PyString_Resize(ar, 10)
+        _PyString_Resize(space, ar, 10)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 10
         assert py_str.c_ob_sval[1] == 'b'
@@ -471,7 +475,7 @@
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
-    def test_string_buffer(self, space, api):
+    def test_string_buffer(self, space):
         py_str = new_empty_str(space, 10)
         c_buf = py_str.c_ob_type.c_tp_as_buffer
         assert c_buf
@@ -489,108 +493,106 @@
         lltype.free(ref, flavor='raw')
         Py_DecRef(space, py_obj)
 
-    def test_Concat(self, space, api):
+    def test_Concat(self, space):
         ref = make_ref(space, space.wrap('abc'))
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref
         prev_refcnt = ref.c_ob_refcnt
-        api.PyString_Concat(ptr, space.wrap('def'))
+        PyString_Concat(space, ptr, space.wrap('def'))
         assert ref.c_ob_refcnt == prev_refcnt - 1
         assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
-        api.PyString_Concat(ptr, space.w_None)
+        with pytest.raises(OperationError):
+            PyString_Concat(space, ptr, space.w_None)
         assert not ptr[0]
-        api.PyErr_Clear()
         ptr[0] = lltype.nullptr(PyObject.TO)
-        api.PyString_Concat(ptr, space.wrap('def')) # should not crash
+        PyString_Concat(space, ptr, space.wrap('def')) # should not crash
         lltype.free(ptr, flavor='raw')
 
-    def test_ConcatAndDel(self, space, api):
+    def test_ConcatAndDel(self, space):
         ref1 = make_ref(space, space.wrap('abc'))
         ref2 = make_ref(space, space.wrap('def'))
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref1
         prev_refcnf = ref2.c_ob_refcnt
-        api.PyString_ConcatAndDel(ptr, ref2)
+        PyString_ConcatAndDel(space, ptr, ref2)
         assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
         assert ref2.c_ob_refcnt == prev_refcnf - 1
         Py_DecRef(space, ptr[0])
         ptr[0] = lltype.nullptr(PyObject.TO)
         ref2 = make_ref(space, space.wrap('foo'))
         prev_refcnf = ref2.c_ob_refcnt
-        api.PyString_ConcatAndDel(ptr, ref2) # should not crash
+        PyString_ConcatAndDel(space, ptr, ref2) # should not crash
         assert ref2.c_ob_refcnt == prev_refcnf - 1
         lltype.free(ptr, flavor='raw')
 
-    def test_format(self, space, api):
+    def test_format(self, space):
         assert "1 2" == space.unwrap(
-            api.PyString_Format(space.wrap('%s %d'), space.wrap((1, 2))))
+            PyString_Format(space, space.wrap('%s %d'), space.wrap((1, 2))))
 
-    def test_asbuffer(self, space, api):
+    def test_asbuffer(self, space):
         bufp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
 
         w_text = space.wrap("text")
         ref = make_ref(space, w_text)
         prev_refcnt = ref.c_ob_refcnt
-        assert api.PyObject_AsCharBuffer(ref, bufp, lenp) == 0
+        assert PyObject_AsCharBuffer(space, ref, bufp, lenp) == 0
         assert ref.c_ob_refcnt == prev_refcnt
         assert lenp[0] == 4
         assert rffi.charp2str(bufp[0]) == 'text'
         lltype.free(bufp, flavor='raw')
         lltype.free(lenp, flavor='raw')
-        api.Py_DecRef(ref)
+        Py_DecRef(space, ref)
 
-    def test_intern(self, space, api):
+    def test_intern(self, space):
         buf = rffi.str2charp("test")
-        w_s1 = api.PyString_InternFromString(buf)
-        w_s2 = api.PyString_InternFromString(buf)
+        w_s1 = PyString_InternFromString(space, buf)
+        w_s2 = PyString_InternFromString(space, buf)
         rffi.free_charp(buf)
         assert w_s1 is w_s2
 
-    def test_AsEncodedObject(self, space, api):
+    def test_AsEncodedObject(self, space):
         ptr = space.wrap('abc')
 
         errors = rffi.str2charp("strict")
 
         encoding = rffi.str2charp("hex")
-        res = api.PyString_AsEncodedObject(
-            ptr, encoding, errors)
+        res = PyString_AsEncodedObject(space, ptr, encoding, errors)
         assert space.unwrap(res) == "616263"
 
-        res = api.PyString_AsEncodedObject(
+        res = PyString_AsEncodedObject(space,
             ptr, encoding, lltype.nullptr(rffi.CCHARP.TO))
         assert space.unwrap(res) == "616263"
         rffi.free_charp(encoding)
 
         encoding = rffi.str2charp("unknown_encoding")
-        self.raises(space, api, LookupError, api.PyString_AsEncodedObject,
-                    ptr, encoding, errors)
+        with raises_w(space, LookupError):
+            PyString_AsEncodedObject(space, ptr, encoding, errors)
         rffi.free_charp(encoding)
 
         rffi.free_charp(errors)
 
-        res = api.PyString_AsEncodedObject(
-            ptr, lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO))
+        NULL = lltype.nullptr(rffi.CCHARP.TO)
+        res = PyString_AsEncodedObject(space, ptr, NULL, NULL)
         assert space.unwrap(res) == "abc"
+        with raises_w(space, TypeError):
+            PyString_AsEncodedObject(space, space.wrap(2), NULL, NULL)
 
-        self.raises(space, api, TypeError, api.PyString_AsEncodedObject,
-            space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
-        )
-
-    def test_AsDecodedObject(self, space, api):
+    def test_AsDecodedObject(self, space):
         w_str = space.wrap('caf\xe9')
         encoding = rffi.str2charp("latin-1")
-        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        w_res = PyString_AsDecodedObject(space, w_str, encoding, None)
         rffi.free_charp(encoding)
         assert space.unwrap(w_res) == u"caf\xe9"
 
-    def test_eq(self, space, api):
-        assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
-        assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
+    def test_eq(self, space):
+        assert 1 == _PyString_Eq(
+            space, space.wrap("hello"), space.wrap("hello"))
+        assert 0 == _PyString_Eq(
+            space, space.wrap("hello"), space.wrap("world"))
 
-    def test_join(self, space, api):
+    def test_join(self, space):
         w_sep = space.wrap('<sep>')
         w_seq = space.wrap(['a', 'b'])
-        w_joined = api._PyString_Join(w_sep, w_seq)
+        w_joined = _PyString_Join(space, w_sep, w_seq)
         assert space.unwrap(w_joined) == 'a<sep>b'
-
diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py
--- a/pypy/module/cpyext/test/test_classobject.py
+++ b/pypy/module/cpyext/test/test_classobject.py
@@ -1,9 +1,13 @@
+from pypy.interpreter.function import Function
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.interpreter.function import Function, Method
+from pypy.module.cpyext.classobject import (
+    PyClass_Check, PyClass_New, PyInstance_Check, PyInstance_New,
+    PyInstance_NewRaw, _PyInstance_Lookup)
+from pypy.module.cpyext.object import PyObject_GetAttr
 
 class TestClassObject(BaseApiTest):
-    def test_newinstance(self, space, api):
+    def test_newinstance(self, space):
         w_class = space.appexec([], """():
             class C:
                 x = None
@@ -14,23 +18,23 @@
             return C
         """)
 
-        assert api.PyClass_Check(w_class)
+        assert PyClass_Check(space, w_class)
 
-        w_instance = api.PyInstance_NewRaw(w_class, None)
-        assert api.PyInstance_Check(w_instance)
+        w_instance = PyInstance_NewRaw(space, w_class, None)
+        assert PyInstance_Check(space, w_instance)
         assert space.getattr(w_instance, space.wrap('x')) is space.w_None
 
-        w_instance = api.PyInstance_NewRaw(w_class, space.wrap(dict(a=3)))
+        w_instance = PyInstance_NewRaw(space, w_class, space.wrap(dict(a=3)))
         assert space.getattr(w_instance, space.wrap('x')) is space.w_None
         assert space.unwrap(space.getattr(w_instance, space.wrap('a'))) == 3
 
-        w_instance = api.PyInstance_New(w_class,
+        w_instance = PyInstance_New(space, w_class,
                                         space.wrap((3,)), space.wrap(dict(y=2)))
         assert space.unwrap(space.getattr(w_instance, space.wrap('x'))) == 1
         assert space.unwrap(space.getattr(w_instance, space.wrap('y'))) == 2
         assert space.unwrap(space.getattr(w_instance, space.wrap('args'))) == (3,)
 
-    def test_lookup(self, space, api):
+    def test_lookup(self, space):
         w_instance = space.appexec([], """():
             class C:
                 def __init__(self):
@@ -39,25 +43,26 @@
             return C()
         """)
 
-        assert api.PyInstance_Check(w_instance)
-        assert api.PyObject_GetAttr(w_instance, space.wrap('x')) is space.w_None
-        assert api._PyInstance_Lookup(w_instance, space.wrap('x')) is space.w_None
-        assert api._PyInstance_Lookup(w_instance, space.wrap('y')) is None
-        assert not api.PyErr_Occurred()
+        assert PyInstance_Check(space, w_instance)
+        assert PyObject_GetAttr(space, w_instance, space.wrap('x')) is space.w_None
+        assert _PyInstance_Lookup(space, w_instance, space.wrap('x')) is space.w_None
+        assert _PyInstance_Lookup(space, w_instance, space.wrap('y')) is None
 
         # getattr returns a bound method
-        assert not isinstance(api.PyObject_GetAttr(w_instance, space.wrap('f')), Function)
+        assert not isinstance(
+            PyObject_GetAttr(space, w_instance, space.wrap('f')), Function)
         # _PyInstance_Lookup returns the raw descriptor
-        assert isinstance(api._PyInstance_Lookup(w_instance, space.wrap('f')), Function)
+        assert isinstance(
+            _PyInstance_Lookup(space, w_instance, space.wrap('f')), Function)
 
-    def test_pyclass_new(self, space, api):
+    def test_pyclass_new(self, space):
         w_bases = space.newtuple([])
         w_dict = space.newdict()
         w_name = space.wrap("C")
-        w_class = api.PyClass_New(w_bases, w_dict, w_name)
+        w_class = PyClass_New(space, w_bases, w_dict, w_name)
         assert not space.isinstance_w(w_class, space.w_type)
         w_instance = space.call_function(w_class)
-        assert api.PyInstance_Check(w_instance)
+        assert PyInstance_Check(space, w_instance)
         assert space.is_true(space.call_method(space.builtin, "isinstance",
                                                w_instance, w_class))
 
@@ -69,5 +74,6 @@
                  Py_INCREF(&PyClass_Type);
                  return (PyObject*)&PyClass_Type;
              """)])
-        class C: pass
+        class C:
+            pass
         assert module.get_classtype() is type(C)
diff --git a/pypy/module/cpyext/test/test_codecs.py b/pypy/module/cpyext/test/test_codecs.py
--- a/pypy/module/cpyext/test/test_codecs.py
+++ b/pypy/module/cpyext/test/test_codecs.py
@@ -1,14 +1,15 @@
 # encoding: iso-8859-15
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.codecs import (
+    PyCodec_IncrementalEncoder, PyCodec_IncrementalDecoder)
 
 class TestCodecs(BaseApiTest):
-    def test_incremental(self, space, api):
+    def test_incremental(self, space):
         utf8 = rffi.str2charp('utf-8')
-        w_encoder = api.PyCodec_IncrementalEncoder(utf8, None)
+        w_encoder = PyCodec_IncrementalEncoder(space, utf8, None)
         w_encoded = space.call_method(w_encoder, 'encode', space.wrap(u'sp&#228;m'))
-        w_decoder = api.PyCodec_IncrementalDecoder(utf8, None)
+        w_decoder = PyCodec_IncrementalDecoder(space, utf8, None)
         w_decoded = space.call_method(w_decoder, 'decode', w_encoded)
         assert space.unwrap(w_decoded) == u'sp&#228;m'
         rffi.free_charp(utf8)
-
diff --git a/pypy/module/cpyext/test/test_complexobject.py b/pypy/module/cpyext/test/test_complexobject.py
--- a/pypy/module/cpyext/test/test_complexobject.py
+++ b/pypy/module/cpyext/test/test_complexobject.py
@@ -1,23 +1,23 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.complexobject import (
+    PyComplex_FromDoubles, PyComplex_RealAsDouble, PyComplex_ImagAsDouble)
 
 class TestComplexObject(BaseApiTest):
-    def test_complexobject(self, space, api):
-        w_value = api.PyComplex_FromDoubles(1.2, 3.4)
+    def test_complexobject(self, space):
+        w_value = PyComplex_FromDoubles(space, 1.2, 3.4)
         assert space.unwrap(w_value) == 1.2+3.4j
-        assert api.PyComplex_RealAsDouble(w_value) == 1.2
-        assert api.PyComplex_ImagAsDouble(w_value) == 3.4
+        assert PyComplex_RealAsDouble(space, w_value) == 1.2
+        assert PyComplex_ImagAsDouble(space, w_value) == 3.4
 
-        assert api.PyComplex_RealAsDouble(space.wrap(42)) == 42
-        assert api.PyComplex_RealAsDouble(space.wrap(1.5)) == 1.5
-        assert api.PyComplex_ImagAsDouble(space.wrap(1.5)) == 0.0
+        assert PyComplex_RealAsDouble(space, space.wrap(42)) == 42
+        assert PyComplex_RealAsDouble(space, space.wrap(1.5)) == 1.5
+        assert PyComplex_ImagAsDouble(space, space.wrap(1.5)) == 0.0
 
         # cpython accepts anything for PyComplex_ImagAsDouble
-        assert api.PyComplex_ImagAsDouble(space.w_None) == 0.0
-        assert not api.PyErr_Occurred()
-        assert api.PyComplex_RealAsDouble(space.w_None) == -1.0
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
+        assert PyComplex_ImagAsDouble(space, space.w_None) == 0.0
+        with raises_w(space, TypeError):
+            PyComplex_RealAsDouble(space, space.w_None)
 
 class AppTestCComplex(AppTestCpythonExtensionBase):
     def test_AsCComplex(self):
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -1,92 +1,96 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.cdatetime import *
+from pypy.module.cpyext.cdatetime import (
+    _PyDateTime_Import, _PyDateTime_FromDateAndTime, _PyDate_FromDate,
+    _PyTime_FromTime, _PyDelta_FromDelta)
 import datetime
 
 class TestDatetime(BaseApiTest):
-    def test_date(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_date = api._PyDate_FromDate(2010, 06, 03, date_api.c_DateType)
+    def test_date(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_date = _PyDate_FromDate(space, 2010, 06, 03, date_api.c_DateType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03'
 
-        assert api.PyDate_Check(w_date)
-        assert api.PyDate_CheckExact(w_date)
+        assert PyDate_Check(space, w_date)
+        assert PyDate_CheckExact(space, w_date)
 
-        assert api.PyDateTime_GET_YEAR(w_date) == 2010
-        assert api.PyDateTime_GET_MONTH(w_date) == 6
-        assert api.PyDateTime_GET_DAY(w_date) == 3
+        assert PyDateTime_GET_YEAR(space, w_date) == 2010
+        assert PyDateTime_GET_MONTH(space, w_date) == 6
+        assert PyDateTime_GET_DAY(space, w_date) == 3
 
-    def test_time(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_time = api._PyTime_FromTime(23, 15, 40, 123456,
-                                      space.w_None, date_api.c_TimeType)
+    def test_time(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_time = _PyTime_FromTime(
+            space, 23, 15, 40, 123456, space.w_None, date_api.c_TimeType)
         assert space.unwrap(space.str(w_time)) == '23:15:40.123456'
 
-        assert api.PyTime_Check(w_time)
-        assert api.PyTime_CheckExact(w_time)
+        assert PyTime_Check(space, w_time)
+        assert PyTime_CheckExact(space, w_time)
 
-        assert api.PyDateTime_TIME_GET_HOUR(w_time) == 23
-        assert api.PyDateTime_TIME_GET_MINUTE(w_time) == 15
-        assert api.PyDateTime_TIME_GET_SECOND(w_time) == 40
-        assert api.PyDateTime_TIME_GET_MICROSECOND(w_time) == 123456
+        assert PyDateTime_TIME_GET_HOUR(space, w_time) == 23
+        assert PyDateTime_TIME_GET_MINUTE(space, w_time) == 15
+        assert PyDateTime_TIME_GET_SECOND(space, w_time) == 40
+        assert PyDateTime_TIME_GET_MICROSECOND(space, w_time) == 123456
 
-    def test_datetime(self, space, api):
-        date_api = api._PyDateTime_Import()
-        w_date = api._PyDateTime_FromDateAndTime(
-            2010, 06, 03, 23, 15, 40, 123456,
-            space.w_None, date_api.c_DateTimeType)
+    def test_datetime(self, space):
+        date_api = _PyDateTime_Import(space)
+        w_date = _PyDateTime_FromDateAndTime(
+            space, 2010, 06, 03, 23, 15, 40, 123456, space.w_None,
+            date_api.c_DateTimeType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03 23:15:40.123456'
 
-        assert api.PyDateTime_Check(w_date)
-        assert api.PyDateTime_CheckExact(w_date)
-        assert api.PyDate_Check(w_date)
-        assert not api.PyDate_CheckExact(w_date)
+        assert PyDateTime_Check(space, w_date)
+        assert PyDateTime_CheckExact(space, w_date)
+        assert PyDate_Check(space, w_date)
+        assert not PyDate_CheckExact(space, w_date)
 
-        assert api.PyDateTime_GET_YEAR(w_date) == 2010
-        assert api.PyDateTime_GET_MONTH(w_date) == 6
-        assert api.PyDateTime_GET_DAY(w_date) == 3
-        assert api.PyDateTime_DATE_GET_HOUR(w_date) == 23
-        assert api.PyDateTime_DATE_GET_MINUTE(w_date) == 15
-        assert api.PyDateTime_DATE_GET_SECOND(w_date) == 40
-        assert api.PyDateTime_DATE_GET_MICROSECOND(w_date) == 123456
+        assert PyDateTime_GET_YEAR(space, w_date) == 2010
+        assert PyDateTime_GET_MONTH(space, w_date) == 6
+        assert PyDateTime_GET_DAY(space, w_date) == 3
+        assert PyDateTime_DATE_GET_HOUR(space, w_date) == 23
+        assert PyDateTime_DATE_GET_MINUTE(space, w_date) == 15
+        assert PyDateTime_DATE_GET_SECOND(space, w_date) == 40
+        assert PyDateTime_DATE_GET_MICROSECOND(space, w_date) == 123456
 
-    def test_delta(self, space, api):
-        date_api = api._PyDateTime_Import()
+    def test_delta(self, space):
+        date_api = _PyDateTime_Import(space)
         w_delta = space.appexec(
             [space.wrap(3), space.wrap(15)], """(days, seconds):
             from datetime import timedelta
             return timedelta(days, seconds)
         """)
-        assert api.PyDelta_Check(w_delta)
-        assert api.PyDelta_CheckExact(w_delta)
+        assert PyDelta_Check(space, w_delta)
+        assert PyDelta_CheckExact(space, w_delta)
 
-        w_delta = api._PyDelta_FromDelta(10, 20, 30, True, date_api.c_DeltaType)
-        assert api.PyDelta_Check(w_delta)
-        assert api.PyDelta_CheckExact(w_delta)
+        w_delta = _PyDelta_FromDelta(space, 10, 20, 30, True, date_api.c_DeltaType)
+        assert PyDelta_Check(space, w_delta)
+        assert PyDelta_CheckExact(space, w_delta)
 
-        assert api.PyDateTime_DELTA_GET_DAYS(w_delta) == 10
-        assert api.PyDateTime_DELTA_GET_SECONDS(w_delta) == 20
-        assert api.PyDateTime_DELTA_GET_MICROSECONDS(w_delta) == 30
+        assert PyDateTime_DELTA_GET_DAYS(space, w_delta) == 10
+        assert PyDateTime_DELTA_GET_SECONDS(space, w_delta) == 20
+        assert PyDateTime_DELTA_GET_MICROSECONDS(space, w_delta) == 30
 
-    def test_fromtimestamp(self, space, api):
+    def test_fromtimestamp(self, space):
         w_args = space.wrap((0,))
-        w_date = api.PyDate_FromTimestamp(w_args)
+        w_date = PyDate_FromTimestamp(space, w_args)
         date = datetime.date.fromtimestamp(0)
         assert space.unwrap(space.str(w_date)) == str(date)
 
         w_args = space.wrap((0,))
-        w_date = api.PyDateTime_FromTimestamp(w_args)
+        w_date = PyDateTime_FromTimestamp(space, w_args)
         date = datetime.datetime.fromtimestamp(0)
         assert space.unwrap(space.str(w_date)) == str(date)
 
-    def test_tzinfo(self, space, api):
+    def test_tzinfo(self, space):
         w_tzinfo = space.appexec(
             [], """():
             from datetime import tzinfo
             return tzinfo()
         """)
-        assert api.PyTZInfo_Check(w_tzinfo)
-        assert api.PyTZInfo_CheckExact(w_tzinfo)
-        assert not api.PyTZInfo_Check(space.w_None)
+        assert PyTZInfo_Check(space, w_tzinfo)
+        assert PyTZInfo_CheckExact(space, w_tzinfo)
+        assert not PyTZInfo_Check(space, space.w_None)
 
 class AppTestDatetime(AppTestCpythonExtensionBase):
     def test_CAPI(self):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -1,83 +1,81 @@
 import py
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP, PyTypeObjectPtr
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.dictobject import *
+from pypy.module.cpyext.pyobject import decref
 
 class TestDictObject(BaseApiTest):
-    def test_dict(self, space, api):
-        d = api.PyDict_New()
+    def test_dict(self, space):
+        d = PyDict_New(space)
         assert space.eq_w(d, space.newdict())
 
-        assert space.eq_w(api.PyDict_GetItem(space.wrap({"a": 72}),
+        assert space.eq_w(PyDict_GetItem(space, space.wrap({"a": 72}),
                                              space.wrap("a")),
                           space.wrap(72))
 
-        assert api.PyDict_SetItem(d, space.wrap("c"), space.wrap(42)) >= 0
+        PyDict_SetItem(space, d, space.wrap("c"), space.wrap(42))
         assert space.eq_w(space.getitem(d, space.wrap("c")),
                           space.wrap(42))
 
         space.setitem(d, space.wrap("name"), space.wrap(3))
-        assert space.eq_w(api.PyDict_GetItem(d, space.wrap("name")),
+        assert space.eq_w(PyDict_GetItem(space, d, space.wrap("name")),
                           space.wrap(3))
 
         space.delitem(d, space.wrap("name"))
-        assert not api.PyDict_GetItem(d, space.wrap("name"))
-        assert not api.PyErr_Occurred()
+        assert not PyDict_GetItem(space, d, space.wrap("name"))
 
         buf = rffi.str2charp("name")
-        assert not api.PyDict_GetItemString(d, buf)
+        assert not PyDict_GetItemString(space, d, buf)
         rffi.free_charp(buf)
-        assert not api.PyErr_Occurred()
 
-        assert api.PyDict_Contains(d, space.wrap("c"))
-        assert not api.PyDict_Contains(d, space.wrap("z"))
+        assert PyDict_Contains(space, d, space.wrap("c"))
+        assert not PyDict_Contains(space, d, space.wrap("z"))
 
-        assert api.PyDict_DelItem(d, space.wrap("c")) == 0
-        assert api.PyDict_DelItem(d, space.wrap("name")) < 0
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
-        assert api.PyDict_Size(d) == 0
+        PyDict_DelItem(space, d, space.wrap("c"))
+        with raises_w(space, KeyError):
+            PyDict_DelItem(space, d, space.wrap("name"))
+        assert PyDict_Size(space, d) == 0
 
         space.setitem(d, space.wrap("some_key"), space.wrap(3))
         buf = rffi.str2charp("some_key")
-        assert api.PyDict_DelItemString(d, buf) == 0
-        assert api.PyDict_Size(d) == 0
-        assert api.PyDict_DelItemString(d, buf) < 0
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
+        PyDict_DelItemString(space, d, buf)
+        assert PyDict_Size(space, d) == 0
+        with raises_w(space, KeyError):
+            PyDict_DelItemString(space, d, buf)
         rffi.free_charp(buf)
 
         d = space.wrap({'a': 'b'})
-        api.PyDict_Clear(d)
-        assert api.PyDict_Size(d) == 0
+        PyDict_Clear(space, d)
+        assert PyDict_Size(space, d) == 0
 
-    def test_check(self, space, api):
-        d = api.PyDict_New()
-        assert api.PyDict_Check(d)
-        assert api.PyDict_CheckExact(d)
+    def test_check(self, space):
+        d = PyDict_New(space, )
+        assert PyDict_Check(space, d)
+        assert PyDict_CheckExact(space, d)
         sub = space.appexec([], """():
             class D(dict):
                 pass
             return D""")
         d = space.call_function(sub)
-        assert api.PyDict_Check(d)
-        assert not api.PyDict_CheckExact(d)
+        assert PyDict_Check(space, d)
+        assert not PyDict_CheckExact(space, d)
         i = space.wrap(2)
-        assert not api.PyDict_Check(i)
-        assert not api.PyDict_CheckExact(i)
+        assert not PyDict_Check(space, i)
+        assert not PyDict_CheckExact(space, i)
 
-    def test_keys(self, space, api):
+    def test_keys(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
-        assert space.eq_w(api.PyDict_Keys(w_d), space.wrap(["a"]))
-        assert space.eq_w(api.PyDict_Values(w_d), space.wrap(["b"]))
-        assert space.eq_w(api.PyDict_Items(w_d), space.wrap([("a", "b")]))
+        assert space.eq_w(PyDict_Keys(space, w_d), space.wrap(["a"]))
+        assert space.eq_w(PyDict_Values(space, w_d), space.wrap(["b"]))
+        assert space.eq_w(PyDict_Items(space, w_d), space.wrap([("a", "b")]))
 
-    def test_merge(self, space, api):
+    def test_merge(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
@@ -86,35 +84,34 @@
         space.setitem(w_d2, space.wrap("c"), space.wrap("d"))
         space.setitem(w_d2, space.wrap("e"), space.wrap("f"))
 
-        api.PyDict_Merge(w_d, w_d2, 0)
+        PyDict_Merge(space, w_d, w_d2, 0)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
-        api.PyDict_Merge(w_d, w_d2, 1)
+        PyDict_Merge(space, w_d, w_d2, 1)
         assert space.unwrap(w_d) == dict(a='c', c='d', e='f')
 
-    def test_update(self, space, api):
+    def test_update(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
-        w_d2 = api.PyDict_Copy(w_d)
+        w_d2 = PyDict_Copy(space, w_d)
         assert not space.is_w(w_d2, w_d)
         space.setitem(w_d, space.wrap("c"), space.wrap("d"))
         space.setitem(w_d2, space.wrap("e"), space.wrap("f"))
 
-        api.PyDict_Update(w_d, w_d2)
+        PyDict_Update(space, w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
-    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+    def test_update_doesnt_accept_list_of_tuples(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
         w_d2 = space.wrap([("c", "d"), ("e", "f")])
 
-        api.PyDict_Update(w_d, w_d2)
-        assert api.PyErr_Occurred() is space.w_AttributeError
-        api.PyErr_Clear()
+        with raises_w(space, AttributeError):
+            PyDict_Update(space, w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b') # unchanged
 
-    def test_iter(self, space, api):
+    def test_iter(self, space):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
 
@@ -125,7 +122,7 @@
 
         try:
             w_copy = space.newdict()
-            while api.PyDict_Next(w_dict, ppos, pkey, pvalue):
+            while PyDict_Next(space, w_dict, ppos, pkey, pvalue):
                 w_key = from_ref(space, pkey[0])
                 w_value = from_ref(space, pvalue[0])
                 space.setitem(w_copy, w_key, w_value)
@@ -134,12 +131,12 @@
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
 
-        api.Py_DecRef(py_dict) # release borrowed references
+        decref(space, py_dict) # release borrowed references
 
         assert space.eq_w(space.len(w_copy), space.len(w_dict))
         assert space.eq_w(w_copy, w_dict)
 
-    def test_iterkeys(self, space, api):
+    def test_iterkeys(self, space):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
 
@@ -151,11 +148,11 @@
         values_w = []
         try:
             ppos[0] = 0
-            while api.PyDict_Next(w_dict, ppos, pkey, None):
+            while PyDict_Next(space, w_dict, ppos, pkey, None):
                 w_key = from_ref(space, pkey[0])
                 keys_w.append(w_key)
             ppos[0] = 0
-            while api.PyDict_Next(w_dict, ppos, None, pvalue):
+            while PyDict_Next(space, w_dict, ppos, None, pvalue):
                 w_value = from_ref(space, pvalue[0])
                 values_w.append(w_value)
         finally:
@@ -163,25 +160,25 @@
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
 
-        api.Py_DecRef(py_dict) # release borrowed references
+        decref(space, py_dict) # release borrowed references
 
         assert space.eq_w(space.newlist(keys_w),
                           space.call_method(w_dict, "keys"))
         assert space.eq_w(space.newlist(values_w),
                           space.call_method(w_dict, "values"))
 
-    def test_dictproxy(self, space, api):
+    def test_dictproxy(self, space):
         w_dict = space.sys.get('modules')
-        w_proxy = api.PyDictProxy_New(w_dict)
+        w_proxy = PyDictProxy_New(space, w_dict)
         assert space.contains_w(w_proxy, space.wrap('sys'))
         raises(OperationError, space.setitem,
                w_proxy, space.wrap('sys'), space.w_None)
         raises(OperationError, space.delitem,
                w_proxy, space.wrap('sys'))
         raises(OperationError, space.call_method, w_proxy, 'clear')
-        assert api.PyDictProxy_Check(w_proxy)
+        assert PyDictProxy_Check(space, w_proxy)
 
-    def test_typedict1(self, space, api):
+    def test_typedict1(self, space):
         py_type = make_ref(space, space.w_int)
         py_dict = rffi.cast(PyTypeObjectPtr, py_type).c_tp_dict
         ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
@@ -191,7 +188,7 @@
         pvalue = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         try:
             w_copy = space.newdict()
-            while api.PyDict_Next(py_dict, ppos, pkey, pvalue):
+            while PyDict_Next(space, py_dict, ppos, pkey, pvalue):
                 w_key = from_ref(space, pkey[0])
                 w_value = from_ref(space, pvalue[0])
                 space.setitem(w_copy, w_key, w_value)
@@ -199,7 +196,7 @@
             lltype.free(ppos, flavor='raw')
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
-        api.Py_DecRef(py_type) # release borrowed references
+        decref(space, py_type) # release borrowed references
         # do something with w_copy ?
 
 class AppTestDictObject(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -1,17 +1,26 @@
+import sys
+import os
+import pytest
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.object import PyObject_Size, PyObject_GetItem
+from pypy.module.cpyext.pythonrun import Py_AtExit
 from pypy.module.cpyext.eval import (
-    Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import (c_fopen, c_fclose, c_fileno, 
-                Py_ssize_tP, is_valid_fd)
+    Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags,
+    PyEval_CallObjectWithKeywords, PyObject_CallObject, PyEval_EvalCode,
+    PyRun_SimpleString, PyRun_String, PyRun_StringFlags, PyRun_File,
+    PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals,
+    _PyEval_SliceIndex)
+from pypy.module.cpyext.api import (
+    c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd)
 from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
-import sys, os
 
 class TestEval(BaseApiTest):
-    def test_eval(self, space, api):
+    def test_eval(self, space):
         w_l, w_f = space.fixedview(space.appexec([], """():
         l = []
         def f(arg1, arg2):
@@ -22,7 +31,7 @@
         """))
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, None)
+        w_res = PyEval_CallObjectWithKeywords(space, w_f, w_t, None)
         assert space.int_w(w_res) == 2
         assert space.len_w(w_l) == 2
         w_f = space.appexec([], """():
@@ -35,10 +44,10 @@
         w_t = space.newtuple([space.w_None, space.w_None])
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("xyz"), space.wrap(3))
-        w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, w_d)
+        w_res = PyEval_CallObjectWithKeywords(space, w_f, w_t, w_d)
         assert space.int_w(w_res) == 21
 
-    def test_call_object(self, space, api):
+    def test_call_object(self, space):
         w_l, w_f = space.fixedview(space.appexec([], """():
         l = []
         def f(arg1, arg2):
@@ -49,7 +58,7 @@
         """))
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyObject_CallObject(w_f, w_t)
+        w_res = PyObject_CallObject(space, w_f, w_t)
         assert space.int_w(w_res) == 2
         assert space.len_w(w_l) == 2
 
@@ -61,11 +70,11 @@
             """)
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyObject_CallObject(w_f, w_t)
+        w_res = PyObject_CallObject(space, w_f, w_t)
 
         assert space.int_w(w_res) == 10
 
-    def test_evalcode(self, space, api):
+    def test_evalcode(self, space):
         w_f = space.appexec([], """():
             def f(*args):
                 assert isinstance(args, tuple)
@@ -77,84 +86,78 @@
         w_globals = space.newdict()
         w_locals = space.newdict()
         space.setitem(w_locals, space.wrap("args"), w_t)
-        w_res = api.PyEval_EvalCode(w_f.code, w_globals, w_locals)
+        w_res = PyEval_EvalCode(space, w_f.code, w_globals, w_locals)
 
         assert space.int_w(w_res) == 10
 
-    def test_run_simple_string(self, space, api):
+    def test_run_simple_string(self, space):
         def run(code):
             buf = rffi.str2charp(code)
             try:
-                return api.PyRun_SimpleString(buf)
+                return PyRun_SimpleString(space, buf)
             finally:
                 rffi.free_charp(buf)
 
-        assert 0 == run("42 * 43")
+        assert run("42 * 43") == 0  # no error
+        with pytest.raises(OperationError):
+            run("4..3 * 43")
 
-        assert -1 == run("4..3 * 43")
-
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
-
-    def test_run_string(self, space, api):
+    def test_run_string(self, space):
         def run(code, start, w_globals, w_locals):
             buf = rffi.str2charp(code)
             try:
-                return api.PyRun_String(buf, start, w_globals, w_locals)
+                return PyRun_String(space, buf, start, w_globals, w_locals)
             finally:
                 rffi.free_charp(buf)
 
         w_globals = space.newdict()
         assert 42 * 43 == space.unwrap(
             run("42 * 43", Py_eval_input, w_globals, w_globals))
-        assert api.PyObject_Size(w_globals) == 0
+        assert PyObject_Size(space, w_globals) == 0
 
         assert run("a = 42 * 43", Py_single_input,
                    w_globals, w_globals) == space.w_None
         assert 42 * 43 == space.unwrap(
-            api.PyObject_GetItem(w_globals, space.wrap("a")))
+            PyObject_GetItem(space, w_globals, space.wrap("a")))
 
-    def test_run_string_flags(self, space, api):
+    def test_run_string_flags(self, space):
         flags = lltype.malloc(PyCompilerFlags, flavor='raw')
         flags.c_cf_flags = rffi.cast(rffi.INT, consts.PyCF_SOURCE_IS_UTF8)
         w_globals = space.newdict()
         buf = rffi.str2charp("a = u'caf\xc3\xa9'")
         try:
-            api.PyRun_StringFlags(buf, Py_single_input,
-                                  w_globals, w_globals, flags)
+            PyRun_StringFlags(space, buf, Py_single_input, w_globals,
+                              w_globals, flags)
         finally:
             rffi.free_charp(buf)
         w_a = space.getitem(w_globals, space.wrap("a"))
         assert space.unwrap(w_a) == u'caf\xe9'
         lltype.free(flags, flavor='raw')
 
-    def test_run_file(self, space, api):
+    def test_run_file(self, space):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
         fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
-        api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
+        with raises_w(space, ZeroDivisionError):
+            PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
         c_fclose(fp)
-        assert api.PyErr_Occurred() is space.w_ZeroDivisionError
-        api.PyErr_Clear()
 
         # try again, but with a closed file
         fp = c_fopen(str(filepath), "rb")
         os.close(c_fileno(fp))
-        api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
+        with raises_w(space, IOError):
+            PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
         if is_valid_fd(c_fileno(fp)):
             c_fclose(fp)
-        assert api.PyErr_Occurred() is space.w_IOError
-        api.PyErr_Clear()
-
         rffi.free_charp(filename)
 
-    def test_getbuiltins(self, space, api):
-        assert api.PyEval_GetBuiltins() is space.builtin.w_dict
+    def test_getbuiltins(self, space):
+        assert PyEval_GetBuiltins(space) is space.builtin.w_dict
 
         def cpybuiltins(space):
-            return api.PyEval_GetBuiltins()
+            return PyEval_GetBuiltins(space)
         w_cpybuiltins = space.wrap(interp2app(cpybuiltins))
 
         w_result = space.appexec([w_cpybuiltins], """(cpybuiltins):
@@ -168,13 +171,13 @@
         """)
         assert space.len_w(w_result) == 1
 
-    def test_getglobals(self, space, api):
-        assert api.PyEval_GetLocals() is None
-        assert api.PyEval_GetGlobals() is None
+    def test_getglobals(self, space):
+        assert PyEval_GetLocals(space) is None
+        assert PyEval_GetGlobals(space) is None
 
         def cpyvars(space):
-            return space.newtuple([api.PyEval_GetGlobals(),
-                                   api.PyEval_GetLocals()])
+            return space.newtuple([PyEval_GetGlobals(space),
+                                   PyEval_GetLocals(space)])
         w_cpyvars = space.wrap(interp2app(cpyvars))
 
         w_result = space.appexec([w_cpyvars], """(cpyvars):
@@ -186,26 +189,26 @@
         assert sorted(locals) == ['cpyvars', 'x']
         assert sorted(globals) == ['__builtins__', 'anonymous', 'y']
 
-    def test_sliceindex(self, space, api):
+    def test_sliceindex(self, space):
         pi = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
-        assert api._PyEval_SliceIndex(space.w_None, pi) == 0
-        api.PyErr_Clear()
+        with pytest.raises(OperationError):
+            _PyEval_SliceIndex(space, space.w_None, pi)
 
-        assert api._PyEval_SliceIndex(space.wrap(123), pi) == 1
+        assert _PyEval_SliceIndex(space, space.wrap(123), pi) == 1
         assert pi[0] == 123
 
-        assert api._PyEval_SliceIndex(space.wrap(1 << 66), pi) == 1
+        assert _PyEval_SliceIndex(space, space.wrap(1 << 66), pi) == 1
         assert pi[0] == sys.maxint
 
         lltype.free(pi, flavor='raw')
 
-    def test_atexit(self, space, api):
+    def test_atexit(self, space):
         lst = []
         def func():
             lst.append(42)
-        api.Py_AtExit(func)
+        Py_AtExit(space, func)
         cpyext = space.getbuiltinmodule('cpyext')
-        cpyext.shutdown(space) # simulate shutdown
+        cpyext.shutdown(space)  # simulate shutdown
         assert lst == [42]
 
 class AppTestCall(AppTestCpythonExtensionBase):
@@ -269,6 +272,7 @@
                 return res;
              """),
             ])
+
         def f(*args):
             return args
         assert module.call_func(f) == (None,)
@@ -322,7 +326,7 @@
             ])
         assert module.get_flags() == (0, 0)
 
-        ns = {'module':module}
+        ns = {'module': module}
         exec """from __future__ import division    \nif 1:
                 def nested_flags():
                     return module.get_flags()""" in ns
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ b/pypy/module/cpyext/test/test_floatobject.py
@@ -1,36 +1,40 @@
+import pytest
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.floatobject import (
+    PyFloat_FromDouble, PyFloat_AsDouble, PyFloat_AS_DOUBLE, PyNumber_Float,
+    _PyFloat_Unpack4, _PyFloat_Unpack8)
 
 class TestFloatObject(BaseApiTest):
-    def test_floatobject(self, space, api):
-        assert space.unwrap(api.PyFloat_FromDouble(3.14)) == 3.14
-        assert api.PyFloat_AsDouble(space.wrap(23.45)) == 23.45
-        assert api.PyFloat_AS_DOUBLE(space.wrap(23.45)) == 23.45
+    def test_floatobject(self, space):
+        assert space.unwrap(PyFloat_FromDouble(space, 3.14)) == 3.14
+        assert PyFloat_AsDouble(space, space.wrap(23.45)) == 23.45
+        assert PyFloat_AS_DOUBLE(space, space.wrap(23.45)) == 23.45
+        with pytest.raises(OperationError):
+            PyFloat_AsDouble(space, space.w_None)
 
-        assert api.PyFloat_AsDouble(space.w_None) == -1
-        api.PyErr_Clear()
-
-    def test_coerce(self, space, api):
-        assert space.type(api.PyNumber_Float(space.wrap(3))) is space.w_float
-        assert space.type(api.PyNumber_Float(space.wrap("3"))) is space.w_float
+    def test_coerce(self, space):
+        assert space.type(PyNumber_Float(space, space.wrap(3))) is space.w_float
+        assert space.type(PyNumber_Float(space, space.wrap("3"))) is space.w_float
 
         w_obj = space.appexec([], """():
             class Coerce(object):
                 def __float__(self):
                     return 42.5
             return Coerce()""")
-        assert space.eq_w(api.PyNumber_Float(w_obj), space.wrap(42.5))
+        assert space.eq_w(PyNumber_Float(space, w_obj), space.wrap(42.5))
 
-    def test_unpack(self, space, api):
+    def test_unpack(self, space):
         with rffi.scoped_str2charp("\x9a\x99\x99?") as ptr:
-            assert abs(api._PyFloat_Unpack4(ptr, 1) - 1.2) < 1e-7
+            assert abs(_PyFloat_Unpack4(space, ptr, 1) - 1.2) < 1e-7
         with rffi.scoped_str2charp("?\x99\x99\x9a") as ptr:
-            assert abs(api._PyFloat_Unpack4(ptr, 0) - 1.2) < 1e-7
+            assert abs(_PyFloat_Unpack4(space, ptr, 0) - 1.2) < 1e-7
         with rffi.scoped_str2charp("\x1f\x85\xebQ\xb8\x1e\t@") as ptr:
-            assert abs(api._PyFloat_Unpack8(ptr, 1) - 3.14) < 1e-15
+            assert abs(_PyFloat_Unpack8(space, ptr, 1) - 3.14) < 1e-15
         with rffi.scoped_str2charp("@\t\x1e\xb8Q\xeb\x85\x1f") as ptr:
-            assert abs(api._PyFloat_Unpack8(ptr, 0) - 3.14) < 1e-15
+            assert abs(_PyFloat_Unpack8(space, ptr, 0) - 3.14) < 1e-15
 
 class AppTestFloatObject(AppTestCpythonExtensionBase):
     def test_fromstring(self):
@@ -79,8 +83,6 @@
         assert math.isinf(neginf)
 
     def test_macro_accepts_wrong_pointer_type(self):
-        import math
-
         module = self.import_extension('foo', [
             ("test_macros", "METH_NOARGS",
              """
diff --git a/pypy/module/cpyext/test/test_funcobject.py b/pypy/module/cpyext/test/test_funcobject.py
--- a/pypy/module/cpyext/test/test_funcobject.py
+++ b/pypy/module/cpyext/test/test_funcobject.py
@@ -1,16 +1,18 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
+from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, decref
+from pypy.module.cpyext.methodobject import PyClassMethod_New
 from pypy.module.cpyext.funcobject import (
-    PyFunctionObject, PyCodeObject, CODE_FLAGS)
-from pypy.interpreter.function import Function, Method
+    PyFunctionObject, PyCodeObject, CODE_FLAGS, PyMethod_Function,
+    PyMethod_Self, PyMethod_Class, PyMethod_New, PyFunction_GetCode,
+    PyCode_NewEmpty, PyCode_GetNumFree)
+from pypy.interpreter.function import Function
 from pypy.interpreter.pycode import PyCode
 
 globals().update(CODE_FLAGS)
 
 class TestFunctionObject(BaseApiTest):
-    def test_function(self, space, api):
+    def test_function(self, space):
         w_function = space.appexec([], """():
             def f(): pass
             return f
@@ -19,10 +21,10 @@
         assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is
                 space.gettypeobject(Function.typedef))
         assert "f" == space.unwrap(
-           from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name))
-        api.Py_DecRef(ref)
+            from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name))
+        decref(space, ref)
 
-    def test_method(self, space, api):
+    def test_method(self, space):
         w_method = space.appexec([], """():
             class C(list):
                 def method(self): pass
@@ -33,30 +35,30 @@
         w_self = space.getattr(w_method, space.wrap("im_self"))
         w_class = space.getattr(w_method, space.wrap("im_class"))
 
-        assert space.is_w(api.PyMethod_Function(w_method), w_function)
-        assert space.is_w(api.PyMethod_Self(w_method), w_self)
-        assert space.is_w(api.PyMethod_Class(w_method), w_class)
+        assert space.is_w(PyMethod_Function(space, w_method), w_function)
+        assert space.is_w(PyMethod_Self(space, w_method), w_self)
+        assert space.is_w(PyMethod_Class(space, w_method), w_class)
 
-        w_method2 = api.PyMethod_New(w_function, w_self, w_class)
+        w_method2 = PyMethod_New(space, w_function, w_self, w_class)
         assert space.eq_w(w_method, w_method2)
 
-    def test_getcode(self, space, api):
+    def test_getcode(self, space):
         w_function = space.appexec([], """():
             def func(x, y, z): return x
             return func
         """)
-        w_code = api.PyFunction_GetCode(w_function)
+        w_code = PyFunction_GetCode(space, w_function)
         assert w_code.co_name == "func"
 
         ref = make_ref(space, w_code)
         assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is
                 space.gettypeobject(PyCode.typedef))
         assert "func" == space.unwrap(
-           from_ref(space, rffi.cast(PyCodeObject, ref).c_co_name))
+            from_ref(space, rffi.cast(PyCodeObject, ref).c_co_name))
         assert 3 == rffi.cast(PyCodeObject, ref).c_co_argcount
-        api.Py_DecRef(ref)
+        decref(space, ref)
 
-    def test_co_flags(self, space, api):
+    def test_co_flags(self, space):
         def get_flags(signature, body="pass"):
             w_code = space.appexec([], """():
                 def func(%s): %s
@@ -64,7 +66,7 @@
             """ % (signature, body))
             ref = make_ref(space, w_code)
             co_flags = rffi.cast(PyCodeObject, ref).c_co_flags
-            api.Py_DecRef(ref)
+            decref(space, ref)
             return co_flags
         assert get_flags("x") == CO_NESTED | CO_OPTIMIZED | CO_NEWLOCALS
         assert get_flags("x", "exec x") == CO_NESTED | CO_NEWLOCALS
@@ -72,29 +74,29 @@
         assert get_flags("x, **kw") & CO_VARKEYWORDS
         assert get_flags("x", "yield x") & CO_GENERATOR
 
-    def test_newcode(self, space, api):
+    def test_newcode(self, space):
         filename = rffi.str2charp('filename')
         funcname = rffi.str2charp('funcname')
-        w_code = api.PyCode_NewEmpty(filename, funcname, 3)
+        w_code = PyCode_NewEmpty(space, filename, funcname, 3)
         assert w_code.co_filename == 'filename'
         assert w_code.co_firstlineno == 3
 
         ref = make_ref(space, w_code)
         assert "filename" == space.unwrap(
             from_ref(space, rffi.cast(PyCodeObject, ref).c_co_filename))
-        api.Py_DecRef(ref)
+        decref(space, ref)
         rffi.free_charp(filename)
         rffi.free_charp(funcname)
 
-    def test_getnumfree(self, space, api):
+    def test_getnumfree(self, space):
         w_function = space.appexec([], """():
             a = 5
             def method(x): return a, x
             return method
         """)
-        assert api.PyCode_GetNumFree(w_function.code) == 1
+        assert PyCode_GetNumFree(space, w_function.code) == 1
 
-    def test_classmethod(self, space, api):
+    def test_classmethod(self, space):
         w_function = space.appexec([], """():
             def method(x): return x
             return method
@@ -106,6 +108,7 @@
         space.setattr(w_class, space.wrap("method"), w_function)
         assert space.is_w(space.call_method(w_instance, "method"), w_instance)
         # now a classmethod
-        w_classmethod = api.PyClassMethod_New(w_function)
+        w_classmethod = PyClassMethod_New(space, w_function)
         space.setattr(w_class, space.wrap("classmethod"), w_classmethod)
-        assert space.is_w(space.call_method(w_instance, "classmethod"), w_class)
+        assert space.is_w(
+            space.call_method(w_instance, "classmethod"), w_class)
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -1,48 +1,51 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.import_ import *
+from pypy.module.cpyext.import_ import (
+    _PyImport_AcquireLock, _PyImport_ReleaseLock)
 from rpython.rtyper.lltypesystem import rffi
 
 class TestImport(BaseApiTest):
-    def test_import(self, space, api):
-        stat = api.PyImport_Import(space.wrap("stat"))
+    def test_import(self, space):
+        stat = PyImport_Import(space, space.wrap("stat"))
         assert stat
         assert space.getattr(stat, space.wrap("S_IMODE"))
 
-    def test_addmodule(self, space, api):
+    def test_addmodule(self, space):
         with rffi.scoped_str2charp("sys") as modname:
-            w_sys = api.PyImport_AddModule(modname)
+            w_sys = PyImport_AddModule(space, modname)
         assert w_sys is space.sys
 
         with rffi.scoped_str2charp("foobar") as modname:
-            w_foobar = api.PyImport_AddModule(modname)
+            w_foobar = PyImport_AddModule(space, modname)
         assert space.str_w(space.getattr(w_foobar,
                                          space.wrap('__name__'))) == 'foobar'
 
-    def test_getmoduledict(self, space, api):
+    def test_getmoduledict(self, space):
         testmod = "_functools"
-        w_pre_dict = api.PyImport_GetModuleDict()
+        w_pre_dict = PyImport_GetModuleDict(space, )
         assert not space.contains_w(w_pre_dict, space.wrap(testmod))
 
         with rffi.scoped_str2charp(testmod) as modname:
-            w_module = api.PyImport_ImportModule(modname)
+            w_module = PyImport_ImportModule(space, modname)
             print w_module
             assert w_module
 
-        w_dict = api.PyImport_GetModuleDict()
+        w_dict = PyImport_GetModuleDict(space, )
         assert space.contains_w(w_dict, space.wrap(testmod))
 
-    def test_reload(self, space, api):
-        stat = api.PyImport_Import(space.wrap("stat"))
+    def test_reload(self, space):
+        stat = PyImport_Import(space, space.wrap("stat"))
         space.delattr(stat, space.wrap("S_IMODE"))
-        stat = api.PyImport_ReloadModule(stat)
+        stat = PyImport_ReloadModule(space, stat)
         assert space.getattr(stat, space.wrap("S_IMODE"))
 
-    def test_lock(self, space, api):
+    def test_lock(self, space):
         # "does not crash"
-        api._PyImport_AcquireLock()
-        api._PyImport_AcquireLock()
-        api._PyImport_ReleaseLock()
-        api._PyImport_ReleaseLock()
+        _PyImport_AcquireLock(space, )
+        _PyImport_AcquireLock(space, )
+        _PyImport_ReleaseLock(space, )
+        _PyImport_ReleaseLock(space, )
 
 
 class AppTestImportLogic(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py
--- a/pypy/module/cpyext/test/test_intobject.py
+++ b/pypy/module/cpyext/test/test_intobject.py
@@ -1,51 +1,52 @@
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.intobject import (
+    PyInt_Check, PyInt_AsLong, PyInt_AS_LONG, PyInt_FromLong,
+    PyInt_AsUnsignedLong, PyInt_AsUnsignedLongMask,
+    PyInt_AsUnsignedLongLongMask)
 import sys
 
 class TestIntObject(BaseApiTest):
-    def test_intobject(self, space, api):
-        assert api.PyInt_Check(space.wrap(3))
-        assert api.PyInt_Check(space.w_True)
-        assert not api.PyInt_Check(space.wrap((1, 2, 3)))
+    def test_intobject(self, space):
+        assert PyInt_Check(space, space.wrap(3))
+        assert PyInt_Check(space, space.w_True)
+        assert not PyInt_Check(space, space.wrap((1, 2, 3)))
         for i in [3, -5, -1, -sys.maxint, sys.maxint - 1]:
-            x = api.PyInt_AsLong(space.wrap(i))
-            y = api.PyInt_AS_LONG(space.wrap(i))
+            x = PyInt_AsLong(space, space.wrap(i))
+            y = PyInt_AS_LONG(space, space.wrap(i))
             assert x == i
             assert y == i
-            w_x = api.PyInt_FromLong(x + 1)
+            w_x = PyInt_FromLong(space, x + 1)
             assert space.type(w_x) is space.w_int
             assert space.eq_w(w_x, space.wrap(i + 1))
 
-        assert api.PyInt_AsLong(space.w_None) == -1
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyInt_AsLong(space, space.w_None)
 
-        assert api.PyInt_AsLong(None) == -1
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyInt_AsLong(space, None)
 
-        assert api.PyInt_AsUnsignedLong(space.wrap(sys.maxint)) == sys.maxint
-        assert api.PyInt_AsUnsignedLong(space.wrap(-5)) == sys.maxint * 2 + 1
-        assert api.PyErr_Occurred() is space.w_ValueError
-        api.PyErr_Clear()
+        assert PyInt_AsUnsignedLong(space, space.wrap(sys.maxint)) == sys.maxint
+        with raises_w(space, ValueError):
+            PyInt_AsUnsignedLong(space, space.wrap(-5))
 
-        assert (api.PyInt_AsUnsignedLongMask(space.wrap(sys.maxint))
+        assert (PyInt_AsUnsignedLongMask(space, space.wrap(sys.maxint))
                 == sys.maxint)
-        assert (api.PyInt_AsUnsignedLongMask(space.wrap(10**30))
-                == 10**30 % ((sys.maxint + 1) * 2))
+        assert (PyInt_AsUnsignedLongMask(space, space.wrap(10 ** 30))
+                == 10 ** 30 % ((sys.maxint + 1) * 2))
 
-        assert (api.PyInt_AsUnsignedLongLongMask(space.wrap(sys.maxint))
+        assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(sys.maxint))
                 == sys.maxint)
-        assert (api.PyInt_AsUnsignedLongLongMask(space.wrap(10**30))
-                == 10**30 % (2**64))
+        assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(10 ** 30))
+                == 10 ** 30 % (2 ** 64))
 
-    def test_coerce(self, space, api):
+    def test_coerce(self, space):
         w_obj = space.appexec([], """():
             class Coerce(object):
                 def __int__(self):
                     return 42
             return Coerce()""")
-        assert api.PyInt_AsLong(w_obj) == 42
+        assert PyInt_AsLong(space, w_obj) == 42
 
 class AppTestIntObject(AppTestCpythonExtensionBase):
     def test_fromstring(self):
@@ -203,4 +204,3 @@
                 """
                 ),
                 ])
-
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -1,5 +1,6 @@
 # encoding: utf-8
-from pypy.module.cpyext.test.test_api import BaseApiTest
+import pytest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.unicodeobject import (
     Py_UNICODE, PyUnicodeObject, new_empty_unicode)
@@ -7,6 +8,7 @@
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref
 from rpython.rtyper.lltypesystem import rffi, lltype
 import sys, py
+from pypy.module.cpyext.unicodeobject import *
 
 class AppTestUnicodeObject(AppTestCpythonExtensionBase):
     def test_unicodeobject(self):
@@ -131,13 +133,13 @@
         assert module.test_macro_invocations() == u''
 
 class TestUnicode(BaseApiTest):
-    def test_unicodeobject(self, space, api):
-        assert api.PyUnicode_GET_SIZE(space.wrap(u'sp&#65533;m')) == 4
-        assert api.PyUnicode_GetSize(space.wrap(u'sp&#65533;m')) == 4
+    def test_unicodeobject(self, space):
+        assert PyUnicode_GET_SIZE(space, space.wrap(u'sp&#65533;m')) == 4
+        assert PyUnicode_GetSize(space, space.wrap(u'sp&#65533;m')) == 4
         unichar = rffi.sizeof(Py_UNICODE)
-        assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp&#65533;m')) == 4 * unichar
+        assert PyUnicode_GET_DATA_SIZE(space, space.wrap(u'sp&#65533;m')) == 4 * unichar
 
-        encoding = rffi.charp2str(api.PyUnicode_GetDefaultEncoding())
+        encoding = rffi.charp2str(PyUnicode_GetDefaultEncoding(space, ))
         w_default_encoding = space.call_function(
             space.sys.get('getdefaultencoding')
         )
@@ -145,43 +147,45 @@
         invalid = rffi.str2charp('invalid')
         utf_8 = rffi.str2charp('utf-8')
         prev_encoding = rffi.str2charp(space.unwrap(w_default_encoding))
-        self.raises(space, api, TypeError, api.PyUnicode_SetDefaultEncoding, lltype.nullptr(rffi.CCHARP.TO))
-        assert api.PyUnicode_SetDefaultEncoding(invalid) == -1
-        assert api.PyErr_Occurred() is space.w_LookupError
-        api.PyErr_Clear()
-        assert api.PyUnicode_SetDefaultEncoding(utf_8) == 0
-        assert rffi.charp2str(api.PyUnicode_GetDefaultEncoding()) == 'utf-8'
-        assert api.PyUnicode_SetDefaultEncoding(prev_encoding) == 0
+        with raises_w(space, TypeError):
+            PyUnicode_SetDefaultEncoding(space, lltype.nullptr(rffi.CCHARP.TO))
+        with raises_w(space, LookupError):
+            PyUnicode_SetDefaultEncoding(space, invalid)
+
+        assert PyUnicode_SetDefaultEncoding(space, utf_8) == 0
+        assert rffi.charp2str(PyUnicode_GetDefaultEncoding(space, )) == 'utf-8'
+        assert PyUnicode_SetDefaultEncoding(space, prev_encoding) == 0

From pypy.commits at gmail.com  Sat Jan 14 13:53:28 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 14 Jan 2017 10:53:28 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: reduce diff in merge,
 document branch
Message-ID: <587a73a8.810b1c0a.48125.08dc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89566:a7f5d5cc77ae
Date: 2017-01-14 20:51 +0200
http://bitbucket.org/pypy/pypy/changeset/a7f5d5cc77ae/

Log:	reduce diff in merge, document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -101,3 +101,14 @@
 
 Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to
 the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak.
+
+.. branch: missing-tp_new
+
+Improve mixing app-level classes in c-extensions, especially if the app-level
+class has a ``tp_new`` or ``tp_dealloc``. The issue is that c-extensions expect
+all the method slots to be filled with a function pointer, where app-level will
+search up the mro for an appropriate function at runtime. With this branch we
+now fill many more slots in the c-extenion type objects.
+Also fix for c-extension type that calls ``tp_hash`` during initialization
+(str, unicode types), and fix instantiating c-extension types from built-in
+classes by enforcing an order of instaniation.
diff --git a/pypy/module/cpyext/include/pyport.h b/pypy/module/cpyext/include/pyport.h
--- a/pypy/module/cpyext/include/pyport.h
+++ b/pypy/module/cpyext/include/pyport.h
@@ -68,6 +68,7 @@
 #   error "Python needs a typedef for Py_uintptr_t in pyport.h."
 #endif /* HAVE_UINTPTR_T */
 
+
 /*******************************
  * stat() and fstat() fiddling *
  *******************************/

From pypy.commits at gmail.com  Sat Jan 14 13:53:30 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 14 Jan 2017 10:53:30 -0800 (PST)
Subject: [pypy-commit] pypy default: behave more like the cpython macros -
 do no type checking (should move to pure c)
Message-ID: <587a73aa.0d1a1c0a.2fc08.0e56@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r89567:597653e02827
Date: 2017-01-12 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/597653e02827/

Log:	behave more like the cpython macros - do no type checking (should
	move to pure c)

diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -90,11 +90,10 @@
     return 0
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
-def PyList_GET_SIZE(space, w_list):
+def PyList_GET_SIZE(space, w_obj):
     """Macro form of PyList_Size() without error checking.
     """
-    assert isinstance(w_list, W_ListObject)
-    return w_list.length()
+    return space.len_w(w_obj)
 
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -205,9 +205,8 @@
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PyUnicode_GET_SIZE(space, w_obj):
-    """Return the size of the object.  o has to be a PyUnicodeObject (not
+    """Return the size of the object.  obj is a PyUnicodeObject (not
     checked)."""
-    assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
     return space.len_w(w_obj)
 
 @cpython_api([rffi.VOIDP], rffi.CWCHARP, error=CANNOT_FAIL)

From pypy.commits at gmail.com  Sat Jan 14 16:19:31 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 14 Jan 2017 13:19:31 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: fix merge
Message-ID: <587a95e3.d01d1c0a.5d9c2.444b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89568:574e0c5b9dc8
Date: 2017-01-14 23:17 +0200
http://bitbucket.org/pypy/pypy/changeset/574e0c5b9dc8/

Log:	fix merge

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -562,7 +562,7 @@
             slot_fn = w_type.getdictvalue(space, attr)
             if slot_fn is None:
                 return
-            @slot_function([PyObject], lltype.Signed, header=header, error=-1)
+            @slot_function([PyObject], lltype.Signed, error=-1)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_obj):
                 return space.int_w(space.call_function(slot_fn, w_obj))

From pypy.commits at gmail.com  Sat Jan 14 16:19:33 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 14 Jan 2017 13:19:33 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <587a95e5.9d711c0a.65f4a.61e9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89569:a69e2a0fae65
Date: 2017-01-14 23:18 +0200
http://bitbucket.org/pypy/pypy/changeset/a69e2a0fae65/

Log:	merge default into branch

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -220,7 +220,7 @@
         self.__statements_counter = 0
         self.__rawstatements = set()
         self._statement_cache = _StatementCache(self, cached_statements)
-        self.__statements_already_committed = weakref.WeakSet()
+        self.__statements_already_committed = []
 
         self.__func_cache = {}
         self.__aggregates = {}
@@ -365,10 +365,12 @@
                     cursor._reset = True
 
     def _reset_already_committed_statements(self):
-        lst = list(self.__statements_already_committed)
-        self.__statements_already_committed.clear()
-        for statement in lst:
-            statement._reset()
+        lst = self.__statements_already_committed
+        self.__statements_already_committed = []
+        for weakref in lst:
+            statement = weakref()
+            if statement is not None:
+                statement._reset()
 
     @_check_thread_wrap
     @_check_closed_wrap
@@ -432,14 +434,11 @@
         # the way and cause the "drop table" to fail.  On CPython the
         # problem is much less important because typically all the old
         # statements are freed already by reference counting.  So here,
-        # we add all the still-alive statements to a WeakSet which is
-        # usually ignored, except if we get SQLITE_LOCKED
+        # we copy all the still-alive statements to another list which
+        # is usually ignored, except if we get SQLITE_LOCKED
         # afterwards---at which point we reset all statements in this
-        # WeakSet.
-        for weakref in self.__statements:
-            statement = weakref()
-            if statement is not None:
-                self.__statements_already_committed.add(statement)
+        # list.
+        self.__statements_already_committed = self.__statements[:]
 
         statement_star = _ffi.new('sqlite3_stmt **')
         ret = _lib.sqlite3_prepare_v2(self._db, b"COMMIT", -1,
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -102,6 +102,10 @@
 Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to
 the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak.
 
+.. branch: issue2464
+
+Give (almost?) all GetSetProperties a valid __objclass__.
+
 .. branch: missing-tp_new
 
 Improve mixing app-level classes in c-extensions, especially if the app-level
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -140,7 +140,11 @@
             pass
         def fget(self, space, w_self):
             assert self is prop
-        prop = typedef.GetSetProperty(fget, use_closure=True)
+        # NB. this GetSetProperty is not copied when creating the
+        # W_TypeObject because of 'cls'.  Without it, a duplicate of the
+        # GetSetProperty is taken and it is given the w_objclass that is
+        # the W_TypeObject
+        prop = typedef.GetSetProperty(fget, use_closure=True, cls=W_SomeType)
         W_SomeType.typedef = typedef.TypeDef(
             'some_type',
             x=prop)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -276,12 +276,15 @@
         self.use_closure = use_closure
 
     def copy_for_type(self, w_objclass):
-        new = instantiate(GetSetProperty)
-        new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
-                  None, self.use_closure)
-        new.name = self.name
-        new.w_objclass = w_objclass
-        return new
+        if self.objclass_getter is None:
+            new = instantiate(GetSetProperty)
+            new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
+                      None, self.use_closure)
+            new.name = self.name
+            new.w_objclass = w_objclass
+            return new
+        else:
+            return self
 
     @unwrap_spec(w_cls = WrappedDefault(None))
     def descr_property_get(self, space, w_obj, w_cls=None):
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -90,11 +90,10 @@
     return 0
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
-def PyList_GET_SIZE(space, w_list):
+def PyList_GET_SIZE(space, w_obj):
     """Macro form of PyList_Size() without error checking.
     """
-    assert isinstance(w_list, W_ListObject)
-    return w_list.length()
+    return space.len_w(w_obj)
 
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -206,9 +206,8 @@
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PyUnicode_GET_SIZE(space, w_obj):
-    """Return the size of the object.  o has to be a PyUnicodeObject (not
+    """Return the size of the object.  obj is a PyUnicodeObject (not
     checked)."""
-    assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
     return space.len_w(w_obj)
 
 @cpython_api([rffi.VOIDP], rffi.CWCHARP, error=CANNOT_FAIL)
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1329,3 +1329,6 @@
             pass
         assert X.__dict__['__dict__'].__objclass__ is X
         assert X.__dict__['__weakref__'].__objclass__ is X
+        assert object.__dict__['__class__'].__objclass__ is object
+        assert int.__dict__['imag'].__objclass__ is int
+        assert file.closed.__objclass__ is file
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1312,10 +1312,13 @@
     def build(self, typedef):
         "NOT_RPYTHON: initialization-time only."
         from pypy.objspace.std.objectobject import W_ObjectObject
+        from pypy.interpreter.typedef import GetSetProperty
+        from rpython.rlib.objectmodel import instantiate
 
         space = self.space
         rawdict = typedef.rawdict
         lazyloaders = {}
+        w_type = instantiate(W_TypeObject)
 
         # compute the bases
         if typedef is W_ObjectObject.typedef:
@@ -1327,13 +1330,16 @@
         # wrap everything
         dict_w = {}
         for descrname, descrvalue in rawdict.items():
+            # special case for GetSetProperties' __objclass__:
+            if isinstance(descrvalue, GetSetProperty):
+                descrvalue = descrvalue.copy_for_type(w_type)
             dict_w[descrname] = space.wrap(descrvalue)
 
         if typedef.applevel_subclasses_base is not None:
             overridetypedef = typedef.applevel_subclasses_base.typedef
         else:
             overridetypedef = typedef
-        w_type = W_TypeObject(space, typedef.name, bases_w, dict_w,
+        w_type.__init__(space, typedef.name, bases_w, dict_w,
                               overridetypedef=overridetypedef,
                               is_heaptype=overridetypedef.heaptype)
         if typedef is not overridetypedef:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -350,6 +350,14 @@
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
                                       args=args + extra,
                                       descr=target_token))
+
+            # XXX see test_issue2465 for a case where the following occurs
+            # XXX (it would be better to really understand the problem
+            # XXX and fix it, but I am failing so far)
+            if label_op is not None and len(args + extra) != label_op.numargs():
+                del self.optimizer._newoperations[-1]    # remove the JUMP
+                raise InvalidLoop
+
             return None # explicit because the return can be non-None
         return virtual_state
 
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4620,3 +4620,19 @@
             time.clock()
             return 0
         self.interp_operations(g, [])
+
+    def test_issue2465(self):
+        driver = JitDriver(greens=[], reds=['i', 'a', 'b'])
+        class F(object):
+            def __init__(self, floatval):
+                self.floatval = floatval
+        def f(i):
+            a = F(0.0)
+            b = None
+            while i > 0:
+                driver.jit_merge_point(i=i, a=a, b=b)
+                b = F(a.floatval / 1.)
+                i -= 1
+            return i
+
+        self.meta_interp(f, [10])

From pypy.commits at gmail.com  Sun Jan 15 01:32:43 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sat, 14 Jan 2017 22:32:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: cpython issue23099: adapt new exported
 buf cpython impl details, tighten getbuffer
Message-ID: <587b178b.52301c0a.536b6.5915@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89570:50b7ded1379c
Date: 2017-01-14 22:26 -0800
http://bitbucket.org/pypy/pypy/changeset/50b7ded1379c/

Log:	cpython issue23099: adapt new exported buf cpython impl details,
	tighten getbuffer

diff --git a/lib-python/3/test/test_memoryio.py b/lib-python/3/test/test_memoryio.py
--- a/lib-python/3/test/test_memoryio.py
+++ b/lib-python/3/test/test_memoryio.py
@@ -424,8 +424,8 @@
             # of the underlying object.
             self.assertRaises(BufferError, memio.write, b'x' * 100)
             self.assertRaises(BufferError, memio.truncate)
-        self.assertRaises(BufferError, memio.close)
-        self.assertFalse(memio.closed)
+            self.assertRaises(BufferError, memio.close)
+            self.assertFalse(memio.closed)
         # Mutating the buffer updates the BytesIO
         buf[3:6] = b"abc"
         self.assertEqual(bytes(buf), b"123abc7890")
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -121,6 +121,7 @@
         return space.wrap(size)
 
     def getbuffer_w(self, space):
+        self._check_closed(space)
         return space.wrap(W_MemoryView(BytesIOBuffer(self)))
 
     def getvalue_w(self, space):
diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py
--- a/pypy/module/_io/test/test_bytesio.py
+++ b/pypy/module/_io/test/test_bytesio.py
@@ -140,6 +140,8 @@
         # After the buffer gets released, we can resize the BytesIO again
         del buf
         memio.truncate()
+        memio.close()
+        raises(ValueError, memio.getbuffer)
 
     def test_readline(self):
         import _io

From pypy.commits at gmail.com  Sun Jan 15 03:03:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 00:03:58 -0800 (PST)
Subject: [pypy-commit] pypy default: issue2465: the test was wrong anyway,
 but finally found the bogus
Message-ID: <587b2cee.d01d1c0a.5d9c2.7cfa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89571:b4c9d4ee5864
Date: 2017-01-15 08:58 +0100
http://bitbucket.org/pypy/pypy/changeset/b4c9d4ee5864/

Log:	issue2465: the test was wrong anyway, but finally found the bogus
	place. (I think I am to blame, it comes from the fix 8cc8067ed404)

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -350,14 +350,6 @@
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
                                       args=args + extra,
                                       descr=target_token))
-
-            # XXX see test_issue2465 for a case where the following occurs
-            # XXX (it would be better to really understand the problem
-            # XXX and fix it, but I am failing so far)
-            if label_op is not None and len(args + extra) != label_op.numargs():
-                del self.optimizer._newoperations[-1]    # remove the JUMP
-                raise InvalidLoop
-
             return None # explicit because the return can be non-None
         return virtual_state
 
@@ -413,7 +405,8 @@
                 i += 1
                 self.optimizer.send_extra_operation(op)
             # force all of them except the virtuals
-            for arg in args_no_virtuals + short_jump_args:
+            for arg in (args_no_virtuals +
+                        self._map_args(mapping, short_jump_args)):
                 self.optimizer.force_box(self.get_box_replacement(arg))
             self.optimizer.flush()
             # done unless "short" has grown again

From pypy.commits at gmail.com  Sun Jan 15 03:08:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 00:08:19 -0800 (PST)
Subject: [pypy-commit] pypy default: add a test case
Message-ID: <587b2df3.d01d1c0a.5d9c2.7f0f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89572:198ebc8390f0
Date: 2017-01-15 09:07 +0100
http://bitbucket.org/pypy/pypy/changeset/198ebc8390f0/

Log:	add a test case

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1332,3 +1332,4 @@
         assert object.__dict__['__class__'].__objclass__ is object
         assert int.__dict__['imag'].__objclass__ is int
         assert file.closed.__objclass__ is file
+        assert type.__dict__['__name__'].__objclass__ is type

From pypy.commits at gmail.com  Sun Jan 15 03:34:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 00:34:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: issue2457: copy some changes that should
 be done in py3.5-newtext, but
Message-ID: <587b3404.d3811c0a.7bfda.8e33@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89573:37f628884dfb
Date: 2017-01-15 09:33 +0100
http://bitbucket.org/pypy/pypy/changeset/37f628884dfb/

Log:	issue2457: copy some changes that should be done in py3.5-newtext,
	but that really fix bugs

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -318,13 +318,13 @@
         return space.newtuple(self.co_names_w)
 
     def fget_co_varnames(self, space):
-        return space.newtuple([space.wrap(name) for name in self.co_varnames])
+        return space.newtuple([space.newtext(name) for name in self.co_varnames])
 
     def fget_co_cellvars(self, space):
-        return space.newtuple([space.wrap(name) for name in self.co_cellvars])
+        return space.newtuple([space.newtext(name) for name in self.co_cellvars])
 
     def fget_co_freevars(self, space):
-        return space.newtuple([space.wrap(name) for name in self.co_freevars])
+        return space.newtuple([space.newtext(name) for name in self.co_freevars])
 
     def descr_code__eq__(self, w_other):
         space = self.space
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -362,6 +362,9 @@
     def newutf8(self, string):
         return self.newunicode(decode_utf8(self, string))
 
+    def newtext(self, s):
+        return self.newunicode(decode_utf8(self, s, allow_surrogates=True))
+
     def newunicode(self, uni):
         return W_UnicodeObject(uni)
 

From pypy.commits at gmail.com  Sun Jan 15 03:45:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 00:45:27 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in
 TreeStain/pypy/TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419
 (pull request #506)
Message-ID: <587b36a7.05371c0a.39090.a6be@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r89575:e714afb7d217
Date: 2017-01-15 09:44 +0100
http://bitbucket.org/pypy/pypy/changeset/e714afb7d217/

Log:	Merged in TreeStain/pypy/TreeStain/fixed-typo-line-29-mostly-to-
	most-1484469416419 (pull request #506)

	Fixed typo. line 29: mostly to most.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -26,7 +26,7 @@
 
 Almost!
 
-The mostly likely stumbling block for any given project is support for
+The most likely stumbling block for any given project is support for
 :ref:`extension modules <extension-modules>`.  PyPy supports a continually growing
 number of extension modules, but so far mostly only those found in the
 standard library.

From pypy.commits at gmail.com  Sun Jan 15 03:45:34 2017
From: pypy.commits at gmail.com (TreeStain)
Date: Sun, 15 Jan 2017 00:45:34 -0800 (PST)
Subject: [pypy-commit] pypy
 TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419: Fixed typo. line
 29: mostly to most.
Message-ID: <587b36ae.05371c0a.39090.a6d0@mx.google.com>

Author: Tristan Arthur <roguesgonnasneak at gmail.com>
Branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419
Changeset: r89574:fb7e574f577e
Date: 2017-01-15 08:38 +0000
http://bitbucket.org/pypy/pypy/changeset/fb7e574f577e/

Log:	Fixed typo. line 29: mostly to most.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -26,7 +26,7 @@
 
 Almost!
 
-The mostly likely stumbling block for any given project is support for
+The most likely stumbling block for any given project is support for
 :ref:`extension modules <extension-modules>`.  PyPy supports a continually growing
 number of extension modules, but so far mostly only those found in the
 standard library.

From pypy.commits at gmail.com  Sun Jan 15 04:05:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 01:05:59 -0800 (PST)
Subject: [pypy-commit] pypy default: Backed out changeset ecfd182b1eb3
Message-ID: <587b3b77.c4251c0a.ead21.a6f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89576:c75c245d161b
Date: 2017-01-15 10:05 +0100
http://bitbucket.org/pypy/pypy/changeset/c75c245d161b/

Log:	Backed out changeset ecfd182b1eb3

	I don't know if it's gmail's fault, but on the nightly tannit32
	tests it fails to connect the first time (only)

diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -175,8 +175,8 @@
     }
 
     def setup_method(self, method):
-        # https://gmail.com/
-        ADDR = "gmail.com", 443
+        # https://www.verisign.net/
+        ADDR = "www.verisign.net", 443
 
         self.w_s = self.space.appexec([self.space.wrap(ADDR)], """(ADDR):
             import socket

From pypy.commits at gmail.com  Sun Jan 15 05:28:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 02:28:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: officialize a CPython difference
Message-ID: <587b4ed9.07941c0a.3db71.d51d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89577:b7305d7e2e07
Date: 2017-01-15 11:26 +0100
http://bitbucket.org/pypy/pypy/changeset/b7305d7e2e07/

Log:	officialize a CPython difference

diff --git a/lib-python/3/test/test_finalization.py b/lib-python/3/test/test_finalization.py
--- a/lib-python/3/test/test_finalization.py
+++ b/lib-python/3/test/test_finalization.py
@@ -171,7 +171,12 @@
             gc.collect()
             self.assert_del_calls(ids)
             self.assert_survivors(ids)
-            self.assertIsNot(wr(), None)
+            if support.check_impl_detail():
+                # CPython >= 3.4 changed this, but not in all cases
+                # (see the similar test in SelfCycleFinalizationTest).
+                # PyPy cannot tell apart the two cases, so we picked the
+                # cycle behavior, as it is the historical one.
+                self.assertIsNot(wr(), None)
             self.clear_survivors()
             gc.collect()
             self.assert_del_calls(ids)
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -166,6 +166,13 @@
 .. __: https://docs.python.org/2/library/weakref.html
 .. __: https://bitbucket.org/pypy/pypy/issue/2030/
 
+A new difference: before CPython 3.4, a weakref to ``x`` was always
+cleared before the ``x.__del__()`` method was called.  Since CPython 3.4
+the picture is more muddy.  Often, the weakref is still alive while
+``x.__del__()`` runs, but not always (e.g. not in case of reference
+cycles).  In PyPy3 we have kept the more consistent pre-3.4 behavior; we
+can't do something really different if there are cycles or not.
+
 ---------------------------------
 
 There are a few extra implications from the difference in the GC.  Most

From pypy.commits at gmail.com  Sun Jan 15 06:28:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 03:28:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix next test
Message-ID: <587b5ccf.0e821c0a.63665.009c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89578:31c1ed35e78b
Date: 2017-01-15 12:27 +0100
http://bitbucket.org/pypy/pypy/changeset/31c1ed35e78b/

Log:	fix next test

diff --git a/lib-python/3/test/test_finalization.py b/lib-python/3/test/test_finalization.py
--- a/lib-python/3/test/test_finalization.py
+++ b/lib-python/3/test/test_finalization.py
@@ -209,8 +209,15 @@
             self.assert_survivors(ids)
             self.clear_survivors()
             gc.collect()
-            self.assert_del_calls(ids * 2)
-            self.assert_survivors(ids)
+            if support.check_impl_detail():
+                # CPython: the __del__ method has been called again
+                self.assert_del_calls(ids * 2)
+                self.assert_survivors(ids)
+            else:
+                # PyPy (and likely others): the __del__ method is only
+                # called once
+                self.assert_del_calls(ids)
+                self.assert_survivors([])
 
 
 class SelfCycleBase:

From pypy.commits at gmail.com  Sun Jan 15 08:07:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 05:07:01 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <587b73f5.c4811c0a.28521.2ff6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r845:f885e0400cc1
Date: 2017-01-15 14:06 +0100
http://bitbucket.org/pypy/pypy.org/changeset/f885e0400cc1/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66541 of $105000 (63.4%)
+   $66545 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2373 left
+    we have $2377 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sun Jan 15 08:35:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 05:35:12 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in
 TreeStain/pypy-1/TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033
 (pull request #507)
Message-ID: <587b7a90.4dd41c0a.517e6.4b1e@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r89583:9a9dee929787
Date: 2017-01-15 14:34 +0100
http://bitbucket.org/pypy/pypy/changeset/9a9dee929787/

Log:	Merged in TreeStain/pypy-1/TreeStain/main-lines-changed-in-l77-l83
	-made-para-1484471558033 (pull request #507)

	Main lines changed in L77 - L83. Made paragraph more
	cohesive/flowing.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -32,10 +32,10 @@
 standard library.
 
 The language features (including builtin types and functions) are very
-complete and well tested, so if your project does not use many
+refined and well tested, so if your project doesn't use many
 extension modules there is a good chance that it will work with PyPy.
 
-We list the differences we know about in :doc:`cpython differences <cpython_differences>`.
+We list the known differences in :doc:`cpython differences <cpython_differences>`.
 
 
 Module xyz does not work with PyPy: ImportError
@@ -76,10 +76,10 @@
 
 You cannot import *any* extension module in a `sandboxed PyPy`_,
 sorry.  Even the built-in modules available are very limited.
-Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
-it is only a proof of concept.  It seriously requires someone working
-on it.  Before this occurs, it can only be used it for "pure Python"
-examples: programs that import mostly nothing (or only pure Python
+Sandboxing in PyPy is a good proof of concept, and is without a doubt
+safe IMHO, however it is only a proof of concept.  It currently requires 
+some work from a motivated developer. However, until then it can only be used for "pure Python"
+example: programs that import mostly nothing (or only pure Python
 modules, recursively).
 
 .. _`sandboxed PyPy`: sandbox.html

From pypy.commits at gmail.com  Sun Jan 15 08:35:23 2017
From: pypy.commits at gmail.com (TreeStain)
Date: Sun, 15 Jan 2017 05:35:23 -0800 (PST)
Subject: [pypy-commit] pypy
 TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033: Changed
 L81 to reflect the need for a new dedicated developer to work on sandboxed
 PyPy
Message-ID: <587b7a9b.878f1c0a.ab837.58b5@mx.google.com>

Author: Tristan Arthur <roguesgonnasneak at gmail.com>
Branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033
Changeset: r89580:30f24864dfa1
Date: 2017-01-15 12:11 +0000
http://bitbucket.org/pypy/pypy/changeset/30f24864dfa1/

Log:	Changed L81 to reflect the need for a new dedicated developer to
	work on sandboxed PyPy

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -78,7 +78,7 @@
 sorry.  Even the built-in modules available are very limited.
 Sandboxing in PyPy is a good proof of concept, and is without a doubt
 safe IMHO, however it is only a proof of concept.  It currently requires
-some work and in the meantime, can only be used for "pure Python"
+some work from a motivated developer. Until then it can only be used for "pure Python"
 examples: programs that import mostly nothing (or only pure Python
 modules, recursively).
 

From pypy.commits at gmail.com  Sun Jan 15 08:35:25 2017
From: pypy.commits at gmail.com (TreeStain)
Date: Sun, 15 Jan 2017 05:35:25 -0800 (PST)
Subject: [pypy-commit] pypy
 TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033: Fixed L81
 to be more cohesive
Message-ID: <587b7a9d.4e9d1c0a.5f1a1.26e9@mx.google.com>

Author: Tristan Arthur <roguesgonnasneak at gmail.com>
Branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033
Changeset: r89581:3607066a145c
Date: 2017-01-15 12:12 +0000
http://bitbucket.org/pypy/pypy/changeset/3607066a145c/

Log:	Fixed L81 to be more cohesive

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -77,8 +77,8 @@
 You cannot import *any* extension module in a `sandboxed PyPy`_,
 sorry.  Even the built-in modules available are very limited.
 Sandboxing in PyPy is a good proof of concept, and is without a doubt
-safe IMHO, however it is only a proof of concept.  It currently requires
-some work from a motivated developer. Until then it can only be used for "pure Python"
+safe IMHO, however it is only a proof of concept.  It currently requires 
+some work from a motivated developer. However, until then it can only be used for "pure Python"
 examples: programs that import mostly nothing (or only pure Python
 modules, recursively).
 

From pypy.commits at gmail.com  Sun Jan 15 08:35:21 2017
From: pypy.commits at gmail.com (TreeStain)
Date: Sun, 15 Jan 2017 05:35:21 -0800 (PST)
Subject: [pypy-commit] pypy
 TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033: Main lines
 changed in L77 - L83. Made paragraph more cohesive/flowing.
Message-ID: <587b7a99.952f1c0a.d5983.98ce@mx.google.com>

Author: Tristan Arthur <roguesgonnasneak at gmail.com>
Branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033
Changeset: r89579:f7e0e9cfbf80
Date: 2017-01-15 09:13 +0000
http://bitbucket.org/pypy/pypy/changeset/f7e0e9cfbf80/

Log:	Main lines changed in L77 - L83. Made paragraph more
	cohesive/flowing.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -32,10 +32,10 @@
 standard library.
 
 The language features (including builtin types and functions) are very
-complete and well tested, so if your project does not use many
+refined and well tested, so if your project doesn't use many
 extension modules there is a good chance that it will work with PyPy.
 
-We list the differences we know about in :doc:`cpython differences <cpython_differences>`.
+We list the known differences in :doc:`cpython differences <cpython_differences>`.
 
 
 Module xyz does not work with PyPy: ImportError
@@ -76,9 +76,9 @@
 
 You cannot import *any* extension module in a `sandboxed PyPy`_,
 sorry.  Even the built-in modules available are very limited.
-Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
-it is only a proof of concept.  It seriously requires someone working
-on it.  Before this occurs, it can only be used it for "pure Python"
+Sandboxing in PyPy is a good proof of concept, and is without a doubt
+safe IMHO, however it is only a proof of concept.  It currently requires
+some work and in the meantime, can only be used for "pure Python"
 examples: programs that import mostly nothing (or only pure Python
 modules, recursively).
 

From pypy.commits at gmail.com  Sun Jan 15 08:35:27 2017
From: pypy.commits at gmail.com (TreeStain)
Date: Sun, 15 Jan 2017 05:35:27 -0800 (PST)
Subject: [pypy-commit] pypy
 TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033: Changed
 'examples' to not be plural as only one example is provided. Thsi can be
 reverted if multiple examples are given.
Message-ID: <587b7a9f.145e1c0a.eb01a.52de@mx.google.com>

Author: Tristan Arthur <roguesgonnasneak at gmail.com>
Branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033
Changeset: r89582:49477bd65973
Date: 2017-01-15 12:17 +0000
http://bitbucket.org/pypy/pypy/changeset/49477bd65973/

Log:	Changed 'examples' to not be plural as only one example is provided.
	Thsi can be reverted if multiple examples are given.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -79,7 +79,7 @@
 Sandboxing in PyPy is a good proof of concept, and is without a doubt
 safe IMHO, however it is only a proof of concept.  It currently requires 
 some work from a motivated developer. However, until then it can only be used for "pure Python"
-examples: programs that import mostly nothing (or only pure Python
+example: programs that import mostly nothing (or only pure Python
 modules, recursively).
 
 .. _`sandboxed PyPy`: sandbox.html

From pypy.commits at gmail.com  Sun Jan 15 11:02:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 08:02:05 -0800 (PST)
Subject: [pypy-commit] cffi default: update README
Message-ID: <587b9cfd.896f1c0a.1c9ae.73bd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2858:e247d2f87c48
Date: 2017-01-15 17:01 +0100
http://bitbucket.org/cffi/cffi/changeset/e247d2f87c48/

Log:	update README

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -15,6 +15,7 @@
 
 [Mailing list](https://groups.google.com/forum/#!forum/python-cffi)
 
-To run tests under CPython, run:
+To run tests under CPython, run::
 
-python setup.py build_ext -i
+    python setup.py build_ext -f -i
+    py.test c/ test/

From pypy.commits at gmail.com  Sun Jan 15 11:18:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 08:18:59 -0800 (PST)
Subject: [pypy-commit] cffi default: Add more tips
Message-ID: <587ba0f3.542e1c0a.ccd21.a9f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2859:7aecd7881291
Date: 2017-01-15 17:18 +0100
http://bitbucket.org/cffi/cffi/changeset/7aecd7881291/

Log:	Add more tips

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -15,7 +15,15 @@
 
 [Mailing list](https://groups.google.com/forum/#!forum/python-cffi)
 
+Testing/development tips
+------------------------
+
 To run tests under CPython, run::
 
+    pip install pytest     # if you don't have py.test already
     python setup.py build_ext -f -i
     py.test c/ test/
+
+If you run in another directory (either the tests or another program),
+you should use the environment variable ``PYTHONPATH=/path`` to point
+to the location that contains the ``_cffi_backend.so`` just compiled.

From pypy.commits at gmail.com  Sun Jan 15 11:44:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 08:44:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix three more tests
Message-ID: <587ba6fc.05a81c0a.fba8d.ce43@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89584:2aa8a0a76ecb
Date: 2017-01-15 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/2aa8a0a76ecb/

Log:	fix three more tests

diff --git a/lib-python/3/test/test_finalization.py b/lib-python/3/test/test_finalization.py
--- a/lib-python/3/test/test_finalization.py
+++ b/lib-python/3/test/test_finalization.py
@@ -364,7 +364,8 @@
             ids = [id(s) for s in nodes]
             wrs = [weakref.ref(s) for s in nodes]
             del nodes
-            gc.collect()
+            for cls in classes:   # PyPy: needs several collections for a chain
+                gc.collect()      # of objects all with a __del__()
             self.assert_del_calls(ids)
             self.assert_survivors([])
             self.assertEqual([wr() for wr in wrs], [None] * N)

From pypy.commits at gmail.com  Sun Jan 15 11:44:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 08:44:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Skip the last group of failing tests
Message-ID: <587ba6fe.810b1c0a.48125.cecd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89585:508048339d2f
Date: 2017-01-15 17:44 +0100
http://bitbucket.org/pypy/pypy/changeset/508048339d2f/

Log:	Skip the last group of failing tests

diff --git a/lib-python/3/test/test_finalization.py b/lib-python/3/test/test_finalization.py
--- a/lib-python/3/test/test_finalization.py
+++ b/lib-python/3/test/test_finalization.py
@@ -373,6 +373,11 @@
             self.assert_del_calls(ids)
 
     def check_resurrecting_chain(self, classes):
+        if support.check_impl_detail(pypy=True):
+            self.skipTest("in CPython, in a cycle of objects with __del__(), "
+                          "all the __del__() are called even if some of them "
+                          "resurrect.  In PyPy the recurrection will stop "
+                          "the other objects from being considered as dead.")
         N = len(classes)
         with SimpleBase.test():
             nodes = self.build_chain(classes)

From pypy.commits at gmail.com  Sun Jan 15 11:48:50 2017
From: pypy.commits at gmail.com (sirtom67)
Date: Sun, 15 Jan 2017 08:48:50 -0800 (PST)
Subject: [pypy-commit] cffi sirtom67/float_complex: initial support for
 'float _Complex' and 'double _Complex'
Message-ID: <587ba7f2.46831c0a.bb4ea.c41c@mx.google.com>

Author: Tom Krauss <thomas.p.krauss at gmail.com>
Branch: sirtom67/float_complex
Changeset: r2860:7ad66643d015
Date: 2017-01-15 10:40 -0600
http://bitbucket.org/cffi/cffi/changeset/7ad66643d015/

Log:	initial support for 'float _Complex' and 'double _Complex' Work In
	Progress - sources have printfs that will be removed.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -127,6 +127,8 @@
 #define CT_FUNCTIONPTR      256    /* pointer to function */
 #define CT_VOID             512    /* void */
 
+#define CT_PRIMITIVE_COMPLEX 16777216  /* float _Complex, double _Complex */
+
 /* other flags that may also be set in addition to the base flag: */
 #define CT_IS_VOIDCHAR_PTR       1024
 #define CT_PRIMITIVE_FITS_LONG   2048
@@ -145,7 +147,8 @@
 #define CT_PRIMITIVE_ANY  (CT_PRIMITIVE_SIGNED |        \
                            CT_PRIMITIVE_UNSIGNED |      \
                            CT_PRIMITIVE_CHAR |          \
-                           CT_PRIMITIVE_FLOAT)
+                           CT_PRIMITIVE_FLOAT |         \
+                           CT_PRIMITIVE_COMPLEX)
 
 typedef struct _ctypedescr {
     PyObject_VAR_HEAD
@@ -3855,6 +3858,8 @@
        EPTYPE(f, float, CT_PRIMITIVE_FLOAT )                    \
        EPTYPE(d, double, CT_PRIMITIVE_FLOAT )                   \
        EPTYPE(ld, long double, CT_PRIMITIVE_FLOAT | CT_IS_LONGDOUBLE ) \
+       EPTYPE(fc, float _Complex, CT_PRIMITIVE_COMPLEX )        \
+       EPTYPE(dc, double _Complex, CT_PRIMITIVE_COMPLEX )       \
        ENUM_PRIMITIVE_TYPES_WCHAR                               \
        EPTYPE(b, _Bool, CT_PRIMITIVE_UNSIGNED | CT_IS_BOOL )    \
      /* the following types are not primitive in the C sense */ \
@@ -3925,6 +3930,8 @@
     int name_size;
     ffi_type *ffitype;
 
+    printf("hello\n");
+
     for (ptypes=types; ; ptypes++) {
         if (ptypes->name == NULL) {
 #ifndef HAVE_WCHAR_H
diff --git a/c/parse_c_type.c b/c/parse_c_type.c
--- a/c/parse_c_type.c
+++ b/c/parse_c_type.c
@@ -25,7 +25,7 @@
     /* keywords */
     TOK__BOOL,
     TOK_CHAR,
-    //TOK__COMPLEX,
+    TOK__COMPLEX,
     TOK_CONST,
     TOK_DOUBLE,
     TOK_ENUM,
@@ -797,6 +797,7 @@
 int parse_c_type_from(struct _cffi_parse_info_s *info, size_t *output_index,
                       const char *input)
 {
+    printf("parse_c_type_from\n");
     int result;
     token_t token;
 
diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -101,6 +101,7 @@
 
 static PyObject *build_primitive_type(int num)
 {
+    fprintf(stderr, "fooooooooooooo    num=%d\n",num);
     /* XXX too many translations between here and new_primitive_type() */
     static const char *primitive_name[] = {
         NULL,
@@ -151,6 +152,8 @@
         "uint_fast64_t",
         "intmax_t",
         "uintmax_t",
+        "float _Complex",
+        "double _Complex",
     };
     PyObject *x;
 
diff --git a/cffi/cffi_opcode.py b/cffi/cffi_opcode.py
--- a/cffi/cffi_opcode.py
+++ b/cffi/cffi_opcode.py
@@ -105,8 +105,11 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -128,6 +131,8 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
     'int8_t':             PRIM_INT8,
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -117,6 +117,8 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
+        'float _Complex':     'f',
+        'double _Complex':    'f',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
diff --git a/cffi/parse_c_type.h b/cffi/parse_c_type.h
--- a/cffi/parse_c_type.h
+++ b/cffi/parse_c_type.h
@@ -79,8 +79,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)

From pypy.commits at gmail.com  Sun Jan 15 11:53:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 08:53:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Use the __qualname__ of the class when
 computing the default repr of instances
Message-ID: <587ba906.85e11c0a.9bfc0.c0dc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89586:c2ed167889ea
Date: 2017-01-15 17:52 +0100
http://bitbucket.org/pypy/pypy/changeset/c2ed167889ea/

Log:	Use the __qualname__ of the class when computing the default repr of
	instances

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -715,7 +715,7 @@
 
     def getfulltypename(self, w_obj):
         w_type = self.type(w_obj)
-        classname = w_type.name.decode('utf-8')
+        classname = w_type.getqualname(self)
         if w_type.is_heaptype():
             w_module = w_type.lookup("__module__")
             if w_module is not None:
diff --git a/pypy/objspace/std/test/test_userobject.py b/pypy/objspace/std/test/test_userobject.py
--- a/pypy/objspace/std/test/test_userobject.py
+++ b/pypy/objspace/std/test/test_userobject.py
@@ -221,8 +221,9 @@
         class Foo(object):
             pass
         Foo.__module__ = 'a.b.c'
+        Foo.__qualname__ = 'd.Foo'
         s = repr(Foo())
-        assert s.startswith('<a.b.c.Foo object at ')
+        assert s.startswith('<a.b.c.d.Foo object at ')
 
     def test_repr_nonascii(self):
         Japan = type('&#26085;&#26412;', (), dict(__module__='&#26085;&#26412;&#22269;'))

From pypy.commits at gmail.com  Sun Jan 15 12:01:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 09:01:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <587baad5.068f1c0a.5f5f0.cc80@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89587:08640fb67b4d
Date: 2017-01-15 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/08640fb67b4d/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -255,7 +255,7 @@
         log = self.run("""
         def main(n):
             for i in range(n):
-                unicode(str(i))
+                (b"x" * (i & 15)).decode('ascii')
             return i
         """, [1000])
         loop, = log.loops_by_filename(self.filepath)
@@ -263,11 +263,13 @@
         i49 = int_lt(i47, i24)
         guard_true(i49, descr=...)
         i50 = int_add(i47, 1)
-        setfield_gc(p15, i50, descr=<FieldS pypy.module.__builtin__.functional.W_XRangeIterator.inst_current 8>)
+        i53 = int_and(i47, 15)
+        setfield_gc(p15, i50, descr=<FieldS pypy.module.__builtin__.functional.W_IntRangeIterator.inst_current 8>)
+        i55 = int_le(i53, 0)
+        guard_false(i55, descr=...)
+        p80 = call_r(ConstClass(ll_char_mul__Char_Signed), 120, i53, descr=<Callr . ii EF=3>)
+        guard_no_exception(descr=...)
         guard_not_invalidated(descr=...)
-        p80 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i47, descr=<Callr . i EF=3>)
-        guard_no_exception(descr=...)
-        guard_nonnull(p80, descr=...)
         p53 = call_r(ConstClass(fast_str_decode_ascii), p80, descr=<Callr . r EF=4>)
         guard_no_exception(descr=...)
         guard_nonnull(p53, descr=...)

From pypy.commits at gmail.com  Sun Jan 15 12:34:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 09:34:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <587bb2a5.52301c0a.536b6.edf4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89589:d03607c05f25
Date: 2017-01-15 18:06 +0100
http://bitbucket.org/pypy/pypy/changeset/d03607c05f25/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -80,7 +80,7 @@
             i113 = call_may_force_i(ConstClass(str_decode_utf_8_impl), p103, 1, ConstPtr(null), 1, 0, 0, p104, descr=<Calli . ririiir EF=7>)
             guard_not_forced(descr=...)
             guard_no_exception(descr=...)
-            p116 = call_r(ConstClass(ll_build_trampoline__v1351___simple_call__function_), p104, descr=<Callr . r EF=5>)
+            p116 = call_r(ConstClass(ll_build_trampoline__), p104, descr=<Callr . r EF=5>)
             guard_no_exception(descr=...)
             guard_nonnull(p116, descr=...)
             p118 = getfield_gc_r(ConstPtr(ptr117), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)

From pypy.commits at gmail.com  Sun Jan 15 12:34:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 09:34:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <587bb2a2.0e821c0a.63665.f2af@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89588:8ef6606d7fc8
Date: 2017-01-15 18:01 +0100
http://bitbucket.org/pypy/pypy/changeset/8ef6606d7fc8/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -245,7 +245,7 @@
         i45 = int_lt(i43, i26)
         guard_true(i45, descr=...)
         i46 = int_add(i43, 1)
-        setfield_gc(p15, i46, descr=<FieldS pypy.module.__builtin__.functional.W_XRangeIterator.inst_current 8>)
+        setfield_gc(p15, i46, descr=<FieldS pypy.module.__builtin__.functional.W_IntRangeIterator.inst_current 8>)
         guard_not_invalidated(descr=...)
         --TICK--
         jump(..., descr=...)

From pypy.commits at gmail.com  Sun Jan 15 12:34:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 09:34:32 -0800 (PST)
Subject: [pypy-commit] pypy default: Add a couple of jit.elidable
Message-ID: <587bb2a8.44641c0a.ad077.898e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89590:acfeb77833ff
Date: 2017-01-15 18:33 +0100
http://bitbucket.org/pypy/pypy/changeset/acfeb77833ff/

Log:	Add a couple of jit.elidable

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -1,5 +1,6 @@
 """The builtin str implementation"""
 
+from rpython.rlib import jit
 from rpython.rlib.jit import we_are_jitted
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin)
@@ -950,6 +951,7 @@
 W_BytesObject.typedef.flag_sequence_bug_compat = True
 
 
+ at jit.elidable
 def string_escape_encode(s, quote):
     buf = StringBuilder(len(s) + 2)
 
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1378,7 +1378,12 @@
         CHR = chr
 
     def unicode_escape(s, size, errors, errorhandler=None):
-        # errorhandler is not used: this function cannot cause Unicode errors
+        # errors and errorhandler are not used: this function cannot cause
+        # Unicode errors
+        return _unicode_escape(s, size)
+
+    @jit.elidable
+    def _unicode_escape(s, size):
         result = STRING_BUILDER(size)
 
         if quotes:

From pypy.commits at gmail.com  Sun Jan 15 12:34:34 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 09:34:34 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Document failing test
Message-ID: <587bb2aa.c3e31c0a.ebbbe.ee1f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89591:e0add63d8c7a
Date: 2017-01-15 18:33 +0100
http://bitbucket.org/pypy/pypy/changeset/e0add63d8c7a/

Log:	Document failing test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -218,6 +218,7 @@
         assert loop.match_by_id('calltwo', '')    # nothing
 
     def test_move_method_call_out_of_loop(self):
+        # XXX not implemented: lower() on unicodes is not considered elidable
         def main(n):
             lst = []
             s = 'Hello %d' % n

From pypy.commits at gmail.com  Sun Jan 15 13:52:23 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 15 Jan 2017 10:52:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: read1 requires a size arg
Message-ID: <587bc4e7.43e61c0a.451d3.316a@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89592:3d8aabbf51c6
Date: 2017-01-15 10:42 -0800
http://bitbucket.org/pypy/pypy/changeset/3d8aabbf51c6/

Log:	read1 requires a size arg

diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -78,6 +78,9 @@
         size = convert_size(space, w_size)
         return space.newbytes(self.read(size))
 
+    def read1_w(self, space, w_size):
+        self.read_w(self, space, w_size)
+
     def readline_w(self, space, w_limit=None):
         self._check_closed(space)
         limit = convert_size(space, w_limit)
@@ -205,7 +208,7 @@
     __init__  = interp2app(W_BytesIO.descr_init),
 
     read = interp2app(W_BytesIO.read_w),
-    read1 = interp2app(W_BytesIO.read_w),
+    read1 = interp2app(W_BytesIO.read1_w),
     readline = interp2app(W_BytesIO.readline_w),
     readinto = interp2app(W_BytesIO.readinto_w),
     readinto1 = interp2app(W_BytesIO.readinto_w),
diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py
--- a/pypy/module/_io/test/test_bytesio.py
+++ b/pypy/module/_io/test/test_bytesio.py
@@ -143,6 +143,12 @@
         memio.close()
         raises(ValueError, memio.getbuffer)
 
+    def test_read1(self):
+        import _io
+        memio = _io.BytesIO(b"1234567890")
+        raises(TypeError, memio.read1)
+        assert memio.read() == b"1234567890"
+
     def test_readline(self):
         import _io
         f = _io.BytesIO(b'abc\ndef\nxyzzy\nfoo\x00bar\nanother line')

From pypy.commits at gmail.com  Sun Jan 15 13:55:20 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 15 Jan 2017 10:55:20 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: oops
Message-ID: <587bc598.518a1c0a.ef230.28d9@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89593:bf9d15d22fb6
Date: 2017-01-15 10:52 -0800
http://bitbucket.org/pypy/pypy/changeset/bf9d15d22fb6/

Log:	oops

diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -79,7 +79,7 @@
         return space.newbytes(self.read(size))
 
     def read1_w(self, space, w_size):
-        self.read_w(self, space, w_size)
+        return self.read_w(space, w_size)
 
     def readline_w(self, space, w_limit=None):
         self._check_closed(space)

From pypy.commits at gmail.com  Sun Jan 15 14:15:01 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 15 Jan 2017 11:15:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add context management
Message-ID: <587bca35.05371c0a.39090.44bd@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89594:a2d0e3000c1f
Date: 2017-01-15 11:13 -0800
http://bitbucket.org/pypy/pypy/changeset/a2d0e3000c1f/

Log:	add context management

diff --git a/lib-python/3/test/test_dbm_gnu.py b/lib-python/3/test/test_dbm_gnu.py
--- a/lib-python/3/test/test_dbm_gnu.py
+++ b/lib-python/3/test/test_dbm_gnu.py
@@ -90,8 +90,7 @@
 
         with self.assertRaises(gdbm.error) as cm:
             db.keys()
-        self.assertEqual(str(cm.exception),
-                         "GDBM object has already been closed")
+        self.assertIn("GDBM object has already been closed", str(cm.exception))
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib_pypy/_dbm.py b/lib_pypy/_dbm.py
--- a/lib_pypy/_dbm.py
+++ b/lib_pypy/_dbm.py
@@ -112,6 +112,13 @@
         if status < 0:
             raise KeyError(key)
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        self.close()
+
+
 ### initialization: Berkeley DB versus normal DB
 
 def _init_func(name, argtypes=None, restype=None):
diff --git a/lib_pypy/_gdbm.py b/lib_pypy/_gdbm.py
--- a/lib_pypy/_gdbm.py
+++ b/lib_pypy/_gdbm.py
@@ -150,6 +150,13 @@
         self[key] = default
         return default
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        self.close()
+
+
 def open(filename, flags='r', mode=0o666):
     if not isinstance(filename, str):
         raise TypeError("must be str, not %s" % type(filename).__name__)

From pypy.commits at gmail.com  Sun Jan 15 14:29:05 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 15 Jan 2017 11:29:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: restore our xfail,
 adapt is_resource_enabled
Message-ID: <587bcd81.44641c0a.ad077.dc78@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89595:c7ca4451d1a3
Date: 2017-01-15 11:28 -0800
http://bitbucket.org/pypy/pypy/changeset/c7ca4451d1a3/

Log:	restore our xfail, adapt is_resource_enabled

diff --git a/lib-python/3/ctypes/test/__init__.py b/lib-python/3/ctypes/test/__init__.py
--- a/lib-python/3/ctypes/test/__init__.py
+++ b/lib-python/3/ctypes/test/__init__.py
@@ -12,3 +12,16 @@
 
 def load_tests(*args):
     return support.load_package_tests(os.path.dirname(__file__), *args)
+
+def xfail(method):
+    """
+    Poor's man xfail: remove it when all the failures have been fixed
+    """
+    def new_method(self, *args, **kwds):
+        try:
+            method(self, *args, **kwds)
+        except:
+            pass
+        else:
+            self.assertTrue(False, "DID NOT RAISE")
+    return new_method
diff --git a/lib-python/3/ctypes/test/test_python_api.py b/lib-python/3/ctypes/test/test_python_api.py
--- a/lib-python/3/ctypes/test/test_python_api.py
+++ b/lib-python/3/ctypes/test/test_python_api.py
@@ -9,8 +9,10 @@
 
 ################################################################
 
-if is_resource_enabled("refcount"):
+try:
     from sys import getrefcount as grc
+except ImportError:
+    grc = None
 if sys.version_info > (2, 4):
     c_py_ssize_t = c_size_t
 else:

From pypy.commits at gmail.com  Sun Jan 15 14:54:33 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 15 Jan 2017 11:54:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add some docstrings expected by
 test_doctest, adapt its counts
Message-ID: <587bd379.068f1c0a.5f5f0.4bea@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89596:1855c0def876
Date: 2017-01-15 11:53 -0800
http://bitbucket.org/pypy/pypy/changeset/1855c0def876/

Log:	add some docstrings expected by test_doctest, adapt its counts

diff --git a/lib-python/3/test/test_doctest.py b/lib-python/3/test/test_doctest.py
--- a/lib-python/3/test/test_doctest.py
+++ b/lib-python/3/test/test_doctest.py
@@ -8,6 +8,7 @@
 import os
 import sys
 
+is_pypy = support.check_impl_detail(pypy=True)
 
 # NOTE: There are some additional tests relating to interaction with
 #       zipimport in the test_zipimport_support test module.
@@ -659,7 +660,8 @@
 
     >>> import builtins
     >>> tests = doctest.DocTestFinder().find(builtins)
-    >>> 790 < len(tests) < 810 # approximate number of objects with docstrings
+    >>> lo, hi = (120, 140) if is_pypy else (790, 810)
+    >>> lo < len(tests) < hi # approximate number of objects with docstrings
     True
     >>> real_tests = [t for t in tests if len(t.examples) > 0]
     >>> len(real_tests) # objects that actually have doctests
diff --git a/pypy/module/__builtin__/app_operation.py b/pypy/module/__builtin__/app_operation.py
--- a/pypy/module/__builtin__/app_operation.py
+++ b/pypy/module/__builtin__/app_operation.py
@@ -1,16 +1,31 @@
 import _operator
 
 def bin(x):
-    """Return the binary representation of an integer."""
+    """Return the binary representation of an integer.
+
+    >>> bin(2796202)
+    '0b1010101010101010101010'
+
+    """
     value = _operator.index(x)
     return value.__format__("#b")
 
 def oct(x):
-    """Return the octal representation of an integer."""
+    """Return the octal representation of an integer.
+
+    >>> oct(342391)
+    '0o1234567'
+
+    """
     x = _operator.index(x)
     return x.__format__("#o")
 
 def hex(x):
-    """Return the hexadecimal representation of an integer."""
+    """Return the hexadecimal representation of an integer.
+
+    >>> hex(12648430)
+    '0xc0ffee'
+
+    """
     x = _operator.index(x)
     return x.__format__("#x")
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -246,6 +246,15 @@
     @staticmethod
     @unwrap_spec(s=str)
     def descr_fromhex(space, w_cls, s):
+        """float.fromhex(string) -> float
+
+        Create a floating-point number from a hexadecimal string.
+        >>> float.fromhex('0x1.ffffp10')
+        2047.984375
+        >>> float.fromhex('-0x1p-1074')
+        -5e-324
+
+        """
         length = len(s)
         i = 0
         value = 0.0
@@ -592,6 +601,20 @@
         return space.wrap(math.floor(v) == v)
 
     def descr_as_integer_ratio(self, space):
+        """float.as_integer_ratio() -> (int, int)
+
+        Return a pair of integers, whose ratio is exactly equal to the
+        original float and with a positive denominator.  Raise
+        OverflowError on infinities and a ValueError on NaNs.
+
+        >>> (10.0).as_integer_ratio()
+        (10, 1)
+        >>> (0.0).as_integer_ratio()
+        (0, 1)
+        >>> (-.25).as_integer_ratio()
+        (-1, 4)
+
+        """
         value = self.floatval
         try:
             num, den = float_as_rbigint_ratio(value)
@@ -608,6 +631,17 @@
         return space.newtuple([space.int(w_num), space.int(w_den)])
 
     def descr_hex(self, space):
+        """float.hex() -> string
+
+        Return a hexadecimal representation of a floating-point
+        number.
+
+        >>> (-0.1).hex()
+        '-0x1.999999999999ap-4'
+        >>> 3.14159.hex()
+        '0x1.921f9f01b866ep+1'
+
+        """
         TOHEX_NBITS = rfloat.DBL_MANT_DIG + 3 - (rfloat.DBL_MANT_DIG + 2) % 4
         value = self.floatval
         if not isfinite(value):

From pypy.commits at gmail.com  Sun Jan 15 15:56:00 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 15 Jan 2017 12:56:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: o stdin/out now default to
 surrogateescape in C locale
Message-ID: <587be1e0.810b1c0a.48125.85b8@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89597:31367d8c9631
Date: 2017-01-15 12:55 -0800
http://bitbucket.org/pypy/pypy/changeset/31367d8c9631/

Log:	o stdin/out now default to surrogateescape in C locale o tighten
	PYTHONIOENCODING parsing

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -291,8 +291,15 @@
     try:
         if encoding and ':' in encoding:
             encoding, errors = encoding.split(':', 1)
+            encoding = encoding or None
+            errors = errors or None
         else:
             errors = None
+        if not (encoding or errors):
+            # stdin/out default to surrogateescape in C locale
+            import _locale
+            if _locale.setlocale(_locale.LC_CTYPE, None) == 'C':
+                errors = 'surrogateescape'
 
         sys.stderr = sys.__stderr__ = create_stdio(
             2, True, "<stderr>", encoding, 'backslashreplace', unbuffered)
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -992,6 +992,24 @@
             data = self.run(p, env=env)
             assert data == expected
 
+    def test_pythonioencoding2(self):
+        for encoding, expected in [
+            ("ascii:", "strict"),
+            (":surrogateescape", "surrogateescape"),
+        ]:
+            p = getscript_in_dir("import sys; print(sys.stdout.errors, end='')")
+            env = os.environ.copy()
+            env["PYTHONIOENCODING"] = encoding
+            data = self.run(p, env=env)
+            assert data == expected
+
+    def test_pythonioencoding_c_locale(self):
+        p = getscript_in_dir("import sys; print(sys.stdout.errors, end='')")
+        env = os.environ.copy()
+        env["LC_ALL"] = "C"
+        data = self.run(p, env=env)
+        assert data == "surrogateescape"
+
     def test_sys_exit_pythonioencoding(self):
         if sys.version_info < (2, 7):
             skip("test required Python >= 2.7")

From pypy.commits at gmail.com  Sun Jan 15 21:25:43 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 15 Jan 2017 18:25:43 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: cleanup
Message-ID: <587c2f27.c80e1c0a.b54e2.19f4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89598:3be3239aea44
Date: 2017-01-16 00:29 +0000
http://bitbucket.org/pypy/pypy/changeset/3be3239aea44/

Log:	cleanup

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -668,9 +668,18 @@
 class DelayedStruct(object):
     def __init__(self, name, fields, TYPE):
         self.struct_name = name
+        self.type_name = None
         self.fields = fields
         self.TYPE = TYPE
 
+    def get_type_name(self):
+        if self.type_name is not None:
+            return self.type_name
+        elif not self.struct_name.startswith('$'):
+            return 'struct %s' % self.struct_name
+        else:
+            raise ValueError('Anonymous struct')
+
     def __repr__(self):
         return "<struct {struct_name}>".format(**vars(self))
 
@@ -687,6 +696,7 @@
         self._Config = type('Config', (object,), {})
         self._TYPES = {}
         self.includes = []
+        self.struct_typedefs = {}
 
     def include(self, other):
         self.ctx.include(other.ctx)
@@ -697,7 +707,9 @@
         assert name not in self.definitions
         tp = self.convert_type(obj, quals)
         if isinstance(tp, DelayedStruct):
-            tp = self.realize_struct(tp, name, configure_now=configure_now)
+            if tp.type_name is None:
+                tp.type_name = name
+            tp = self.realize_struct(tp, configure_now=configure_now)
         self.definitions[name] = tp
 
     def add_macro(self, name, value):
@@ -713,11 +725,18 @@
         if obj.fldtypes is not None:
             struct.fields = zip(
                  obj.fldnames,
-                 [self.convert_type(field) for field in obj.fldtypes])
+                 [self.convert_field(field) for field in obj.fldtypes])
         return struct
 
-    def realize_struct(self, struct, type_name, configure_now=False):
+    def convert_field(self, obj):
+        tp = self.convert_type(obj)
+        if isinstance(tp, DelayedStruct):
+            tp = tp.TYPE
+        return tp
+
+    def realize_struct(self, struct, configure_now=False):
         from pypy.module.cpyext.api import cpython_struct
+        type_name = struct.get_type_name()
         configname = type_name.replace(' ', '__')
         if configure_now:
             setattr(self._Config, configname,

From pypy.commits at gmail.com  Sun Jan 15 21:25:45 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 15 Jan 2017 18:25:45 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Move PyHeapTypeObject definition
 to C
Message-ID: <587c2f29.4f831c0a.33253.20e7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89599:85a419c311cb
Date: 2017-01-16 02:24 +0000
http://bitbucket.org/pypy/pypy/changeset/85a419c311cb/

Log:	Move PyHeapTypeObject definition to C

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -149,6 +149,7 @@
     for name in ["pypy_macros.h"] + FUNCTIONS_BY_HEADER.keys():
         headers.append(udir.join(name))
     headers.append(parse_dir / 'cpyext_object.h')
+    headers.append(parse_dir / 'cpyext_typeobject.h')
     _copy_header_files(headers, dstdir)
 
     if copy_numpy_headers:
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -112,14 +112,7 @@
 #define PyBUF_SHADOW 0x400
 /* end Py3k buffer interface */
 
-typedef struct {
-    PyTypeObject ht_type;
-    PyNumberMethods as_number;
-    PyMappingMethods as_mapping;
-    PySequenceMethods as_sequence;
-    PyBufferProcs as_buffer;
-    PyObject *ht_name, *ht_slots;
-} PyHeapTypeObject;
+#include <cpyext_typeobject.h>
 
 #define PyObject_Bytes PyObject_Str
 
diff --git a/pypy/module/cpyext/parse/cpyext_typeobject.h b/pypy/module/cpyext/parse/cpyext_typeobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_typeobject.h
@@ -0,0 +1,8 @@
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -17,7 +17,8 @@
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
     PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
-    Py_TPFLAGS_HAVE_INPLACEOPS)
+    Py_TPFLAGS_HAVE_INPLACEOPS, object_h, parse_dir)
+from pypy.module.cpyext.cparser import parse_source
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -32,7 +33,7 @@
 from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
 from pypy.module.cpyext.typeobjectdefs import (
     PyGetSetDef, PyMemberDef, newfunc, getter, setter,
-    PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs)
+    PyNumberMethods, PySequenceMethods, PyBufferProcs)
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 
@@ -40,18 +41,11 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
-PyHeapTypeObjectStruct = lltype.ForwardReference()
-PyHeapTypeObject = lltype.Ptr(PyHeapTypeObjectStruct)
-PyHeapTypeObjectFields = (
-    ("ht_type", PyTypeObject),
-    ("ht_name", PyObject),
-    ("as_number", PyNumberMethods),
-    ("as_mapping", PyMappingMethods),
-    ("as_sequence", PySequenceMethods),
-    ("as_buffer", PyBufferProcs),
-    )
-cpython_struct("PyHeapTypeObject", PyHeapTypeObjectFields, PyHeapTypeObjectStruct,
-               level=2)
+cdef = (parse_dir / 'cpyext_typeobject.h').read()
+typeobject_h = parse_source(cdef, includes=[object_h], configure_now=True)
+PyHeapTypeObjectStruct = typeobject_h.gettype('PyHeapTypeObject')
+PyHeapTypeObject = typeobject_h.gettype('PyHeapTypeObject *')
+
 
 class W_GetSetPropertyEx(GetSetProperty):
     def __init__(self, getset, w_type):

From pypy.commits at gmail.com  Sun Jan 15 21:45:34 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 15 Jan 2017 18:45:34 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Always use configure_now=True in
 parse_source() and kill the option
Message-ID: <587c33ce.05371c0a.39090.2e06@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89600:3331e359a75e
Date: 2017-01-16 02:44 +0000
http://bitbucket.org/pypy/pypy/changeset/3331e359a75e/

Log:	Always use configure_now=True in parse_source() and kill the option

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -672,7 +672,7 @@
 
 object_cdef = (parse_dir / 'cpyext_object.h').read()
 object_h = parse_source(object_cdef,
-    headers=['sys/types.h', 'stdarg.h', 'stdio.h'], configure_now=True)
+    headers=['sys/types.h', 'stdarg.h', 'stdio.h'])
 
 Py_ssize_t = object_h.gettype('Py_ssize_t')
 Py_ssize_tP = object_h.gettype('Py_ssize_t *')
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -703,13 +703,13 @@
         self.structs.update(other.structs)
         self.includes.append(other)
 
-    def add_typedef(self, name, obj, quals, configure_now=False):
+    def add_typedef(self, name, obj, quals):
         assert name not in self.definitions
         tp = self.convert_type(obj, quals)
         if isinstance(tp, DelayedStruct):
             if tp.type_name is None:
                 tp.type_name = name
-            tp = self.realize_struct(tp, configure_now=configure_now)
+            tp = self.realize_struct(tp)
         self.definitions[name] = tp
 
     def add_macro(self, name, value):
@@ -734,16 +734,12 @@
             tp = tp.TYPE
         return tp
 
-    def realize_struct(self, struct, configure_now=False):
-        from pypy.module.cpyext.api import cpython_struct
+    def realize_struct(self, struct):
         type_name = struct.get_type_name()
         configname = type_name.replace(' ', '__')
-        if configure_now:
-            setattr(self._Config, configname,
-                rffi_platform.Struct(type_name, struct.fields))
-            self._TYPES[configname] = struct.TYPE
-        else:
-            cpython_struct(type_name, struct.fields, forward=struct.TYPE)
+        setattr(self._Config, configname,
+            rffi_platform.Struct(type_name, struct.fields))
+        self._TYPES[configname] = struct.TYPE
         return struct.TYPE
 
     def build_eci(self):
@@ -756,13 +752,13 @@
         return ExternalCompilationInfo(
             post_include_bits=all_sources, includes=all_headers)
 
-    def configure_types(self, configure_now=False):
+    def configure_types(self):
         for name, (obj, quals) in self.ctx._declarations.iteritems():
             if obj in self.ctx._included_declarations:
                 continue
             if name.startswith('typedef '):
                 name = name[8:]
-                self.add_typedef(name, obj, quals, configure_now=configure_now)
+                self.add_typedef(name, obj, quals)
             elif name.startswith('macro '):
                 name = name[6:]
                 self.add_macro(name, obj)
@@ -825,12 +821,12 @@
         return decl.name, FUNCP.TO
 
 
-def parse_source(source, includes=None, headers=None, configure_now=False):
+def parse_source(source, includes=None, headers=None, configure_now=True):
     ctx = Parser()
     src = ParsedSource(source, ctx, headers=headers)
     if includes is not None:
         for header in includes:
             src.include(header)
     ctx.parse(source)
-    src.configure_types(configure_now=configure_now)
+    src.configure_types()
     return src
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -11,7 +11,7 @@
         double ob_fval;
     } TestFloatObject;
     """
-    res = parse_source(decl, configure_now=True)
+    res = parse_source(decl)
     TestFloatObject = res.definitions['TestFloatObject']
     assert isinstance(TestFloatObject, lltype.Struct)
     assert TestFloatObject.c_ob_refcnt == rffi.SSIZE_T
@@ -59,10 +59,10 @@
         Type *type;
     } Object;
     """
-    hdr1 = parse_source(cdef1, configure_now=True)
+    hdr1 = parse_source(cdef1)
     Type = hdr1.definitions['Type']
     assert isinstance(Type, lltype.Struct)
-    hdr2 = parse_source(cdef2, includes=[hdr1], configure_now=True)
+    hdr2 = parse_source(cdef2, includes=[hdr1])
     assert 'Type' not in hdr2.definitions
     Object = hdr2.definitions['Object']
     assert Object.c_type.TO is Type
@@ -85,7 +85,7 @@
     """
     foo_h = parse_source(cdef)
     Object = foo_h.gettype('Object')
-    assert isinstance(Object, lltype.ForwardReference)
+    assert isinstance(Object, lltype.Struct)
 
 def test_recursive():
     cdef = """
@@ -106,7 +106,7 @@
         Object *obj;
     } Type;
     """
-    foo_h = parse_source(cdef, configure_now=True)
+    foo_h = parse_source(cdef)
     Object = foo_h.definitions['Object']
     assert isinstance(Object, lltype.Struct)
     hash(Object)
@@ -117,7 +117,7 @@
         const char * const foo;
     } bar;
     """
-    hdr = parse_source(cdef, configure_now=True)
+    hdr = parse_source(cdef)
     assert hdr.definitions['bar'].c_foo == rffi.CONST_CCHARP != rffi.CCHARP
 
 def test_gettype():
@@ -133,7 +133,7 @@
         double ob_fval;
     } TestFloatObject;
     """
-    res = parse_source(decl, configure_now=True)
+    res = parse_source(decl)
     assert res.gettype('Py_ssize_t') == rffi.SSIZE_T
     assert res.gettype('TestFloatObject *').TO.c_ob_refcnt == rffi.SSIZE_T
 
@@ -152,7 +152,7 @@
 
     typedef TestFloatObject* (*func_t)(int, int);
     """
-    res = parse_source(decl, configure_now=True)
+    res = parse_source(decl)
     name, FUNC = res.parse_func("func_t some_func(TestFloatObject*)")
     assert name == 'some_func'
     assert FUNC.RESULT == res.gettype('func_t')
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -42,7 +42,7 @@
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
 cdef = (parse_dir / 'cpyext_typeobject.h').read()
-typeobject_h = parse_source(cdef, includes=[object_h], configure_now=True)
+typeobject_h = parse_source(cdef, includes=[object_h])
 PyHeapTypeObjectStruct = typeobject_h.gettype('PyHeapTypeObject')
 PyHeapTypeObject = typeobject_h.gettype('PyHeapTypeObject *')
 

From pypy.commits at gmail.com  Mon Jan 16 00:04:48 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 15 Jan 2017 21:04:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix PYTHONIOENCODING=""
Message-ID: <587c5470.d30f1c0a.95ccc.dd1d@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89601:2fb902af2c9c
Date: 2017-01-15 20:55 -0800
http://bitbucket.org/pypy/pypy/changeset/2fb902af2c9c/

Log:	fix PYTHONIOENCODING=""

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -291,10 +291,10 @@
     try:
         if encoding and ':' in encoding:
             encoding, errors = encoding.split(':', 1)
-            encoding = encoding or None
             errors = errors or None
         else:
             errors = None
+        encoding = encoding or None
         if not (encoding or errors):
             # stdin/out default to surrogateescape in C locale
             import _locale
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1004,11 +1004,17 @@
             assert data == expected
 
     def test_pythonioencoding_c_locale(self):
-        p = getscript_in_dir("import sys; print(sys.stdout.errors, end='')")
-        env = os.environ.copy()
-        env["LC_ALL"] = "C"
-        data = self.run(p, env=env)
-        assert data == "surrogateescape"
+        for encoding, expected in [
+            (None, "surrogateescape"),
+            ("", "surrogateescape")
+        ]:
+            p = getscript_in_dir("import sys; print(sys.stdout.errors, end='')")
+            env = os.environ.copy()
+            env["LC_ALL"] = "C"
+            if encoding is not None:
+                env["PYTHONIOENCODING"] = encoding
+            data = self.run(p, env=env)
+            assert data == "surrogateescape"
 
     def test_sys_exit_pythonioencoding(self):
         if sys.version_info < (2, 7):

From pypy.commits at gmail.com  Mon Jan 16 00:04:50 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 15 Jan 2017 21:04:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: missing import
Message-ID: <587c5472.542e1c0a.ccd21.57d9@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89602:2bff68cef0d8
Date: 2017-01-15 21:01 -0800
http://bitbucket.org/pypy/pypy/changeset/2bff68cef0d8/

Log:	missing import

diff --git a/lib-python/3/ctypes/test/test_python_api.py b/lib-python/3/ctypes/test/test_python_api.py
--- a/lib-python/3/ctypes/test/test_python_api.py
+++ b/lib-python/3/ctypes/test/test_python_api.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import xfail
 import unittest, sys
 from test import support
 

From pypy.commits at gmail.com  Mon Jan 16 02:43:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 23:43:21 -0800 (PST)
Subject: [pypy-commit] pypy default: Revert to gmail.com again because
 verisign makes test_connection pass
Message-ID: <587c7999.ce841c0a.6102b.cdf6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89603:297ff07dc274
Date: 2017-01-16 08:42 +0100
http://bitbucket.org/pypy/pypy/changeset/297ff07dc274/

Log:	Revert to gmail.com again because verisign makes test_connection
	pass but most other tests fail :-(

	Work around the fact that the first connection attempt fails from
	some machines.

diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -175,8 +175,8 @@
     }
 
     def setup_method(self, method):
-        # https://www.verisign.net/
-        ADDR = "www.verisign.net", 443
+        # https://gmail.com/
+        ADDR = "gmail.com", 443
 
         self.w_s = self.space.appexec([self.space.wrap(ADDR)], """(ADDR):
             import socket
@@ -194,6 +194,17 @@
         from ..interp_ssl import SOCKET_STORAGE
         SOCKET_STORAGE._dict.clear()
 
+    def test_warmup_connection(self):
+        # not sure it is gmail.com's fault, but on some machines the
+        # very first connection attempt fails.  So we make one here and
+        # ignore the result.  The first real test is test_connect().
+        import socket, ssl
+        try:
+            ss = socket.ssl(self.s)
+            self.s.close()
+        except ssl.SSLError:
+            pass
+
     def test_connect(self):
         import socket, gc
         ss = socket.ssl(self.s)

From pypy.commits at gmail.com  Mon Jan 16 02:44:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 23:44:23 -0800 (PST)
Subject: [pypy-commit] pypy default: empty branch document
Message-ID: <587c79d7.896f1c0a.1c9ae.96ee@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89604:08b6741f85eb
Date: 2017-01-16 08:43 +0100
http://bitbucket.org/pypy/pypy/changeset/08b6741f85eb/

Log:	empty branch document

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -105,3 +105,6 @@
 .. branch: issue2464
 
 Give (almost?) all GetSetProperties a valid __objclass__.
+
+.. branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419
+.. branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033

From pypy.commits at gmail.com  Mon Jan 16 02:57:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 23:57:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix (see _cffi_backend/test)
Message-ID: <587c7cdb.4f831c0a.33253.d774@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89605:c58c4ce4b2ad
Date: 2017-01-16 08:56 +0100
http://bitbucket.org/pypy/pypy/changeset/c58c4ce4b2ad/

Log:	Fix (see _cffi_backend/test)

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -715,8 +715,8 @@
 
     def getfulltypename(self, w_obj):
         w_type = self.type(w_obj)
-        classname = w_type.getqualname(self)
         if w_type.is_heaptype():
+            classname = w_type.getqualname(self)
             w_module = w_type.lookup("__module__")
             if w_module is not None:
                 try:
@@ -726,4 +726,6 @@
                         raise
                 else:
                     classname = u'%s.%s' % (modulename, classname)
+        else:
+            classname = w_type.name.decode('utf-8')
         return classname

From pypy.commits at gmail.com  Mon Jan 16 02:58:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 15 Jan 2017 23:58:35 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix for test_ztranslation
Message-ID: <587c7d2b.c4811c0a.28521.c15e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89606:a07ce8aafea1
Date: 2017-01-16 08:57 +0100
http://bitbucket.org/pypy/pypy/changeset/a07ce8aafea1/

Log:	fix for test_ztranslation

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -212,6 +212,8 @@
     def newutf8(self, x):
         return w_some_obj()
 
+    newtext = newutf8
+
     @specialize.argtype(1)
     def wrap(self, x):
         if not we_are_translated():

From pypy.commits at gmail.com  Mon Jan 16 04:10:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 16 Jan 2017 01:10:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip test on pypy
Message-ID: <587c8dfe.46bb1c0a.b5632.e7f5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89607:7c74a79913f5
Date: 2017-01-16 10:09 +0100
http://bitbucket.org/pypy/pypy/changeset/7c74a79913f5/

Log:	skip test on pypy

diff --git a/lib-python/3/test/test_logging.py b/lib-python/3/test/test_logging.py
--- a/lib-python/3/test/test_logging.py
+++ b/lib-python/3/test/test_logging.py
@@ -3407,6 +3407,7 @@
         logging.setLoggerClass(logging.Logger)
         self.assertEqual(logging.getLoggerClass(), logging.Logger)
 
+    @support.cpython_only    # PyPy doesn't call __del__() at shutdown
     def test_logging_at_shutdown(self):
         # Issue #20037
         code = """if 1:

From pypy.commits at gmail.com  Mon Jan 16 09:57:06 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 06:57:06 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Rename ParsedSource to CTypeSpace
Message-ID: <587cdf42.952f1c0a.d5983.f810@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89608:edc637d97ac6
Date: 2017-01-16 14:56 +0000
http://bitbucket.org/pypy/pypy/changeset/edc637d97ac6/

Log:	Rename ParsedSource to CTypeSpace

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -684,9 +684,8 @@
         return "<struct {struct_name}>".format(**vars(self))
 
 
-class ParsedSource(object):
+class CTypeSpace(object):
     def __init__(self, source, parser, definitions=None, macros=None, headers=None):
-        from pypy.module.cpyext.api import configure_eci
         self.source = source
         self.definitions = definitions if definitions is not None else {}
         self.macros = macros if macros is not None else {}
@@ -823,7 +822,7 @@
 
 def parse_source(source, includes=None, headers=None, configure_now=True):
     ctx = Parser()
-    src = ParsedSource(source, ctx, headers=headers)
+    src = CTypeSpace(source, ctx, headers=headers)
     if includes is not None:
         for header in includes:
             src.include(header)
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -11,8 +11,8 @@
         double ob_fval;
     } TestFloatObject;
     """
-    res = parse_source(decl)
-    TestFloatObject = res.definitions['TestFloatObject']
+    cts = parse_source(decl)
+    TestFloatObject = cts.definitions['TestFloatObject']
     assert isinstance(TestFloatObject, lltype.Struct)
     assert TestFloatObject.c_ob_refcnt == rffi.SSIZE_T
     assert TestFloatObject.c_ob_pypy_link == rffi.SSIZE_T
@@ -20,8 +20,8 @@
 
 def test_simple():
     decl = "typedef ssize_t Py_ssize_t;"
-    hdr = parse_source(decl)
-    assert hdr.definitions == {'Py_ssize_t': rffi.SSIZE_T}
+    cts = parse_source(decl)
+    assert cts.definitions == {'Py_ssize_t': rffi.SSIZE_T}
 
 def test_macro():
     decl = """
@@ -36,9 +36,9 @@
         double ob_fval;
     } PyFloatObject;
     """
-    hdr = parse_source(decl)
-    assert 'PyFloatObject' in hdr.definitions
-    assert 'PyObject_HEAD' in hdr.macros
+    cts = parse_source(decl)
+    assert 'PyFloatObject' in cts.definitions
+    assert 'PyObject_HEAD' in cts.macros
 
 def test_include():
     cdef1 = """
@@ -59,12 +59,12 @@
         Type *type;
     } Object;
     """
-    hdr1 = parse_source(cdef1)
-    Type = hdr1.definitions['Type']
+    cts1 = parse_source(cdef1)
+    Type = cts1.definitions['Type']
     assert isinstance(Type, lltype.Struct)
-    hdr2 = parse_source(cdef2, includes=[hdr1])
-    assert 'Type' not in hdr2.definitions
-    Object = hdr2.definitions['Object']
+    cts2 = parse_source(cdef2, includes=[cts1])
+    assert 'Type' not in cts2.definitions
+    Object = cts2.definitions['Object']
     assert Object.c_type.TO is Type
 
 def test_incomplete():
@@ -83,8 +83,8 @@
     } Buffer;
 
     """
-    foo_h = parse_source(cdef)
-    Object = foo_h.gettype('Object')
+    cts = parse_source(cdef)
+    Object = cts.gettype('Object')
     assert isinstance(Object, lltype.Struct)
 
 def test_recursive():
@@ -106,8 +106,8 @@
         Object *obj;
     } Type;
     """
-    foo_h = parse_source(cdef)
-    Object = foo_h.definitions['Object']
+    cts = parse_source(cdef)
+    Object = cts.definitions['Object']
     assert isinstance(Object, lltype.Struct)
     hash(Object)
 
@@ -117,8 +117,8 @@
         const char * const foo;
     } bar;
     """
-    hdr = parse_source(cdef)
-    assert hdr.definitions['bar'].c_foo == rffi.CONST_CCHARP != rffi.CCHARP
+    cts = parse_source(cdef)
+    assert cts.definitions['bar'].c_foo == rffi.CONST_CCHARP != rffi.CCHARP
 
 def test_gettype():
     decl = """
@@ -133,9 +133,9 @@
         double ob_fval;
     } TestFloatObject;
     """
-    res = parse_source(decl)
-    assert res.gettype('Py_ssize_t') == rffi.SSIZE_T
-    assert res.gettype('TestFloatObject *').TO.c_ob_refcnt == rffi.SSIZE_T
+    cts = parse_source(decl)
+    assert cts.gettype('Py_ssize_t') == rffi.SSIZE_T
+    assert cts.gettype('TestFloatObject *').TO.c_ob_refcnt == rffi.SSIZE_T
 
 def test_parse_funcdecl():
     decl = """
@@ -152,8 +152,8 @@
 
     typedef TestFloatObject* (*func_t)(int, int);
     """
-    res = parse_source(decl)
-    name, FUNC = res.parse_func("func_t some_func(TestFloatObject*)")
+    cts = parse_source(decl)
+    name, FUNC = cts.parse_func("func_t some_func(TestFloatObject*)")
     assert name == 'some_func'
-    assert FUNC.RESULT == res.gettype('func_t')
-    assert FUNC.ARGS == (res.gettype('TestFloatObject *'),)
+    assert FUNC.RESULT == cts.gettype('func_t')
+    assert FUNC.ARGS == (cts.gettype('TestFloatObject *'),)

From pypy.commits at gmail.com  Mon Jan 16 10:32:49 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 07:32:49 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Move some logic from parse_source
 to CTypeSpace.__init__
Message-ID: <587ce7a1.c4811c0a.28521.b2fe@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89609:f2837cdef16e
Date: 2017-01-16 15:18 +0000
http://bitbucket.org/pypy/pypy/changeset/f2837cdef16e/

Log:	Move some logic from parse_source to CTypeSpace.__init__

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -685,17 +685,21 @@
 
 
 class CTypeSpace(object):
-    def __init__(self, source, parser, definitions=None, macros=None, headers=None):
+    def __init__(self, source, parser=None, definitions=None, macros=None,
+                 headers=None, includes=None):
         self.source = source
         self.definitions = definitions if definitions is not None else {}
         self.macros = macros if macros is not None else {}
         self.structs = {}
-        self.ctx = parser
+        self.ctx = parser if parser else Parser()
         self.headers = headers if headers is not None else ['sys/types.h']
         self._Config = type('Config', (object,), {})
         self._TYPES = {}
         self.includes = []
         self.struct_typedefs = {}
+        if includes is not None:
+            for header in includes:
+                self.include(header)
 
     def include(self, other):
         self.ctx.include(other.ctx)
@@ -821,11 +825,7 @@
 
 
 def parse_source(source, includes=None, headers=None, configure_now=True):
-    ctx = Parser()
-    src = CTypeSpace(source, ctx, headers=headers)
-    if includes is not None:
-        for header in includes:
-            src.include(header)
-    ctx.parse(source)
+    src = CTypeSpace(source, headers=headers, includes=includes)
+    src.ctx.parse(source)
     src.configure_types()
     return src

From pypy.commits at gmail.com  Mon Jan 16 11:46:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 16 Jan 2017 08:46:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix the __repr__ of bound methods to use
 the qualname of the function
Message-ID: <587cf8f3.0b561c0a.ea73d.e8c0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89610:babebbeb4488
Date: 2017-01-16 17:45 +0100
http://bitbucket.org/pypy/pypy/changeset/babebbeb4488/

Log:	Fix the __repr__ of bound methods to use the qualname of the
	function

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -535,11 +535,18 @@
 
     def descr_method_repr(self):
         space = self.space
-        name = self.w_function.getname(self.space)
-        w_class = space.type(self.w_instance)
-        typename = w_class.getname(self.space)
+        w_name = space.findattr(self.w_function, space.wrap('__qualname__'))
+        if w_name is None:
+            name = self.w_function.getname(self.space)
+        else:
+            try:
+                name = space.unicode_w(w_name)
+            except OperationError as e:
+                if not e.match(space, space.w_TypeError):
+                    raise
+                name = '?'
         objrepr = space.unicode_w(space.repr(self.w_instance))
-        s = u'<bound method %s.%s of %s>' % (typename, name, objrepr)
+        s = u'<bound method %s of %s>' % (name, objrepr)
         return space.wrap(s)
 
     def descr_method_getattribute(self, w_attr):
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -516,14 +516,19 @@
         class A(object):
             def f(self):
                 pass
-        assert repr(A().f).startswith("<bound method A.f of <")
+        assert repr(A().f).startswith("<bound method %s.f of <" %
+                                      A.__qualname__)
         assert repr(A().f).endswith(">>")
-        class B:
+
+    def test_method_repr_2(self):
+        class ClsA(object):
             def f(self):
                 pass
-        assert repr(B().f).startswith("<bound method B.f of <")
-        assert repr(A().f).endswith(">>")
-
+        class ClsB(ClsA):
+            pass
+        r = repr(ClsB().f)
+        assert "ClsA.f of <" in r
+        assert "ClsB object at " in r
 
     def test_method_call(self):
         class C(object):

From pypy.commits at gmail.com  Mon Jan 16 12:08:32 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 16 Jan 2017 09:08:32 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: typo (rlamy)
Message-ID: <587cfe10.05a81c0a.fba8d.0c98@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89611:e5365fbac53f
Date: 2017-01-16 19:07 +0200
http://bitbucket.org/pypy/pypy/changeset/e5365fbac53f/

Log:	typo (rlamy)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1126,7 +1126,7 @@
     setup_init_functions(eci, prefix)
     return modulename.new(ext='')
 
-def attach_recusively(space, static_pyobjs, static_objs_w, attached_objs, i):
+def attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i):
     # Start at i but make sure all the base classes are already attached
     from pypy.module.cpyext.pyobject import get_typedescr, make_ref
     if i in attached_objs:
@@ -1145,7 +1145,7 @@
             except ValueError:
                 j = -1
             if j >=0 and j not in attached_objs:
-                attach_recusively(space, static_pyobjs, static_objs_w,
+                attach_recursively(space, static_pyobjs, static_objs_w,
                                                  attached_objs, j)
     w_type = space.type(w_obj)
     typedescr = get_typedescr(w_type.layout.typedef)
@@ -1185,7 +1185,7 @@
         self.cpyext_type_init = []
         attached_objs = []
         for i in range(len(static_objs_w)):
-            attach_recusively(space, static_pyobjs, static_objs_w, attached_objs, i)
+            attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i)
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:

From pypy.commits at gmail.com  Mon Jan 16 12:33:56 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 09:33:56 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Allow parsing multiple headers
 into the same cts
Message-ID: <587d0404.9d711c0a.65f4a.0d79@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89612:ce1401635012
Date: 2017-01-16 17:04 +0000
http://bitbucket.org/pypy/pypy/changeset/ce1401635012/

Log:	Allow parsing multiple headers into the same cts

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -685,9 +685,9 @@
 
 
 class CTypeSpace(object):
-    def __init__(self, source, parser=None, definitions=None, macros=None,
+    def __init__(self, parser=None, definitions=None, macros=None,
                  headers=None, includes=None):
-        self.source = source
+        self.sources = []
         self.definitions = definitions if definitions is not None else {}
         self.macros = macros if macros is not None else {}
         self.structs = {}
@@ -697,6 +697,7 @@
         self._TYPES = {}
         self.includes = []
         self.struct_typedefs = {}
+        self._handled = set()
         if includes is not None:
             for header in includes:
                 self.include(header)
@@ -706,6 +707,11 @@
         self.structs.update(other.structs)
         self.includes.append(other)
 
+    def parse_source(self, source):
+        self.sources.append(source)
+        self.ctx.parse(source)
+        self.configure_types()
+
     def add_typedef(self, name, obj, quals):
         assert name not in self.definitions
         tp = self.convert_type(obj, quals)
@@ -746,7 +752,10 @@
         return struct.TYPE
 
     def build_eci(self):
-        all_sources = [x.source for x in self.includes] + [self.source]
+        all_sources = []
+        for cts in self.includes:
+            all_sources.extend(cts.sources)
+        all_sources.extend(self.sources)
         all_headers = self.headers
         for x in self.includes:
             for hdr in x.headers:
@@ -759,6 +768,9 @@
         for name, (obj, quals) in self.ctx._declarations.iteritems():
             if obj in self.ctx._included_declarations:
                 continue
+            if name in self._handled:
+                continue
+            self._handled.add(name)
             if name.startswith('typedef '):
                 name = name[8:]
                 self.add_typedef(name, obj, quals)
@@ -771,6 +783,7 @@
             del TYPE._hints['eci']
             if name in self._TYPES:
                 self._TYPES[name].become(TYPE)
+                del self._TYPES[name]
 
     def convert_type(self, obj, quals=0):
         if isinstance(obj, model.PrimitiveType):
@@ -825,7 +838,6 @@
 
 
 def parse_source(source, includes=None, headers=None, configure_now=True):
-    src = CTypeSpace(source, headers=headers, includes=includes)
-    src.ctx.parse(source)
-    src.configure_types()
-    return src
+    cts = CTypeSpace(headers=headers, includes=includes)
+    cts.parse_source(source)
+    return cts
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -1,5 +1,5 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.cparser import parse_source
+from pypy.module.cpyext.cparser import parse_source, CTypeSpace
 
 def test_configure():
     decl = """
@@ -67,6 +67,34 @@
     Object = cts2.definitions['Object']
     assert Object.c_type.TO is Type
 
+def test_multiple_sources():
+    cdef1 = """
+    typedef ssize_t Py_ssize_t;
+
+    #define PyObject_HEAD  \
+        Py_ssize_t ob_refcnt;        \
+        Py_ssize_t ob_pypy_link;     \
+
+    typedef struct {
+        char *name;
+    } Type;
+    """
+    cdef2 = """
+    typedef struct {
+        PyObject_HEAD
+        Py_ssize_t ob_foo;
+        Type *type;
+    } Object;
+    """
+    cts = CTypeSpace()
+    cts.parse_source(cdef1)
+    Type = cts.definitions['Type']
+    assert isinstance(Type, lltype.Struct)
+    assert 'Object' not in cts.definitions
+    cts.parse_source(cdef2)
+    Object = cts.definitions['Object']
+    assert Object.c_type.TO is Type
+
 def test_incomplete():
     cdef = """
     typedef ssize_t Py_ssize_t;

From pypy.commits at gmail.com  Mon Jan 16 12:33:58 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 09:33:58 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Use only one cts for all of cpyext
Message-ID: <587d0406.07941c0a.3db71.0e88@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89613:d218f1328c08
Date: 2017-01-16 17:28 +0000
http://bitbucket.org/pypy/pypy/changeset/d218f1328c08/

Log:	Use only one cts for all of cpyext

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,7 +40,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
-from pypy.module.cpyext.cparser import parse_source
+from pypy.module.cpyext.cparser import CTypeSpace
 
 DEBUG_WRAPPER = True
 
@@ -671,30 +671,30 @@
 build_exported_objects()
 
 object_cdef = (parse_dir / 'cpyext_object.h').read()
-object_h = parse_source(object_cdef,
-    headers=['sys/types.h', 'stdarg.h', 'stdio.h'])
+cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h'])
+cts.parse_source(object_cdef)
 
-Py_ssize_t = object_h.gettype('Py_ssize_t')
-Py_ssize_tP = object_h.gettype('Py_ssize_t *')
+Py_ssize_t = cts.gettype('Py_ssize_t')
+Py_ssize_tP = cts.gettype('Py_ssize_t *')
 size_t = rffi.ULONG
 ADDR = lltype.Signed
 
 # Note: as a special case, "PyObject" is the pointer type in RPython,
 # corresponding to "PyObject *" in C.  We do that only for PyObject.
 # For example, "PyTypeObject" is the struct type even in RPython.
-PyTypeObject = object_h.gettype('PyTypeObject')
-PyTypeObjectPtr = object_h.gettype('PyTypeObject *')
-PyObjectStruct = object_h.gettype('PyObject')
-PyObject = object_h.gettype('PyObject *')
+PyTypeObject = cts.gettype('PyTypeObject')
+PyTypeObjectPtr = cts.gettype('PyTypeObject *')
+PyObjectStruct = cts.gettype('PyObject')
+PyObject = cts.gettype('PyObject *')
 PyObjectFields = (("ob_refcnt", lltype.Signed),
                   ("ob_pypy_link", lltype.Signed),
                   ("ob_type", PyTypeObjectPtr))
 PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), )
-PyVarObjectStruct = object_h.gettype('PyVarObject')
-PyVarObject = object_h.gettype('PyVarObject *')
+PyVarObjectStruct = cts.gettype('PyVarObject')
+PyVarObject = cts.gettype('PyVarObject *')
 
-Py_buffer = object_h.gettype('Py_buffer')
-Py_bufferP = object_h.gettype('Py_buffer *')
+Py_buffer = cts.gettype('Py_buffer')
+Py_bufferP = cts.gettype('Py_buffer *')
 
 
 @specialize.memo()
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,13 +11,13 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr, slot_function, object_h, api_decl)
+    PyTypeObjectPtr, slot_function, cts, api_decl)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
-PyMethodDef = object_h.gettype('PyMethodDef')
-PyCFunction = object_h.gettype('PyCFunction')
-PyCFunctionKwArgs = object_h.gettype('PyCFunctionWithKeywords')
+PyMethodDef = cts.gettype('PyMethodDef')
+PyCFunction = cts.gettype('PyCFunction')
+PyCFunctionKwArgs = cts.gettype('PyCFunctionWithKeywords')
 PyCFunctionObjectStruct = cpython_struct(
     'PyCFunctionObject',
     PyObjectFields + (
@@ -282,7 +282,7 @@
 def PyCFunction_NewEx(space, ml, w_self, w_name):
     return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name))
 
- at api_decl("PyCFunction PyCFunction_GetFunction(PyObject *)", object_h)
+ at api_decl("PyCFunction PyCFunction_GetFunction(PyObject *)", cts)
 def PyCFunction_GetFunction(space, w_obj):
     try:
         cfunction = space.interp_w(W_PyCFunctionObject, w_obj)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -17,7 +17,7 @@
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
     PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
-    Py_TPFLAGS_HAVE_INPLACEOPS, object_h, parse_dir)
+    Py_TPFLAGS_HAVE_INPLACEOPS, cts, parse_dir)
 from pypy.module.cpyext.cparser import parse_source
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
@@ -42,9 +42,9 @@
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
 cdef = (parse_dir / 'cpyext_typeobject.h').read()
-typeobject_h = parse_source(cdef, includes=[object_h])
-PyHeapTypeObjectStruct = typeobject_h.gettype('PyHeapTypeObject')
-PyHeapTypeObject = typeobject_h.gettype('PyHeapTypeObject *')
+cts.parse_source(cdef)
+PyHeapTypeObjectStruct = cts.gettype('PyHeapTypeObject')
+PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
 
 class W_GetSetPropertyEx(GetSetProperty):
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -1,59 +1,59 @@
-from pypy.module.cpyext.api import object_h
+from pypy.module.cpyext.api import cts
 
 
-freefunc = object_h.definitions['freefunc']
-destructor = object_h.definitions['destructor']
-printfunc = object_h.definitions['printfunc']
-getattrfunc = object_h.definitions['getattrfunc']
-getattrofunc = object_h.definitions['getattrofunc']
-setattrfunc = object_h.definitions['setattrfunc']
-setattrofunc = object_h.definitions['setattrofunc']
-cmpfunc = object_h.definitions['cmpfunc']
-reprfunc = object_h.definitions['reprfunc']
-hashfunc = object_h.definitions['hashfunc']
-richcmpfunc = object_h.definitions['richcmpfunc']
-getiterfunc = object_h.definitions['getiterfunc']
-iternextfunc = object_h.definitions['iternextfunc']
-descrgetfunc = object_h.definitions['descrgetfunc']
-descrsetfunc = object_h.definitions['descrsetfunc']
-initproc = object_h.definitions['initproc']
-newfunc = object_h.definitions['newfunc']
-allocfunc = object_h.definitions['allocfunc']
+freefunc = cts.definitions['freefunc']
+destructor = cts.definitions['destructor']
+printfunc = cts.definitions['printfunc']
+getattrfunc = cts.definitions['getattrfunc']
+getattrofunc = cts.definitions['getattrofunc']
+setattrfunc = cts.definitions['setattrfunc']
+setattrofunc = cts.definitions['setattrofunc']
+cmpfunc = cts.definitions['cmpfunc']
+reprfunc = cts.definitions['reprfunc']
+hashfunc = cts.definitions['hashfunc']
+richcmpfunc = cts.definitions['richcmpfunc']
+getiterfunc = cts.definitions['getiterfunc']
+iternextfunc = cts.definitions['iternextfunc']
+descrgetfunc = cts.definitions['descrgetfunc']
+descrsetfunc = cts.definitions['descrsetfunc']
+initproc = cts.definitions['initproc']
+newfunc = cts.definitions['newfunc']
+allocfunc = cts.definitions['allocfunc']
 
-unaryfunc = object_h.definitions['unaryfunc']
-binaryfunc = object_h.definitions['binaryfunc']
-ternaryfunc = object_h.definitions['ternaryfunc']
-inquiry = object_h.definitions['inquiry']
-lenfunc = object_h.definitions['lenfunc']
-coercion = object_h.definitions['coercion']
-intargfunc = object_h.definitions['intargfunc']
-intintargfunc = object_h.definitions['intintargfunc']
-ssizeargfunc = object_h.definitions['ssizeargfunc']
-ssizessizeargfunc = object_h.definitions['ssizessizeargfunc']
-intobjargproc = object_h.definitions['intobjargproc']
-intintobjargproc = object_h.definitions['intintobjargproc']
-ssizeobjargproc = object_h.definitions['ssizeobjargproc']
-ssizessizeobjargproc = object_h.definitions['ssizessizeobjargproc']
-objobjargproc = object_h.definitions['objobjargproc']
+unaryfunc = cts.definitions['unaryfunc']
+binaryfunc = cts.definitions['binaryfunc']
+ternaryfunc = cts.definitions['ternaryfunc']
+inquiry = cts.definitions['inquiry']
+lenfunc = cts.definitions['lenfunc']
+coercion = cts.definitions['coercion']
+intargfunc = cts.definitions['intargfunc']
+intintargfunc = cts.definitions['intintargfunc']
+ssizeargfunc = cts.definitions['ssizeargfunc']
+ssizessizeargfunc = cts.definitions['ssizessizeargfunc']
+intobjargproc = cts.definitions['intobjargproc']
+intintobjargproc = cts.definitions['intintobjargproc']
+ssizeobjargproc = cts.definitions['ssizeobjargproc']
+ssizessizeobjargproc = cts.definitions['ssizessizeobjargproc']
+objobjargproc = cts.definitions['objobjargproc']
 
-objobjproc = object_h.definitions['objobjproc']
-visitproc = object_h.definitions['visitproc']
-traverseproc = object_h.definitions['traverseproc']
+objobjproc = cts.definitions['objobjproc']
+visitproc = cts.definitions['visitproc']
+traverseproc = cts.definitions['traverseproc']
 
-getter = object_h.definitions['getter']
-setter = object_h.definitions['setter']
+getter = cts.definitions['getter']
+setter = cts.definitions['setter']
 
-readbufferproc = object_h.definitions['readbufferproc']
-writebufferproc = object_h.definitions['writebufferproc']
-segcountproc = object_h.definitions['segcountproc']
-charbufferproc = object_h.definitions['charbufferproc']
-getbufferproc = object_h.definitions['getbufferproc']
-releasebufferproc = object_h.definitions['releasebufferproc']
+readbufferproc = cts.definitions['readbufferproc']
+writebufferproc = cts.definitions['writebufferproc']
+segcountproc = cts.definitions['segcountproc']
+charbufferproc = cts.definitions['charbufferproc']
+getbufferproc = cts.definitions['getbufferproc']
+releasebufferproc = cts.definitions['releasebufferproc']
 
 
-PyGetSetDef = object_h.definitions['PyGetSetDef']
-PyNumberMethods = object_h.definitions['PyNumberMethods']
-PySequenceMethods = object_h.definitions['PySequenceMethods']
-PyMappingMethods = object_h.definitions['PyMappingMethods']
-PyBufferProcs = object_h.definitions['PyBufferProcs']
-PyMemberDef = object_h.definitions['PyMemberDef']
+PyGetSetDef = cts.definitions['PyGetSetDef']
+PyNumberMethods = cts.definitions['PyNumberMethods']
+PySequenceMethods = cts.definitions['PySequenceMethods']
+PyMappingMethods = cts.definitions['PyMappingMethods']
+PyBufferProcs = cts.definitions['PyBufferProcs']
+PyMemberDef = cts.definitions['PyMemberDef']

From pypy.commits at gmail.com  Mon Jan 16 13:14:50 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 10:14:50 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Add CTypeSpace.parse_header()
Message-ID: <587d0d9a.54b31c0a.39a7b.0bc6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89614:47602e2d9f0e
Date: 2017-01-16 18:14 +0000
http://bitbucket.org/pypy/pypy/changeset/47602e2d9f0e/

Log:	Add CTypeSpace.parse_header()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -142,14 +142,14 @@
         target.chmod(0444) # make the file read-only, to make sure that nobody
                            # edits it by mistake
 
-def copy_header_files(dstdir, copy_numpy_headers):
+def copy_header_files(cts, dstdir, copy_numpy_headers):
     # XXX: 20 lines of code to recursively copy a directory, really??
     assert dstdir.check(dir=True)
     headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl')
     for name in ["pypy_macros.h"] + FUNCTIONS_BY_HEADER.keys():
         headers.append(udir.join(name))
-    headers.append(parse_dir / 'cpyext_object.h')
-    headers.append(parse_dir / 'cpyext_typeobject.h')
+    for path in cts.parsed_headers:
+        headers.append(path)
     _copy_header_files(headers, dstdir)
 
     if copy_numpy_headers:
@@ -670,9 +670,8 @@
                              % (cpyname, ))
 build_exported_objects()
 
-object_cdef = (parse_dir / 'cpyext_object.h').read()
 cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h'])
-cts.parse_source(object_cdef)
+cts.parse_header(parse_dir / 'cpyext_object.h')
 
 Py_ssize_t = cts.gettype('Py_ssize_t')
 Py_ssize_tP = cts.gettype('Py_ssize_t *')
@@ -1410,7 +1409,7 @@
 
     setup_init_functions(eci, prefix)
     trunk_include = pypydir.dirpath() / 'include'
-    copy_header_files(trunk_include, use_micronumpy)
+    copy_header_files(cts, trunk_include, use_micronumpy)
 
 
 def _load_from_cffi(space, name, path, initptr):
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -687,12 +687,13 @@
 class CTypeSpace(object):
     def __init__(self, parser=None, definitions=None, macros=None,
                  headers=None, includes=None):
-        self.sources = []
         self.definitions = definitions if definitions is not None else {}
         self.macros = macros if macros is not None else {}
         self.structs = {}
         self.ctx = parser if parser else Parser()
         self.headers = headers if headers is not None else ['sys/types.h']
+        self.parsed_headers = []
+        self.sources = []
         self._Config = type('Config', (object,), {})
         self._TYPES = {}
         self.includes = []
@@ -712,6 +713,12 @@
         self.ctx.parse(source)
         self.configure_types()
 
+    def parse_header(self, header_path):
+        self.headers.append(str(header_path))
+        self.parsed_headers.append(header_path)
+        self.ctx.parse(header_path.read())
+        self.configure_types()
+
     def add_typedef(self, name, obj, quals):
         assert name not in self.definitions
         tp = self.convert_type(obj, quals)
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -5,7 +5,7 @@
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext.api import (
     slot_function, cpython_api, copy_header_files, INTERPLEVEL_API,
-    Py_ssize_t, Py_ssize_tP, PyObject)
+    Py_ssize_t, Py_ssize_tP, PyObject, cts)
 from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest
 from pypy.interpreter.error import OperationError
 from rpython.rlib import rawrefcount
@@ -113,7 +113,7 @@
 @pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
-    copy_header_files(tmpdir, True)
+    copy_header_files(cts, tmpdir, True)
     def check(name):
         f = tmpdir.join(name)
         assert f.check(file=True)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -41,8 +41,7 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
-cdef = (parse_dir / 'cpyext_typeobject.h').read()
-cts.parse_source(cdef)
+cts.parse_header(parse_dir / 'cpyext_typeobject.h')
 PyHeapTypeObjectStruct = cts.gettype('PyHeapTypeObject')
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 

From pypy.commits at gmail.com  Mon Jan 16 13:19:27 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 16 Jan 2017 10:19:27 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: close branch to be merged
Message-ID: <587d0eaf.4e9d1c0a.5f1a1.fd49@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r89615:20df0e922a73
Date: 2017-01-16 19:13 +0200
http://bitbucket.org/pypy/pypy/changeset/20df0e922a73/

Log:	close branch to be merged


From pypy.commits at gmail.com  Mon Jan 16 13:19:30 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 16 Jan 2017 10:19:30 -0800 (PST)
Subject: [pypy-commit] pypy default: merge missing-tp_new which improves
 support for app-level classes in cpyext
Message-ID: <587d0eb2.810b1c0a.48125.37e6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r89616:5b9b0f8a10c5
Date: 2017-01-16 19:15 +0200
http://bitbucket.org/pypy/pypy/changeset/5b9b0f8a10c5/

Log:	merge missing-tp_new which improves support for app-level classes in
	cpyext

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -108,3 +108,14 @@
 
 .. branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419
 .. branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033
+
+.. branch: missing-tp_new
+
+Improve mixing app-level classes in c-extensions, especially if the app-level
+class has a ``tp_new`` or ``tp_dealloc``. The issue is that c-extensions expect
+all the method slots to be filled with a function pointer, where app-level will
+search up the mro for an appropriate function at runtime. With this branch we
+now fill many more slots in the c-extenion type objects.
+Also fix for c-extension type that calls ``tp_hash`` during initialization
+(str, unicode types), and fix instantiating c-extension types from built-in
+classes by enforcing an order of instaniation.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,6 +40,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 DEBUG_WRAPPER = True
 
@@ -900,6 +901,7 @@
         retval = fatal_value
         boxed_args = ()
         tb = None
+        state = space.fromcache(State)
         try:
             if not we_are_translated() and DEBUG_WRAPPER:
                 print >>sys.stderr, callable,
@@ -918,7 +920,6 @@
                 boxed_args += (arg_conv, )
             if pygilstate_ensure:
                 boxed_args += (args[-1], )
-            state = space.fromcache(State)
             try:
                 result = callable(space, *boxed_args)
                 if not we_are_translated() and DEBUG_WRAPPER:
@@ -964,6 +965,7 @@
         except Exception as e:
             unexpected_exception(pname, e, tb)
             _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
+            state.check_and_raise_exception(always=True)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
@@ -1124,6 +1126,34 @@
     setup_init_functions(eci, prefix)
     return modulename.new(ext='')
 
+def attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i):
+    # Start at i but make sure all the base classes are already attached
+    from pypy.module.cpyext.pyobject import get_typedescr, make_ref
+    if i in attached_objs:
+        return
+    py_obj = static_pyobjs[i]
+    w_obj = static_objs_w[i]
+    w_base = None
+    # w_obj can be NotImplemented, which is not a W_TypeObject
+    if isinstance(w_obj, W_TypeObject):
+        bases_w = w_obj.bases_w
+        if bases_w:
+            w_base = find_best_base(bases_w)
+        if w_base:
+            try:
+                j = static_objs_w.index(w_base)
+            except ValueError:
+                j = -1
+            if j >=0 and j not in attached_objs:
+                attach_recursively(space, static_pyobjs, static_objs_w,
+                                                 attached_objs, j)
+    w_type = space.type(w_obj)
+    typedescr = get_typedescr(w_type.layout.typedef)
+    py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
+                                 make_ref(space, w_type))
+    typedescr.attach(space, py_obj, w_obj)
+    attached_objs.append(i)
+
 
 class StaticObjectBuilder(object):
     def __init__(self):
@@ -1144,7 +1174,6 @@
 
     def attach_all(self, space):
         # this is RPython, called once in pypy-c when it imports cpyext
-        from pypy.module.cpyext.pyobject import get_typedescr, make_ref
         from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
         from pypy.module.cpyext.pyobject import track_reference
         #
@@ -1154,14 +1183,9 @@
             track_reference(space, static_pyobjs[i], static_objs_w[i])
         #
         self.cpyext_type_init = []
+        attached_objs = []
         for i in range(len(static_objs_w)):
-            py_obj = static_pyobjs[i]
-            w_obj = static_objs_w[i]
-            w_type = space.type(w_obj)
-            typedescr = get_typedescr(w_type.layout.typedef)
-            py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
-                                         make_ref(space, w_type))
-            typedescr.attach(space, py_obj, w_obj)
+            attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i)
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -80,14 +80,18 @@
     """
     py_str = rffi.cast(PyBytesObject, py_obj)
     s = space.str_w(w_obj)
-    if py_str.c_ob_size  < len(s):
+    len_s = len(s)
+    if py_str.c_ob_size  < len_s:
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
-            py_str.c_ob_size, len(s))
+            py_str.c_ob_size, len_s)
     with rffi.scoped_nonmovingbuffer(s) as s_ptr:
-        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
-    py_str.c_ob_sval[len(s)] = '\0'
-    py_str.c_ob_shash = space.hash_w(w_obj)
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len_s)
+    py_str.c_ob_sval[len_s] = '\0'
+    # if py_obj has a tp_hash, this will try to call it, but the objects are
+    # not fully linked yet
+    #py_str.c_ob_shash = space.hash_w(w_obj)
+    py_str.c_ob_shash = space.hash_w(space.newbytes(s))
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
 def bytes_realize(space, py_obj):
@@ -100,7 +104,9 @@
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(W_BytesObject, w_type)
     w_obj.__init__(s)
-    py_str.c_ob_shash = space.hash_w(w_obj)
+    # if py_obj has a tp_hash, this will try to call it but the object is
+    # not realized yet
+    py_str.c_ob_shash = space.hash_w(space.newbytes(s))
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
     return w_obj
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -18,6 +18,7 @@
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
+from pypy.module.cpyext import userslot
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
 from rpython.rlib.buffer import Buffer
@@ -512,6 +513,10 @@
 
 @specialize.memo()
 def get_slot_tp_function(space, typedef, name):
+    """Return a description of the slot C function to use for the built-in
+    type for 'typedef'.  The 'name' is the slot name.  This is a memo
+    function that, after translation, returns one of a built-in finite set.
+    """
     key = (typedef, name)
     try:
         return SLOTS[key]
@@ -551,6 +556,19 @@
                 return space.call_function(slot_fn, w_self)
             handled = True
 
+    for tp_name, attr in [('tp_hash', '__hash__'),
+                         ]:
+        if name == tp_name:
+            slot_fn = w_type.getdictvalue(space, attr)
+            if slot_fn is None:
+                return
+            @slot_function([PyObject], lltype.Signed, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_obj):
+                return space.int_w(space.call_function(slot_fn, w_obj))
+            handled = True
+
+
     # binary functions
     for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'),
                           ('tp_as_number.c_nb_subtract', '__sub__'),
@@ -755,7 +773,7 @@
         else:
             assert False
 
-    function = globals().get(FUNCTION, None)
+    function = getattr(userslot, FUNCTION or '!missing', None)
     assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS
     if FLAGS:
         if wrapper is Ellipsis:
@@ -818,7 +836,7 @@
 static slotdef slotdefs[] = {
         SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc,
                "x.__len__() <==> len(x)"),
-        SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc,
+        SQSLOT("__add__", sq_concat, slot_sq_concat, wrap_binaryfunc,
           "x.__add__(y) <==> x+y"),
         SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc,
           "x.__mul__(n) <==> x*n"),
@@ -966,9 +984,7 @@
                "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS),
         TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook,
                wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"),
-        TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""),
-        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""),
-        TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""),
+        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""),
         TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr,
                "x.__setattr__('name', value) <==> x.name = value"),
         TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""),
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -8,6 +8,24 @@
     return newType;
 }
 
+PyObject * typ = NULL;
+PyObject* datetimetype_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
+{
+    PyObject* newType;
+    if (typ == NULL)
+    {
+        PyErr_Format(PyExc_TypeError, "could not import datetime.datetime");
+        return NULL;
+    }
+    newType = ((PyTypeObject*)typ)->tp_new(metatype, args, kwds);
+    return newType;
+}
+
+void datetimetype_tp_dealloc(PyObject* self)
+{
+    return ((PyTypeObject*)typ)->tp_dealloc(self);
+}
+
 #define BASEFLAGS Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES
 
 PyTypeObject footype = {
@@ -58,6 +76,54 @@
     /*tp_weaklist*/         0
 };
 
+PyTypeObject datetimetype = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    /*tp_name*/             "foo3.datetimetype",
+    /*tp_basicsize*/        sizeof(PyTypeObject),
+    /*tp_itemsize*/         0,
+    /*tp_dealloc*/          datetimetype_tp_dealloc,
+    /*tp_print*/            0,
+    /*tp_getattr*/          0,
+    /*tp_setattr*/          0,
+    /*tp_compare*/          0,
+    /*tp_repr*/             0,
+    /*tp_as_number*/        0,
+    /*tp_as_sequence*/      0,
+    /*tp_as_mapping*/       0,
+    /*tp_hash*/             0,
+    /*tp_call*/             0,
+    /*tp_str*/              0,
+    /*tp_getattro*/         0,
+    /*tp_setattro*/         0,
+    /*tp_as_buffer*/        0,
+    /*tp_flags*/            BASEFLAGS,
+    /*tp_doc*/              0,
+    /*tp_traverse*/         0,
+    /*tp_clear*/            0,
+    /*tp_richcompare*/      0,
+    /*tp_weaklistoffset*/   0,
+    /*tp_iter*/             0,
+    /*tp_iternext*/         0,
+    /*tp_methods*/          0,
+    /*tp_members*/          0,
+    /*tp_getset*/           0,
+    /*tp_base*/             0,  //  set to &PyType_Type in module init function (why can it not be done here?)
+    /*tp_dict*/             0,
+    /*tp_descr_get*/        0,
+    /*tp_descr_set*/        0,
+    /*tp_dictoffset*/       0,
+    /*tp_init*/             0,
+    /*tp_alloc*/            0,
+    /*tp_new*/              datetimetype_tp_new,
+    /*tp_free*/             0,
+    /*tp_is_gc*/            0,
+    /*tp_bases*/            0,
+    /*tp_mro*/              0,
+    /*tp_cache*/            0,
+    /*tp_subclasses*/       0,
+    /*tp_weaklist*/         0
+};
+
 static PyMethodDef sbkMethods[] = {{NULL, NULL, 0, NULL}};
 
 /* Initialize this module. */
@@ -69,6 +135,16 @@
 initfoo3(void)
 {
     PyObject *mod, *d;
+    PyObject *module = NULL;
+    module = PyImport_ImportModule("datetime");
+    typ = PyObject_GetAttr(module, PyString_FromString("datetime"));
+    Py_DECREF(module);
+    if (!PyType_Check(typ)) {
+        PyErr_Format(PyExc_TypeError, "datetime.datetime is not a type object");
+        return;
+    }
+    datetimetype.tp_base = (PyTypeObject*)typ;
+    PyType_Ready(&datetimetype);
     footype.tp_base = &PyType_Type;
     PyType_Ready(&footype);
     mod = Py_InitModule("foo3", sbkMethods);
@@ -79,5 +155,7 @@
         return;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return;
+    if (PyDict_SetItemString(d, "datetimetype", (PyObject *)&datetimetype) < 0)
+        return;
     Py_INCREF(&footype);
 }
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -357,11 +357,15 @@
 
                 data = PyString_AS_STRING(args);
                 len = PyString_GET_SIZE(args);
-                if (data == NULL || len < 1)
+                if (data == NULL)
                     Py_RETURN_NONE;
                 obj = PyArray_Scalar(data, len);
                 return obj;
              """),
+            ("get_len", "METH_O",
+             """
+                return PyLong_FromLong(PyObject_Size(args));
+             """),
             ], prologue="""
                 #include <Python.h>
                 PyTypeObject PyStringArrType_Type = {
@@ -437,12 +441,16 @@
                 PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
                 PyStringArrType_Type.tp_itemsize = sizeof(char);
                 PyStringArrType_Type.tp_base = &PyString_Type;
+                PyStringArrType_Type.tp_hash = PyString_Type.tp_hash;
                 if (PyType_Ready(&PyStringArrType_Type) < 0) INITERROR;
             ''')
 
         a = module.newsubstr('abc')
         assert type(a).__name__ == 'string_'
         assert a == 'abc'
+        assert 3 == module.get_len(a)
+        b = module.newsubstr('')
+        assert 0 == module.get_len(b)
 
 class TestBytes(BaseApiTest):
     def test_bytes_resize(self, space):
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.api import generic_cpy_call
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 
@@ -663,30 +664,6 @@
         assert module.tp_init(list, x, ("hi",)) is None
         assert x == ["h", "i"]
 
-    def test_tp_str(self):
-        module = self.import_extension('foo', [
-           ("tp_str", "METH_VARARGS",
-            '''
-                 PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0);
-                 PyObject *obj = PyTuple_GET_ITEM(args, 1);
-                 if (!type->tp_str)
-                 {
-                     PyErr_SetNone(PyExc_ValueError);
-                     return NULL;
-                 }
-                 return type->tp_str(obj);
-             '''
-             )
-            ])
-        class C:
-            def __str__(self):
-                return "text"
-        assert module.tp_str(type(C()), C()) == "text"
-        class D(int):
-            def __str__(self):
-                return "more text"
-        assert module.tp_str(int, D(42)) == "42"
-
     def test_mp_ass_subscript(self):
         module = self.import_extension('foo', [
            ("new_obj", "METH_NOARGS",
@@ -978,9 +955,12 @@
         assert (d + a) == 5
         assert pow(d,b) == 16
 
-    def test_tp_new_in_subclass_of_type(self):
+    def test_tp_new_in_subclass(self):
+        import datetime
         module = self.import_module(name='foo3')
         module.footype("X", (object,), {})
+        a = module.datetimetype(1, 1, 1)
+        assert isinstance(a, module.datetimetype)
 
     def test_app_subclass_of_c_type(self):
         import sys
@@ -1165,7 +1145,6 @@
             self._check_type_object(FooType)
         class X(object):
             __metaclass__ = FooType
-        print repr(X)
         X()
 
     def test_multiple_inheritance3(self):
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -0,0 +1,208 @@
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.pyobject import make_ref, from_ref
+from pypy.module.cpyext.api import generic_cpy_call
+from pypy.module.cpyext.typeobject import PyTypeObjectPtr
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class TestAppLevelObject(BaseApiTest):
+    def test_nb_add_from_python(self, space, api):
+        w_date = space.appexec([], """():
+            class DateType(object):
+                def __add__(self, other):
+                    return 'sum!'
+            return DateType()
+            """)
+        w_datetype = space.type(w_date)
+        py_date = make_ref(space, w_date)
+        py_datetype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_datetype))
+        assert py_datetype.c_tp_as_number
+        assert py_datetype.c_tp_as_number.c_nb_add
+        w_obj = generic_cpy_call(space, py_datetype.c_tp_as_number.c_nb_add,
+                                 py_date, py_date)
+        assert space.str_w(w_obj) == 'sum!'
+
+    def test_tp_new_from_python(self, space, api):
+        w_date = space.appexec([], """():
+            class Date(object):
+                def __new__(cls, year, month, day):
+                    self = object.__new__(cls)
+                    self.year = year
+                    self.month = month
+                    self.day = day
+                    return self
+            return Date
+            """)
+        py_datetype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_date))
+        one = space.newint(1)
+        arg = space.newtuple([one, one, one])
+        # call w_date.__new__
+        w_obj = space.call_function(w_date, one, one, one)
+        w_year = space.getattr(w_obj, space.newbytes('year'))
+        assert space.int_w(w_year) == 1
+
+        w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype, 
+                                 arg, space.newdict({}))
+        w_year = space.getattr(w_obj, space.newbytes('year'))
+        assert space.int_w(w_year) == 1
+
+class AppTestUserSlots(AppTestCpythonExtensionBase):
+    def test_tp_hash_from_python(self):
+        # to see that the functions are being used,
+        # run pytest with -s
+        module = self.import_extension('foo', [
+           ("use_hash", "METH_O",
+            '''
+                long hash = args->ob_type->tp_hash(args);
+                return PyLong_FromLong(hash);
+            ''')])
+        class C(object):
+            def __hash__(self):
+                return -23
+        c = C()
+        # uses the userslot slot_tp_hash
+        ret = module.use_hash(C())
+        assert hash(c) == ret
+        # uses the slotdef renamed cpyext_tp_hash_int
+        ret = module.use_hash(3)
+        assert hash(3) == ret
+
+    def test_tp_str(self):
+        module = self.import_extension('foo', [
+           ("tp_str", "METH_VARARGS",
+            '''
+                 PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0);
+                 PyObject *obj = PyTuple_GET_ITEM(args, 1);
+                 if (!type->tp_str)
+                 {
+                     PyErr_SetString(PyExc_ValueError, "no tp_str");
+                     return NULL;
+                 }
+                 return type->tp_str(obj);
+             '''
+             )
+            ])
+        class C:
+            def __str__(self):
+                return "text"
+        assert module.tp_str(type(C()), C()) == "text"
+        class D(int):
+            def __str__(self):
+                return "more text"
+        assert module.tp_str(int, D(42)) == "42"
+        class A(object):
+            pass
+        s = module.tp_str(type(A()), A())
+        assert 'A object' in s
+
+    def test_tp_deallocate(self):
+        module = self.import_extension('foo', [
+            ("get_cnt", "METH_NOARGS",
+            '''
+                return PyLong_FromLong(foocnt);
+            '''),
+            ("get__timestamp", "METH_NOARGS",
+            '''
+                PyObject * one = PyLong_FromLong(1);
+                PyObject * a = PyTuple_Pack(3, one, one, one);
+                PyObject * k = NULL;
+                obj = _Timestamp.tp_new(&_Timestamp, a, k);
+                Py_DECREF(one);
+                return obj;
+             '''),
+            ("get_timestamp", "METH_NOARGS",
+            '''
+                PyObject * one = PyLong_FromLong(1);
+                PyObject * a = PyTuple_Pack(3, one, one, one);
+                PyObject * k = NULL;
+                obj = Timestamp.tp_new(&Timestamp, a, k);
+                Py_DECREF(one);
+                return obj;
+             '''),
+            ], prologue='''
+                static int foocnt = 0;
+                static PyTypeObject* datetime_cls = NULL;
+                static PyObject * obj = NULL;
+                static PyObject*
+                _timestamp_new(PyTypeObject* t, PyObject* a, PyObject* k)
+                {
+                    foocnt ++;
+                    return datetime_cls->tp_new(t, a, k);
+                }
+
+                static PyObject*
+                timestamp_new(PyTypeObject* t, PyObject* a, PyObject* k)
+                {
+                    return datetime_cls->tp_new(t, a, k);
+                }
+
+                static void 
+                _timestamp_dealloc(PyObject *op)
+                {
+                    foocnt --;
+                    datetime_cls->tp_dealloc(op);
+                }
+                 
+
+                static PyTypeObject _Timestamp = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,                            /* ob_size */
+                    "foo._Timestamp",   /* tp_name*/
+                    0,                  /* tp_basicsize*/
+                    0,                  /* tp_itemsize */
+                    _timestamp_dealloc  /* tp_dealloc  */
+                };
+                static PyTypeObject Timestamp = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,                            /* ob_size */
+                    "foo.Timestamp",   /* tp_name*/
+                    0,                  /* tp_basicsize*/
+                    0                  /* tp_itemsize */
+                };
+            ''', more_init='''
+                PyObject * mod = PyImport_ImportModule("datetime");
+                if (mod == NULL) INITERROR;
+                PyObject * dt = PyString_FromString("datetime");
+                datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod, dt); 
+                if (datetime_cls == NULL) INITERROR;
+                _Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                _Timestamp.tp_base = datetime_cls;
+                _Timestamp.tp_new = _timestamp_new;
+                Py_DECREF(mod);
+                Py_DECREF(dt);
+                if (PyType_Ready(&_Timestamp) < 0) INITERROR;
+
+                Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                Timestamp.tp_base = &_Timestamp;
+                Timestamp.tp_new = timestamp_new;
+                Timestamp.tp_dealloc = datetime_cls->tp_dealloc;
+                if (PyType_Ready(&Timestamp) < 0) INITERROR;
+            ''')
+        # _Timestamp has __new__, __del__ and 
+        #      inherits from datetime.datetime
+        # Timestamp has __new__, default __del__ (subtype_dealloc) and
+        #      inherits from _Timestamp
+        import gc, sys
+        cnt = module.get_cnt()
+        assert cnt == 0
+        obj = module.get__timestamp() #_Timestamp
+        cnt = module.get_cnt()
+        assert cnt == 1
+        assert obj.year == 1
+        del obj
+        self.debug_collect()
+        cnt = module.get_cnt()
+        assert cnt == 0
+
+        obj = module.get_timestamp() #Timestamp
+        cnt = module.get_cnt()
+        assert cnt == 0
+        assert obj.year == 1
+        # XXX calling Timestamp.tp_dealloc which is subtype_dealloc
+        #     causes infinite recursion
+        del obj
+        self.debug_collect()
+        cnt = module.get_cnt()
+        assert cnt == 0
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -1,7 +1,7 @@
 import os
 
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
 
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
@@ -36,7 +36,7 @@
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 
-WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
+#WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
@@ -258,6 +258,7 @@
             dict_w[name] = w_descr
             i += 1
 
+missing_slots={}
 def update_all_slots(space, w_type, pto):
     # fill slots in pto
     # Not very sure about it, but according to
@@ -265,22 +266,46 @@
     # overwrite slots that are already set: these ones are probably
     # coming from a parent C type.
 
-    typedef = w_type.layout.typedef
+    if w_type.is_heaptype():
+        typedef = None
+        search_dict_w = w_type.dict_w
+    else:
+        typedef = w_type.layout.typedef
+        search_dict_w = None
+
     for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots:
-        w_descr = w_type.lookup(method_name)
-        if w_descr is None:
-            # XXX special case iternext
-            continue
+        slot_func_helper = None
+        if search_dict_w is None:
+            # built-in types: expose as many slots as possible, even
+            # if it happens to come from some parent class
+            slot_apifunc = None # use get_slot_tp_function 
+        else:
+            # For heaptypes, w_type.layout.typedef will be object's typedef, and
+            # get_slot_tp_function will fail
+            w_descr = search_dict_w.get(method_name, None)
+            if w_descr: 
+                # use the slot_apifunc (userslots) to lookup at runtime
+                pass
+            elif len(slot_names) ==1:
+                # 'inherit' from tp_base
+                slot_func_helper = getattr(pto.c_tp_base, slot_names[0])
+            else:
+                struct = getattr(pto.c_tp_base, slot_names[0])
+                if struct:
+                    slot_func_helper = getattr(struct, slot_names[1])
 
-        if slot_apifunc is None and typedef is not None:
-            slot_apifunc = get_slot_tp_function(space, typedef, slot_name)
-        if not slot_apifunc:
-            if WARN_ABOUT_MISSING_SLOT_FUNCTIONS:
-                os.write(2,
-                    "%s defined by %s but no slot function defined!\n" % (
-                        method_name, w_type.getname(space)))
-            continue
-        slot_func_helper = slot_apifunc.get_llhelper(space)
+        if not slot_func_helper:
+            if typedef is not None:
+                if slot_apifunc is None:
+                    slot_apifunc = get_slot_tp_function(space, typedef, slot_name)
+            if not slot_apifunc:
+                if not we_are_translated():
+                    if slot_name not in missing_slots:
+                        missing_slots[slot_name] = w_type.getname(space)
+                        print "missing slot %r/%r, discovered on %r" % (
+                            method_name, slot_name, w_type.getname(space))
+                continue
+            slot_func_helper = slot_apifunc.get_llhelper(space)
 
         # XXX special case wrapper-functions and use a "specific" slot func
 
@@ -308,6 +333,8 @@
                     STRUCT_TYPE = PySequenceMethods
                 elif slot_names[0] == 'c_tp_as_buffer':
                     STRUCT_TYPE = PyBufferProcs
+                elif slot_names[0] == 'c_tp_as_mapping':
+                    STRUCT_TYPE = PyMappingMethods
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
@@ -405,13 +432,6 @@
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
-    flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
-    base_object_pyo = make_ref(space, space.w_object)
-    base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
-    if base_pto != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
-        if not pto.c_tp_new:
-            pto.c_tp_new = base_pto.c_tp_new
-    Py_DecRef(space, base_object_pyo)
 
 def check_descr(space, w_self, w_type):
     if not space.isinstance_w(w_self, w_type):
@@ -511,9 +531,25 @@
     pto = obj.c_ob_type
     base = pto
     this_func_ptr = llslot(space, subtype_dealloc)
+    w_obj = from_ref(space, rffi.cast(PyObject, base))
+    # This wrapper is created on a specific type, call it w_A. 
+    # We wish to call the dealloc function from one of the base classes of w_A,
+    # the first of which is not this function itself.
+    # w_obj is an instance of w_A or one of its subclasses. So climb up the
+    # inheritance chain until base.c_tp_dealloc is exactly this_func, and then
+    # continue on up until they differ.
+    #print 'subtype_dealloc, start from', rffi.charp2str(base.c_tp_name)
+    while base.c_tp_dealloc != this_func_ptr:
+        base = base.c_tp_base
+        assert base
+        #print '                 ne move to', rffi.charp2str(base.c_tp_name)
+        w_obj = from_ref(space, rffi.cast(PyObject, base))
     while base.c_tp_dealloc == this_func_ptr:
         base = base.c_tp_base
         assert base
+        #print '                 eq move to', rffi.charp2str(base.c_tp_name)
+        w_obj = from_ref(space, rffi.cast(PyObject, base))
+    #print '                   end with', rffi.charp2str(base.c_tp_name)
     dealloc = base.c_tp_dealloc
     # XXX call tp_del if necessary
     generic_cpy_call(space, dealloc, obj)
@@ -685,13 +721,6 @@
 
     typedescr = get_typedescr(w_type.layout.typedef)
 
-    # dealloc
-    if space.gettypeobject(w_type.layout.typedef) is w_type:
-        # only for the exact type, like 'space.w_tuple' or 'space.w_list'
-        pto.c_tp_dealloc = typedescr.get_dealloc().get_llhelper(space)
-    else:
-        # for all subtypes, use subtype_dealloc()
-        pto.c_tp_dealloc = llslot(space, subtype_dealloc)
     if space.is_w(w_type, space.w_str):
         pto.c_tp_itemsize = 1
     elif space.is_w(w_type, space.w_tuple):
@@ -722,6 +751,17 @@
     w_base = best_base(space, w_type.bases_w)
     pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base))
 
+    # dealloc
+    if space.gettypeobject(w_type.layout.typedef) is w_type:
+        # only for the exact type, like 'space.w_tuple' or 'space.w_list'
+        pto.c_tp_dealloc = typedescr.get_dealloc().get_llhelper(space)
+    else:
+        # for all subtypes, use base's dealloc (requires sorting in attach_all)
+        pto.c_tp_dealloc = pto.c_tp_base.c_tp_dealloc
+        if not pto.c_tp_dealloc:
+            # strange, but happens (ABCMeta)
+            pto.c_tp_dealloc = llslot(space, subtype_dealloc)
+
     if builder.cpyext_type_init is not None:
         builder.cpyext_type_init.append((pto, w_type))
     else:
@@ -735,11 +775,18 @@
         if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
             pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
-    # will be filled later on with the correct value
-    # may not be 0
     if space.is_w(w_type, space.w_object):
+        # will be filled later on with the correct value
+        # may not be 0
         pto.c_tp_new = rffi.cast(newfunc, 1)
     update_all_slots(space, w_type, pto)
+    if not pto.c_tp_new:
+        base_object_pyo = make_ref(space, space.w_object)
+        base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
+        flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
+        if pto.c_tp_base != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
+                pto.c_tp_new = pto.c_tp_base.c_tp_new
+        Py_DecRef(space, base_object_pyo)
     pto.c_tp_flags |= Py_TPFLAGS_READY
     return pto
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -65,9 +65,10 @@
 def unicode_attach(space, py_obj, w_obj, w_userdata=None):
     "Fills a newly allocated PyUnicodeObject with a unicode string"
     py_unicode = rffi.cast(PyUnicodeObject, py_obj)
-    py_unicode.c_length = len(space.unicode_w(w_obj))
+    s = space.unicode_w(w_obj)
+    py_unicode.c_length = len(s)
     py_unicode.c_str = lltype.nullptr(rffi.CWCHARP.TO)
-    py_unicode.c_hash = space.hash_w(w_obj)
+    py_unicode.c_hash = space.hash_w(space.newunicode(s))
     py_unicode.c_defenc = lltype.nullptr(PyObject.TO)
 
 def unicode_realize(space, py_obj):
@@ -80,7 +81,7 @@
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
     w_obj.__init__(s)
-    py_uni.c_hash = space.hash_w(w_obj)
+    py_uni.c_hash = space.hash_w(space.newunicode(s))
     track_reference(space, py_obj, w_obj)
     return w_obj
 
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/userslot.py
@@ -0,0 +1,112 @@
+"""
+These are the default implementation for type slots that we put
+in user-defined app-level Python classes, if the class implements
+the corresponding '__xxx__' special method.  It should mostly just
+call back the general version of the space operation.
+
+This is only approximately correct.  One problem is that some
+details are likely subtly wrong.  Another problem is that we don't
+track changes to an app-level Python class (addition or removal of
+'__xxx__' special methods) after initalization of the PyTypeObject.
+"""
+
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.argument import Arguments
+from pypy.module.cpyext.api import slot_function, PyObject, Py_ssize_t
+from pypy.module.cpyext.api import PyTypeObjectPtr
+from rpython.rtyper.lltypesystem import rffi, lltype
+
+ at slot_function([PyObject], Py_ssize_t, error=-1)
+def slot_sq_length(space, w_obj):
+    return space.int_w(space.len(w_obj))
+
+ at slot_function([PyObject], lltype.Signed, error=-1)
+def slot_tp_hash(space, w_obj):
+    return space.int_w(space.hash(w_obj))
+
+ at slot_function([PyObject, Py_ssize_t], PyObject)
+def slot_sq_item(space, w_obj, index):
+    return space.getitem(w_obj, space.wrap(index))
+
+ at slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
+def slot_tp_new(space, w_type, w_args, w_kwds):
+    # XXX problem - we need to find the actual __new__ function to call.
+    #     but we have no 'self' argument. Theoretically, self will be
+    #     w_type, but if w_type is a subclass of self, and w_type has a
+    #     __new__ function that calls super().__new__, and that call ends
+    #     up here, we will get infinite recursion. Prevent the recursion
+    #     in the simple case (from cython) where w_type is a cpytype, but
+    #     we know (since we are in this function) that self is not a cpytype
+    from pypy.module.cpyext.typeobject import W_PyCTypeObject
+    w_type0 = w_type
+    mro_w = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
+    for w_m in mro_w[1:]:
+        if not w_type0.is_cpytype():
+            break
+        w_type0 = w_m
+    w_impl = space.getattr(w_type0, space.wrap('__new__'))
+    args = Arguments(space, [w_type],
+                     w_stararg=w_args, w_starstararg=w_kwds)
+    return space.call_args(w_impl, args)
+
+# unary functions
+
+ at slot_function([PyObject], PyObject)
+def slot_tp_str(space, w_obj):
+    return space.str(w_obj)
+
+ at slot_function([PyObject], PyObject)
+def slot_tp_repr(space, w_obj):
+    return space.repr(w_obj)
+
+#binary functions
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_add(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_subtract(space, w_obj1, w_obj2):
+    return space.sub(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_multiply(space, w_obj1, w_obj2):
+    return space.mul(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_divide(space, w_obj1, w_obj2):
+    return space.div(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_add(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_subtract(space, w_obj1, w_obj2):
+    return space.sub(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_multiply(space, w_obj1, w_obj2):
+    return space.mul(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_divide(space, w_obj1, w_obj2):
+    return space.div(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_sq_concat(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_sq_inplace_concat(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_mp_subscript(space, w_obj1, w_obj2):
+    return space.getitem(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_tp_getattr(space, w_obj1, w_obj2):
+    return space.getattr(w_obj1, w_obj2)
+
+

From pypy.commits at gmail.com  Mon Jan 16 14:21:18 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 11:21:18 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Add RPython compatible cts.cast()
Message-ID: <587d1d2e.d5091c0a.81af2.611a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89617:ac221fcd46ab
Date: 2017-01-16 19:20 +0000
http://bitbucket.org/pypy/pypy/changeset/ac221fcd46ab/

Log:	Add RPython compatible cts.cast()

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -10,6 +10,9 @@
 from rpython.rlib.rfile import FILEP
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rfficache, rffi_platform
+from rpython.flowspace.model import Constant, const
+from rpython.flowspace.specialcase import register_flow_sc
+from rpython.flowspace.flowcontext import FlowingError
 
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
@@ -699,6 +702,7 @@
         self.includes = []
         self.struct_typedefs = {}
         self._handled = set()
+        self._frozen = False
         if includes is not None:
             for header in includes:
                 self.include(header)
@@ -740,8 +744,8 @@
         self.structs[obj] = struct
         if obj.fldtypes is not None:
             struct.fields = zip(
-                 obj.fldnames,
-                 [self.convert_field(field) for field in obj.fldtypes])
+                obj.fldnames,
+                [self.convert_field(field) for field in obj.fldtypes])
         return struct
 
     def convert_field(self, obj):
@@ -833,6 +837,9 @@
             result = result.TYPE
         return result
 
+    def cast(self, cdecl, value):
+        return rffi.cast(self.gettype(cdecl), value)
+
     def parse_func(self, cdecl):
         cdecl = cdecl.strip()
         if cdecl[-1] != ';':
@@ -843,6 +850,20 @@
         FUNCP = self.convert_type(tp.as_function_pointer())
         return decl.name, FUNCP.TO
 
+    def _freeze_(self):
+        if self._frozen:
+            return True
+
+        @register_flow_sc(self.cast)
+        def sc_cast(ctx, v_decl, v_arg):
+            if not isinstance(v_decl, Constant):
+                raise FlowingError(
+                    "The first argument of cts.cast() must be a constant.")
+            TP = self.gettype(v_decl.value)
+            return ctx.appcall(rffi.cast, const(TP), v_arg)
+        self._frozen = True
+        return True
+
 
 def parse_source(source, includes=None, headers=None, configure_now=True):
     cts = CTypeSpace(headers=headers, includes=includes)
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -1,3 +1,6 @@
+from rpython.flowspace.model import const
+from rpython.flowspace.objspace import build_flow
+from rpython.translator.simplify import simplify_graph
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.cparser import parse_source, CTypeSpace
 
@@ -164,6 +167,7 @@
     cts = parse_source(decl)
     assert cts.gettype('Py_ssize_t') == rffi.SSIZE_T
     assert cts.gettype('TestFloatObject *').TO.c_ob_refcnt == rffi.SSIZE_T
+    assert cts.cast('Py_ssize_t', 42) == rffi.cast(rffi.SSIZE_T, 42)
 
 def test_parse_funcdecl():
     decl = """
@@ -185,3 +189,16 @@
     assert name == 'some_func'
     assert FUNC.RESULT == cts.gettype('func_t')
     assert FUNC.ARGS == (cts.gettype('TestFloatObject *'),)
+
+def test_translate_cast():
+    cdef = "typedef ssize_t Py_ssize_t;"
+    cts = parse_source(cdef)
+
+    def f():
+        return cts.cast('Py_ssize_t*', 0)
+    graph = build_flow(f)
+    simplify_graph(graph)
+    assert len(graph.startblock.operations) == 1
+    op = graph.startblock.operations[0]
+    assert op.args[0] == const(rffi.cast)
+    assert op.args[1].value is cts.gettype('Py_ssize_t*')

From pypy.commits at gmail.com  Mon Jan 16 14:34:22 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 11:34:22 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Make cts.gettype() RPython
Message-ID: <587d203e.46831c0a.bb4ea.5eb0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89618:660f34981e06
Date: 2017-01-16 19:33 +0000
http://bitbucket.org/pypy/pypy/changeset/660f34981e06/

Log:	Make cts.gettype() RPython

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -861,6 +861,14 @@
                     "The first argument of cts.cast() must be a constant.")
             TP = self.gettype(v_decl.value)
             return ctx.appcall(rffi.cast, const(TP), v_arg)
+
+        @register_flow_sc(self.gettype)
+        def sc_gettype(ctx, v_decl):
+            if not isinstance(v_decl, Constant):
+                raise FlowingError(
+                    "The argument of cts.gettype() must be a constant.")
+            return const(self.gettype(v_decl.value))
+
         self._frozen = True
         return True
 
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -202,3 +202,17 @@
     op = graph.startblock.operations[0]
     assert op.args[0] == const(rffi.cast)
     assert op.args[1].value is cts.gettype('Py_ssize_t*')
+
+def test_translate_gettype():
+    cdef = "typedef ssize_t Py_ssize_t;"
+    cts = parse_source(cdef)
+
+    def f():
+        return cts.gettype('Py_ssize_t*')
+    graph = build_flow(f)
+    simplify_graph(graph)
+    # Check that the result is constant-folded
+    assert graph.startblock.operations == []
+    [link] = graph.startblock.exits
+    assert link.target is graph.returnblock
+    assert link.args[0] == const(rffi.CArrayPtr(rffi.SSIZE_T))

From pypy.commits at gmail.com  Mon Jan 16 14:38:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 16 Jan 2017 11:38:00 -0800 (PST)
Subject: [pypy-commit] pypy default: Last missing __objclass__
Message-ID: <587d2118.07941c0a.3db71.60d9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89619:4dc79c0c13a6
Date: 2017-01-16 20:37 +0100
http://bitbucket.org/pypy/pypy/changeset/4dc79c0c13a6/

Log:	Last missing __objclass__

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1333,3 +1333,4 @@
         assert int.__dict__['imag'].__objclass__ is int
         assert file.closed.__objclass__ is file
         assert type.__dict__['__name__'].__objclass__ is type
+        assert type.__dict__['__doc__'].__objclass__ is type
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -958,7 +958,7 @@
     __base__ = GetSetProperty(descr__base),
     __mro__ = GetSetProperty(descr_get__mro__),
     __dict__=GetSetProperty(type_get_dict),
-    __doc__ = GetSetProperty(descr__doc),
+    __doc__ = GetSetProperty(descr__doc, cls=W_TypeObject),
     mro = gateway.interp2app(descr_mro),
     __flags__ = GetSetProperty(descr__flags),
     __module__ = GetSetProperty(descr_get__module, descr_set__module),

From pypy.commits at gmail.com  Mon Jan 16 16:01:31 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 16 Jan 2017 13:01:31 -0800 (PST)
Subject: [pypy-commit] pypy default: fix failing test, which now passes
Message-ID: <587d34ab.54b31c0a.39a7b.79cf@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r89620:87475b57a469
Date: 2017-01-16 23:00 +0200
http://bitbucket.org/pypy/pypy/changeset/87475b57a469/

Log:	fix failing test, which now passes

diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py
--- a/lib-python/2.7/test/test_capi.py
+++ b/lib-python/2.7/test/test_capi.py
@@ -26,7 +26,6 @@
 skips = []
 if support.check_impl_detail(pypy=True):
     skips += [
-            'test_broken_memoryview',
             'test_buildvalue_N',
             'test_capsule',
             'test_lazy_hash_inheritance',
diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -2780,6 +2780,9 @@
     m = Py_InitModule("_testcapi", TestMethods);
     if (m == NULL)
         return;
+    
+    if (PyType_Ready(&_MemoryViewTester_Type) < 0)
+        return;
 
     Py_TYPE(&_HashInheritanceTester_Type)=&PyType_Type;
 

From pypy.commits at gmail.com  Mon Jan 16 16:21:07 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 16 Jan 2017 13:21:07 -0800 (PST)
Subject: [pypy-commit] pypy default: fix test - call Py_DecRef on PyObject
 to allow finalizer to run
Message-ID: <587d3943.8a281c0a.8919e.91b8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r89621:13bdc04dc6d2
Date: 2017-01-16 23:20 +0200
http://bitbucket.org/pypy/pypy/changeset/13bdc04dc6d2/

Log:	fix test - call Py_DecRef on PyObject to allow finalizer to run

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -30,13 +30,14 @@
         assert view.c_len == 5
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
-        w_mv = from_ref(space, api.PyMemoryView_FromBuffer(view))
+        ref = api.PyMemoryView_FromBuffer(view)
+        w_mv = from_ref(space, ref)
         for f in ('format', 'itemsize', 'ndim', 'readonly', 
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)
             assert space.eq_w(space.getattr(w_mv, w_f), 
                               space.getattr(w_memoryview, w_f))
-
+        api.Py_DecRef(ref)
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     def test_fillWithObject(self):

From pypy.commits at gmail.com  Mon Jan 16 16:35:39 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 16 Jan 2017 13:35:39 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update for recent release of numpy
 1.12
Message-ID: <587d3cab.8a281c0a.8919e.9a15@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r846:badcb41df7e1
Date: 2017-01-16 23:35 +0200
http://bitbucket.org/pypy/pypy.org/changeset/badcb41df7e1/

Log:	update for recent release of numpy 1.12

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -235,14 +235,14 @@
 the Python side and NumPy objects are mediated through the slower cpyext
 layer (which hurts a few benchmarks that do a lot of element-by-element
 array accesses, for example).</p>
-<p>Installation works on any recent PyPy (the <a class="reference internal" href="#release">release</a> above is fine),
-but you need the current developement version <em>of NumPy</em>.  The reason
-is that some PyPy-specific fixes have been merged back into NumPy,
-and they are not yet in a released version of NumPy.
+<p>Installation works on any recent PyPy (the <a class="reference internal" href="#release">release</a> above is fine, a recent
+nightly will implement more of the new buffer protocol).
+The currently released numpy 1.12 works except for <tt class="docutils literal">nditers</tt> with the
+<tt class="docutils literal">updateifcopy</tt> flag.
 For example, without using a virtualenv:</p>
 <pre class="literal-block">
 $ ./pypy-xxx/bin/pypy -m ensurepip
-$ ./pypy-xxx/bin/pip install cython git+https://github.com/numpy/numpy.git
+$ ./pypy-xxx/bin/pip install cython numpy
 </pre>
 <p>(See the general <a class="reference external" href="http://doc.pypy.org/en/latest/install.html">installation documentation</a> for more.)</p>
 </div>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -259,14 +259,14 @@
 layer (which hurts a few benchmarks that do a lot of element-by-element
 array accesses, for example).
 
-Installation works on any recent PyPy (the release_ above is fine),
-but you need the current developement version *of NumPy*.  The reason
-is that some PyPy-specific fixes have been merged back into NumPy,
-and they are not yet in a released version of NumPy.
+Installation works on any recent PyPy (the release_ above is fine, a recent
+nightly will implement more of the new buffer protocol).
+The currently released numpy 1.12 works except for ``nditers`` with the
+``updateifcopy`` flag.
 For example, without using a virtualenv::
 
     $ ./pypy-xxx/bin/pypy -m ensurepip
-    $ ./pypy-xxx/bin/pip install cython git+https://github.com/numpy/numpy.git
+    $ ./pypy-xxx/bin/pip install cython numpy
 
 (See the general `installation documentation`_ for more.)
 

From pypy.commits at gmail.com  Mon Jan 16 17:04:05 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 14:04:05 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Use cts.cast() in a few random
 places
Message-ID: <587d4355.9d711c0a.65f4a.ba12@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89622:b370fab11d05
Date: 2017-01-16 22:03 +0000
http://bitbucket.org/pypy/pypy/changeset/b370fab11d05/

Log:	Use cts.cast() in a few random places

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -32,7 +32,7 @@
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
 from pypy.module.cpyext.typeobjectdefs import (
-    PyGetSetDef, PyMemberDef, newfunc, getter, setter,
+    PyGetSetDef, PyMemberDef,
     PyNumberMethods, PySequenceMethods, PyBufferProcs)
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
@@ -42,7 +42,6 @@
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
 cts.parse_header(parse_dir / 'cpyext_typeobject.h')
-PyHeapTypeObjectStruct = cts.gettype('PyHeapTypeObject')
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
 
@@ -75,9 +74,9 @@
         py_getsetdef.c_doc = rffi.cast(rffi.CCHARP, 0)
     py_getsetdef.c_name = rffi.str2charp(getsetprop.getname(space))
     # XXX FIXME - actually assign these !!!
-    py_getsetdef.c_get = rffi.cast(getter, 0)
-    py_getsetdef.c_set = rffi.cast(setter, 0)
-    py_getsetdef.c_closure = rffi.cast(rffi.VOIDP, 0)
+    py_getsetdef.c_get = cts.cast('getter', 0)
+    py_getsetdef.c_set = cts.cast('setter', 0)
+    py_getsetdef.c_closure = cts.cast('void*', 0)
     return py_getsetdef
 
 
@@ -178,7 +177,7 @@
 
 def memberdescr_realize(space, obj):
     # XXX NOT TESTED When is this ever called?
-    member = rffi.cast(lltype.Ptr(PyMemberDef), obj)
+    member = cts.cast('PyMemberDef*', obj)
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
     w_obj = space.allocate_instance(W_MemberDescr, w_type)
     w_obj.__init__(member, w_type)
@@ -701,7 +700,7 @@
             # things are not initialized yet.  So in this case, simply use
             # str2charp() and "leak" the string.
         w_typename = space.getattr(w_type, space.wrap('__name__'))
-        heaptype = rffi.cast(PyHeapTypeObject, pto)
+        heaptype = cts.cast('PyHeapTypeObject*', pto)
         heaptype.c_ht_name = make_ref(space, w_typename)
         from pypy.module.cpyext.bytesobject import PyString_AsString
         pto.c_tp_name = PyString_AsString(space, heaptype.c_ht_name)
@@ -730,7 +729,7 @@
     # will be filled later on with the correct value
     # may not be 0
     if space.is_w(w_type, space.w_object):
-        pto.c_tp_new = rffi.cast(newfunc, 1)
+        pto.c_tp_new = cts.cast('newfunc', 1)
     update_all_slots(space, w_type, pto)
     pto.c_tp_flags |= Py_TPFLAGS_READY
     return pto
@@ -847,7 +846,8 @@
         if not py_type.c_tp_as_sequence:
             py_type.c_tp_as_sequence = base.c_tp_as_sequence
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
-        if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
+        if not py_type.c_tp_as_mapping:
+            py_type.c_tp_as_mapping = base.c_tp_as_mapping
         #if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
     return w_obj

From pypy.commits at gmail.com  Mon Jan 16 20:12:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 17:12:25 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: hg merge default
Message-ID: <587d6f79.c3e31c0a.ebbbe.0d10@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89623:ebbd6a0d8773
Date: 2017-01-17 01:11 +0000
http://bitbucket.org/pypy/pypy/changeset/ebbd6a0d8773/

Log:	hg merge default

diff --git a/lib-python/2.7/sqlite3/test/regression.py b/lib-python/2.7/sqlite3/test/regression.py
--- a/lib-python/2.7/sqlite3/test/regression.py
+++ b/lib-python/2.7/sqlite3/test/regression.py
@@ -351,10 +351,7 @@
         self.assertRaises(ValueError, cur.execute, " \0select 2")
         self.assertRaises(ValueError, cur.execute, "select 2\0")
 
-    @test_support.impl_detail(pypy=False)
     def CheckCommitCursorReset(self):
-        # This test is for logic added in 2.7.13 which PyPy doesn't
-        # implement.  See http://bugs.python.org/issue29006
         """
         Connection.commit() did reset cursors, which made sqlite3
         to return rows multiple times when fetched from cursors
diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py
--- a/lib-python/2.7/test/test_capi.py
+++ b/lib-python/2.7/test/test_capi.py
@@ -26,7 +26,6 @@
 skips = []
 if support.check_impl_detail(pypy=True):
     skips += [
-            'test_broken_memoryview',
             'test_buildvalue_N',
             'test_capsule',
             'test_lazy_hash_inheritance',
diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py
--- a/lib-python/2.7/test/test_multiprocessing.py
+++ b/lib-python/2.7/test/test_multiprocessing.py
@@ -1969,9 +1969,10 @@
         if not gc.isenabled():
             gc.enable()
             self.addCleanup(gc.disable)
-        #thresholds = gc.get_threshold()
-        #self.addCleanup(gc.set_threshold, *thresholds)
-        #gc.set_threshold(10)
+        if test_support.check_impl_detail(cpython=True):
+            thresholds = gc.get_threshold()
+            self.addCleanup(gc.set_threshold, *thresholds)
+            gc.set_threshold(10)
 
         # perform numerous block allocations, with cyclic references to make
         # sure objects are collected asynchronously by the gc
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -220,6 +220,7 @@
         self.__statements_counter = 0
         self.__rawstatements = set()
         self._statement_cache = _StatementCache(self, cached_statements)
+        self.__statements_already_committed = []
 
         self.__func_cache = {}
         self.__aggregates = {}
@@ -363,6 +364,14 @@
                 if cursor is not None:
                     cursor._reset = True
 
+    def _reset_already_committed_statements(self):
+        lst = self.__statements_already_committed
+        self.__statements_already_committed = []
+        for weakref in lst:
+            statement = weakref()
+            if statement is not None:
+                statement._reset()
+
     @_check_thread_wrap
     @_check_closed_wrap
     def __call__(self, sql):
@@ -418,15 +427,18 @@
         if not self._in_transaction:
             return
 
-        # The following line is a KNOWN DIFFERENCE with CPython 2.7.13.
-        # More precisely, the corresponding line was removed in the
-        # version 2.7.13 of CPython, but this is causing troubles for
-        # PyPy (and potentially for CPython too):
-        #
-        #     http://bugs.python.org/issue29006
-        #
-        # So for now, we keep this line.
-        self.__do_all_statements(Statement._reset, False)
+        # PyPy fix for non-refcounting semantics: since 2.7.13 (and in
+        # <= 2.6.x), the statements are not automatically reset upon
+        # commit.  However, if this is followed by some specific SQL
+        # operations like "drop table", these open statements come in
+        # the way and cause the "drop table" to fail.  On CPython the
+        # problem is much less important because typically all the old
+        # statements are freed already by reference counting.  So here,
+        # we copy all the still-alive statements to another list which
+        # is usually ignored, except if we get SQLITE_LOCKED
+        # afterwards---at which point we reset all statements in this
+        # list.
+        self.__statements_already_committed = self.__statements[:]
 
         statement_star = _ffi.new('sqlite3_stmt **')
         ret = _lib.sqlite3_prepare_v2(self._db, b"COMMIT", -1,
@@ -827,8 +839,18 @@
                 self.__statement._set_params(params)
 
                 # Actually execute the SQL statement
+
                 ret = _lib.sqlite3_step(self.__statement._statement)
 
+                # PyPy: if we get SQLITE_LOCKED, it's probably because
+                # one of the cursors created previously is still alive
+                # and not reset and the operation we're trying to do
+                # makes Sqlite unhappy about that.  In that case, we
+                # automatically reset all old cursors and try again.
+                if ret == _lib.SQLITE_LOCKED:
+                    self.__connection._reset_already_committed_statements()
+                    ret = _lib.sqlite3_step(self.__statement._statement)
+
                 if ret == _lib.SQLITE_ROW:
                     if multiple:
                         raise ProgrammingError("executemany() can only execute DML statements.")
diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -2780,6 +2780,9 @@
     m = Py_InitModule("_testcapi", TestMethods);
     if (m == NULL)
         return;
+    
+    if (PyType_Ready(&_MemoryViewTester_Type) < 0)
+        return;
 
     Py_TYPE(&_HashInheritanceTester_Type)=&PyType_Type;
 
diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -374,7 +374,7 @@
 
     sample_count = _sample_count(cp, size)
 
-    rv = ffi.new("unsigned char[]", len(cp) * 2)
+    rv = ffi.new("char[]", len(cp) * 2)
     lib.tostereo(rv, cp, len(cp), size, fac1, fac2)
     return ffi.buffer(rv)[:]
 
@@ -385,7 +385,7 @@
     if len(cp1) != len(cp2):
         raise error("Lengths should be the same")
 
-    rv = ffi.new("unsigned char[]", len(cp1))
+    rv = ffi.new("char[]", len(cp1))
     lib.add(rv, cp1, cp2, len(cp1), size)
     return ffi.buffer(rv)[:]
 
@@ -488,7 +488,7 @@
     ceiling = (q + 1) * outrate
     nbytes = ceiling * bytes_per_frame
 
-    rv = ffi.new("unsigned char[]", nbytes)
+    rv = ffi.new("char[]", nbytes)
     trim_index = lib.ratecv(rv, cp, frame_count, size,
                             nchannels, inrate, outrate,
                             state_d, prev_i, cur_i,
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -483,11 +483,6 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* The ``sqlite`` module was updated on 2.7.13 to no longer reset all
-  cursors when there is a commit.  This causes troubles for PyPy (and
-  potentially for CPython too), and so for now we didn't port this change:
-  see http://bugs.python.org/issue29006.
-
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -26,16 +26,16 @@
 
 Almost!
 
-The mostly likely stumbling block for any given project is support for
+The most likely stumbling block for any given project is support for
 :ref:`extension modules <extension-modules>`.  PyPy supports a continually growing
 number of extension modules, but so far mostly only those found in the
 standard library.
 
 The language features (including builtin types and functions) are very
-complete and well tested, so if your project does not use many
+refined and well tested, so if your project doesn't use many
 extension modules there is a good chance that it will work with PyPy.
 
-We list the differences we know about in :doc:`cpython differences <cpython_differences>`.
+We list the known differences in :doc:`cpython differences <cpython_differences>`.
 
 
 Module xyz does not work with PyPy: ImportError
@@ -76,10 +76,10 @@
 
 You cannot import *any* extension module in a `sandboxed PyPy`_,
 sorry.  Even the built-in modules available are very limited.
-Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
-it is only a proof of concept.  It seriously requires someone working
-on it.  Before this occurs, it can only be used it for "pure Python"
-examples: programs that import mostly nothing (or only pure Python
+Sandboxing in PyPy is a good proof of concept, and is without a doubt
+safe IMHO, however it is only a proof of concept.  It currently requires 
+some work from a motivated developer. However, until then it can only be used for "pure Python"
+example: programs that import mostly nothing (or only pure Python
 modules, recursively).
 
 .. _`sandboxed PyPy`: sandbox.html
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -336,6 +336,6 @@
    that fixes some bugs and is translatable.
 
 *  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the files ``xxxtype.py`` and
-   ``xxxobject.py`` contain respectively the definition of the type and its
-   (default) implementation.
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -101,3 +101,21 @@
 
 Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to
 the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak.
+
+.. branch: issue2464
+
+Give (almost?) all GetSetProperties a valid __objclass__.
+
+.. branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419
+.. branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033
+
+.. branch: missing-tp_new
+
+Improve mixing app-level classes in c-extensions, especially if the app-level
+class has a ``tp_new`` or ``tp_dealloc``. The issue is that c-extensions expect
+all the method slots to be filled with a function pointer, where app-level will
+search up the mro for an appropriate function at runtime. With this branch we
+now fill many more slots in the c-extenion type objects.
+Also fix for c-extension type that calls ``tp_hash`` during initialization
+(str, unicode types), and fix instantiating c-extension types from built-in
+classes by enforcing an order of instaniation.
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -140,7 +140,11 @@
             pass
         def fget(self, space, w_self):
             assert self is prop
-        prop = typedef.GetSetProperty(fget, use_closure=True)
+        # NB. this GetSetProperty is not copied when creating the
+        # W_TypeObject because of 'cls'.  Without it, a duplicate of the
+        # GetSetProperty is taken and it is given the w_objclass that is
+        # the W_TypeObject
+        prop = typedef.GetSetProperty(fget, use_closure=True, cls=W_SomeType)
         W_SomeType.typedef = typedef.TypeDef(
             'some_type',
             x=prop)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -276,12 +276,15 @@
         self.use_closure = use_closure
 
     def copy_for_type(self, w_objclass):
-        new = instantiate(GetSetProperty)
-        new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
-                  None, self.use_closure)
-        new.name = self.name
-        new.w_objclass = w_objclass
-        return new
+        if self.objclass_getter is None:
+            new = instantiate(GetSetProperty)
+            new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
+                      None, self.use_closure)
+            new.name = self.name
+            new.w_objclass = w_objclass
+            return new
+        else:
+            return self
 
     @unwrap_spec(w_cls = WrappedDefault(None))
     def descr_property_get(self, space, w_obj, w_cls=None):
@@ -483,7 +486,7 @@
     return lifeline.get_any_weakref(space)
 
 dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
-                            doc="dictionary for instance variables")
+                            doc="dictionary for instance variables (if defined)")
 dict_descr.name = '__dict__'
 
 
@@ -515,7 +518,7 @@
     return space.newtuple([w_docstring])
 
 weakref_descr = GetSetProperty(descr_get_weakref,
-                               doc="list of weak references to the object")
+                    doc="list of weak references to the object (if defined)")
 weakref_descr.name = '__weakref__'
 
 def make_weakref_descr(cls):
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -194,6 +194,17 @@
         from ..interp_ssl import SOCKET_STORAGE
         SOCKET_STORAGE._dict.clear()
 
+    def test_warmup_connection(self):
+        # not sure it is gmail.com's fault, but on some machines the
+        # very first connection attempt fails.  So we make one here and
+        # ignore the result.  The first real test is test_connect().
+        import socket, ssl
+        try:
+            ss = socket.ssl(self.s)
+            self.s.close()
+        except ssl.SSLError:
+            pass
+
     def test_connect(self):
         import socket, gc
         ss = socket.ssl(self.s)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,6 +40,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.module.cpyext.cparser import CTypeSpace
 
 DEBUG_WRAPPER = True
@@ -906,6 +907,7 @@
         retval = fatal_value
         boxed_args = ()
         tb = None
+        state = space.fromcache(State)
         try:
             if not we_are_translated() and DEBUG_WRAPPER:
                 print >>sys.stderr, callable,
@@ -924,7 +926,6 @@
                 boxed_args += (arg_conv, )
             if pygilstate_ensure:
                 boxed_args += (args[-1], )
-            state = space.fromcache(State)
             try:
                 result = callable(space, *boxed_args)
                 if not we_are_translated() and DEBUG_WRAPPER:
@@ -970,6 +971,7 @@
         except Exception as e:
             unexpected_exception(pname, e, tb)
             _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
+            state.check_and_raise_exception(always=True)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
@@ -1130,6 +1132,34 @@
     setup_init_functions(eci, prefix)
     return modulename.new(ext='')
 
+def attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i):
+    # Start at i but make sure all the base classes are already attached
+    from pypy.module.cpyext.pyobject import get_typedescr, make_ref
+    if i in attached_objs:
+        return
+    py_obj = static_pyobjs[i]
+    w_obj = static_objs_w[i]
+    w_base = None
+    # w_obj can be NotImplemented, which is not a W_TypeObject
+    if isinstance(w_obj, W_TypeObject):
+        bases_w = w_obj.bases_w
+        if bases_w:
+            w_base = find_best_base(bases_w)
+        if w_base:
+            try:
+                j = static_objs_w.index(w_base)
+            except ValueError:
+                j = -1
+            if j >=0 and j not in attached_objs:
+                attach_recursively(space, static_pyobjs, static_objs_w,
+                                                 attached_objs, j)
+    w_type = space.type(w_obj)
+    typedescr = get_typedescr(w_type.layout.typedef)
+    py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
+                                 make_ref(space, w_type))
+    typedescr.attach(space, py_obj, w_obj)
+    attached_objs.append(i)
+
 
 class StaticObjectBuilder(object):
     def __init__(self):
@@ -1150,7 +1180,6 @@
 
     def attach_all(self, space):
         # this is RPython, called once in pypy-c when it imports cpyext
-        from pypy.module.cpyext.pyobject import get_typedescr, make_ref
         from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
         from pypy.module.cpyext.pyobject import track_reference
         #
@@ -1160,14 +1189,9 @@
             track_reference(space, static_pyobjs[i], static_objs_w[i])
         #
         self.cpyext_type_init = []
+        attached_objs = []
         for i in range(len(static_objs_w)):
-            py_obj = static_pyobjs[i]
-            w_obj = static_objs_w[i]
-            w_type = space.type(w_obj)
-            typedescr = get_typedescr(w_type.layout.typedef)
-            py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
-                                         make_ref(space, w_type))
-            typedescr.attach(space, py_obj, w_obj)
+            attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i)
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -80,14 +80,18 @@
     """
     py_str = rffi.cast(PyBytesObject, py_obj)
     s = space.str_w(w_obj)
-    if py_str.c_ob_size  < len(s):
+    len_s = len(s)
+    if py_str.c_ob_size  < len_s:
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
-            py_str.c_ob_size, len(s))
+            py_str.c_ob_size, len_s)
     with rffi.scoped_nonmovingbuffer(s) as s_ptr:
-        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
-    py_str.c_ob_sval[len(s)] = '\0'
-    py_str.c_ob_shash = space.hash_w(w_obj)
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len_s)
+    py_str.c_ob_sval[len_s] = '\0'
+    # if py_obj has a tp_hash, this will try to call it, but the objects are
+    # not fully linked yet
+    #py_str.c_ob_shash = space.hash_w(w_obj)
+    py_str.c_ob_shash = space.hash_w(space.newbytes(s))
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
 def bytes_realize(space, py_obj):
@@ -100,7 +104,9 @@
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(W_BytesObject, w_type)
     w_obj.__init__(s)
-    py_str.c_ob_shash = space.hash_w(w_obj)
+    # if py_obj has a tp_hash, this will try to call it but the object is
+    # not realized yet
+    py_str.c_ob_shash = space.hash_w(space.newbytes(s))
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
     return w_obj
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -90,11 +90,10 @@
     return 0
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
-def PyList_GET_SIZE(space, w_list):
+def PyList_GET_SIZE(space, w_obj):
     """Macro form of PyList_Size() without error checking.
     """
-    assert isinstance(w_list, W_ListObject)
-    return w_list.length()
+    return space.len_w(w_obj)
 
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -18,6 +18,7 @@
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
+from pypy.module.cpyext import userslot
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
 from rpython.rlib.buffer import Buffer
@@ -512,6 +513,10 @@
 
 @specialize.memo()
 def get_slot_tp_function(space, typedef, name):
+    """Return a description of the slot C function to use for the built-in
+    type for 'typedef'.  The 'name' is the slot name.  This is a memo
+    function that, after translation, returns one of a built-in finite set.
+    """
     key = (typedef, name)
     try:
         return SLOTS[key]
@@ -551,6 +556,19 @@
                 return space.call_function(slot_fn, w_self)
             handled = True
 
+    for tp_name, attr in [('tp_hash', '__hash__'),
+                         ]:
+        if name == tp_name:
+            slot_fn = w_type.getdictvalue(space, attr)
+            if slot_fn is None:
+                return
+            @slot_function([PyObject], lltype.Signed, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_obj):
+                return space.int_w(space.call_function(slot_fn, w_obj))
+            handled = True
+
+
     # binary functions
     for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'),
                           ('tp_as_number.c_nb_subtract', '__sub__'),
@@ -755,7 +773,7 @@
         else:
             assert False
 
-    function = globals().get(FUNCTION, None)
+    function = getattr(userslot, FUNCTION or '!missing', None)
     assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS
     if FLAGS:
         if wrapper is Ellipsis:
@@ -818,7 +836,7 @@
 static slotdef slotdefs[] = {
         SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc,
                "x.__len__() <==> len(x)"),
-        SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc,
+        SQSLOT("__add__", sq_concat, slot_sq_concat, wrap_binaryfunc,
           "x.__add__(y) <==> x+y"),
         SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc,
           "x.__mul__(n) <==> x*n"),
@@ -966,9 +984,7 @@
                "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS),
         TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook,
                wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"),
-        TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""),
-        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""),
-        TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""),
+        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""),
         TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr,
                "x.__setattr__('name', value) <==> x.name = value"),
         TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""),
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -8,6 +8,24 @@
     return newType;
 }
 
+PyObject * typ = NULL;
+PyObject* datetimetype_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
+{
+    PyObject* newType;
+    if (typ == NULL)
+    {
+        PyErr_Format(PyExc_TypeError, "could not import datetime.datetime");
+        return NULL;
+    }
+    newType = ((PyTypeObject*)typ)->tp_new(metatype, args, kwds);
+    return newType;
+}
+
+void datetimetype_tp_dealloc(PyObject* self)
+{
+    return ((PyTypeObject*)typ)->tp_dealloc(self);
+}
+
 #define BASEFLAGS Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES
 
 PyTypeObject footype = {
@@ -58,6 +76,54 @@
     /*tp_weaklist*/         0
 };
 
+PyTypeObject datetimetype = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    /*tp_name*/             "foo3.datetimetype",
+    /*tp_basicsize*/        sizeof(PyTypeObject),
+    /*tp_itemsize*/         0,
+    /*tp_dealloc*/          datetimetype_tp_dealloc,
+    /*tp_print*/            0,
+    /*tp_getattr*/          0,
+    /*tp_setattr*/          0,
+    /*tp_compare*/          0,
+    /*tp_repr*/             0,
+    /*tp_as_number*/        0,
+    /*tp_as_sequence*/      0,
+    /*tp_as_mapping*/       0,
+    /*tp_hash*/             0,
+    /*tp_call*/             0,
+    /*tp_str*/              0,
+    /*tp_getattro*/         0,
+    /*tp_setattro*/         0,
+    /*tp_as_buffer*/        0,
+    /*tp_flags*/            BASEFLAGS,
+    /*tp_doc*/              0,
+    /*tp_traverse*/         0,
+    /*tp_clear*/            0,
+    /*tp_richcompare*/      0,
+    /*tp_weaklistoffset*/   0,
+    /*tp_iter*/             0,
+    /*tp_iternext*/         0,
+    /*tp_methods*/          0,
+    /*tp_members*/          0,
+    /*tp_getset*/           0,
+    /*tp_base*/             0,  //  set to &PyType_Type in module init function (why can it not be done here?)
+    /*tp_dict*/             0,
+    /*tp_descr_get*/        0,
+    /*tp_descr_set*/        0,
+    /*tp_dictoffset*/       0,
+    /*tp_init*/             0,
+    /*tp_alloc*/            0,
+    /*tp_new*/              datetimetype_tp_new,
+    /*tp_free*/             0,
+    /*tp_is_gc*/            0,
+    /*tp_bases*/            0,
+    /*tp_mro*/              0,
+    /*tp_cache*/            0,
+    /*tp_subclasses*/       0,
+    /*tp_weaklist*/         0
+};
+
 static PyMethodDef sbkMethods[] = {{NULL, NULL, 0, NULL}};
 
 /* Initialize this module. */
@@ -69,6 +135,16 @@
 initfoo3(void)
 {
     PyObject *mod, *d;
+    PyObject *module = NULL;
+    module = PyImport_ImportModule("datetime");
+    typ = PyObject_GetAttr(module, PyString_FromString("datetime"));
+    Py_DECREF(module);
+    if (!PyType_Check(typ)) {
+        PyErr_Format(PyExc_TypeError, "datetime.datetime is not a type object");
+        return;
+    }
+    datetimetype.tp_base = (PyTypeObject*)typ;
+    PyType_Ready(&datetimetype);
     footype.tp_base = &PyType_Type;
     PyType_Ready(&footype);
     mod = Py_InitModule("foo3", sbkMethods);
@@ -79,5 +155,7 @@
         return;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return;
+    if (PyDict_SetItemString(d, "datetimetype", (PyObject *)&datetimetype) < 0)
+        return;
     Py_INCREF(&footype);
 }
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -358,11 +358,15 @@
 
                 data = PyString_AS_STRING(args);
                 len = PyString_GET_SIZE(args);
-                if (data == NULL || len < 1)
+                if (data == NULL)
                     Py_RETURN_NONE;
                 obj = PyArray_Scalar(data, len);
                 return obj;
              """),
+            ("get_len", "METH_O",
+             """
+                return PyLong_FromLong(PyObject_Size(args));
+             """),
             ], prologue="""
                 #include <Python.h>
                 PyTypeObject PyStringArrType_Type = {
@@ -438,12 +442,16 @@
                 PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
                 PyStringArrType_Type.tp_itemsize = sizeof(char);
                 PyStringArrType_Type.tp_base = &PyString_Type;
+                PyStringArrType_Type.tp_hash = PyString_Type.tp_hash;
                 if (PyType_Ready(&PyStringArrType_Type) < 0) INITERROR;
             ''')
 
         a = module.newsubstr('abc')
         assert type(a).__name__ == 'string_'
         assert a == 'abc'
+        assert 3 == module.get_len(a)
+        b = module.newsubstr('')
+        assert 0 == module.get_len(b)
 
 class TestBytes(BaseApiTest):
     def test_bytes_resize(self, space):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -30,13 +30,14 @@
         assert view.c_len == 5
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
-        w_mv = from_ref(space, api.PyMemoryView_FromBuffer(view))
+        ref = api.PyMemoryView_FromBuffer(view)
+        w_mv = from_ref(space, ref)
         for f in ('format', 'itemsize', 'ndim', 'readonly', 
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)
             assert space.eq_w(space.getattr(w_mv, w_f), 
                               space.getattr(w_memoryview, w_f))
-
+        api.Py_DecRef(ref)
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     def test_fillWithObject(self):
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.api import generic_cpy_call
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 
@@ -663,30 +664,6 @@
         assert module.tp_init(list, x, ("hi",)) is None
         assert x == ["h", "i"]
 
-    def test_tp_str(self):
-        module = self.import_extension('foo', [
-           ("tp_str", "METH_VARARGS",
-            '''
-                 PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0);
-                 PyObject *obj = PyTuple_GET_ITEM(args, 1);
-                 if (!type->tp_str)
-                 {
-                     PyErr_SetNone(PyExc_ValueError);
-                     return NULL;
-                 }
-                 return type->tp_str(obj);
-             '''
-             )
-            ])
-        class C:
-            def __str__(self):
-                return "text"
-        assert module.tp_str(type(C()), C()) == "text"
-        class D(int):
-            def __str__(self):
-                return "more text"
-        assert module.tp_str(int, D(42)) == "42"
-
     def test_mp_ass_subscript(self):
         module = self.import_extension('foo', [
            ("new_obj", "METH_NOARGS",
@@ -978,9 +955,12 @@
         assert (d + a) == 5
         assert pow(d,b) == 16
 
-    def test_tp_new_in_subclass_of_type(self):
+    def test_tp_new_in_subclass(self):
+        import datetime
         module = self.import_module(name='foo3')
         module.footype("X", (object,), {})
+        a = module.datetimetype(1, 1, 1)
+        assert isinstance(a, module.datetimetype)
 
     def test_app_subclass_of_c_type(self):
         import sys
@@ -1165,7 +1145,6 @@
             self._check_type_object(FooType)
         class X(object):
             __metaclass__ = FooType
-        print repr(X)
         X()
 
     def test_multiple_inheritance3(self):
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -0,0 +1,208 @@
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.pyobject import make_ref, from_ref
+from pypy.module.cpyext.api import generic_cpy_call
+from pypy.module.cpyext.typeobject import PyTypeObjectPtr
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class TestAppLevelObject(BaseApiTest):
+    def test_nb_add_from_python(self, space, api):
+        w_date = space.appexec([], """():
+            class DateType(object):
+                def __add__(self, other):
+                    return 'sum!'
+            return DateType()
+            """)
+        w_datetype = space.type(w_date)
+        py_date = make_ref(space, w_date)
+        py_datetype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_datetype))
+        assert py_datetype.c_tp_as_number
+        assert py_datetype.c_tp_as_number.c_nb_add
+        w_obj = generic_cpy_call(space, py_datetype.c_tp_as_number.c_nb_add,
+                                 py_date, py_date)
+        assert space.str_w(w_obj) == 'sum!'
+
+    def test_tp_new_from_python(self, space, api):
+        w_date = space.appexec([], """():
+            class Date(object):
+                def __new__(cls, year, month, day):
+                    self = object.__new__(cls)
+                    self.year = year
+                    self.month = month
+                    self.day = day
+                    return self
+            return Date
+            """)
+        py_datetype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_date))
+        one = space.newint(1)
+        arg = space.newtuple([one, one, one])
+        # call w_date.__new__
+        w_obj = space.call_function(w_date, one, one, one)
+        w_year = space.getattr(w_obj, space.newbytes('year'))
+        assert space.int_w(w_year) == 1
+
+        w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype, 
+                                 arg, space.newdict({}))
+        w_year = space.getattr(w_obj, space.newbytes('year'))
+        assert space.int_w(w_year) == 1
+
+class AppTestUserSlots(AppTestCpythonExtensionBase):
+    def test_tp_hash_from_python(self):
+        # to see that the functions are being used,
+        # run pytest with -s
+        module = self.import_extension('foo', [
+           ("use_hash", "METH_O",
+            '''
+                long hash = args->ob_type->tp_hash(args);
+                return PyLong_FromLong(hash);
+            ''')])
+        class C(object):
+            def __hash__(self):
+                return -23
+        c = C()
+        # uses the userslot slot_tp_hash
+        ret = module.use_hash(C())
+        assert hash(c) == ret
+        # uses the slotdef renamed cpyext_tp_hash_int
+        ret = module.use_hash(3)
+        assert hash(3) == ret
+
+    def test_tp_str(self):
+        module = self.import_extension('foo', [
+           ("tp_str", "METH_VARARGS",
+            '''
+                 PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0);
+                 PyObject *obj = PyTuple_GET_ITEM(args, 1);
+                 if (!type->tp_str)
+                 {
+                     PyErr_SetString(PyExc_ValueError, "no tp_str");
+                     return NULL;
+                 }
+                 return type->tp_str(obj);
+             '''
+             )
+            ])
+        class C:
+            def __str__(self):
+                return "text"
+        assert module.tp_str(type(C()), C()) == "text"
+        class D(int):
+            def __str__(self):
+                return "more text"
+        assert module.tp_str(int, D(42)) == "42"
+        class A(object):
+            pass
+        s = module.tp_str(type(A()), A())
+        assert 'A object' in s
+
+    def test_tp_deallocate(self):
+        module = self.import_extension('foo', [
+            ("get_cnt", "METH_NOARGS",
+            '''
+                return PyLong_FromLong(foocnt);
+            '''),
+            ("get__timestamp", "METH_NOARGS",
+            '''
+                PyObject * one = PyLong_FromLong(1);
+                PyObject * a = PyTuple_Pack(3, one, one, one);
+                PyObject * k = NULL;
+                obj = _Timestamp.tp_new(&_Timestamp, a, k);
+                Py_DECREF(one);
+                return obj;
+             '''),
+            ("get_timestamp", "METH_NOARGS",
+            '''
+                PyObject * one = PyLong_FromLong(1);
+                PyObject * a = PyTuple_Pack(3, one, one, one);
+                PyObject * k = NULL;
+                obj = Timestamp.tp_new(&Timestamp, a, k);
+                Py_DECREF(one);
+                return obj;
+             '''),
+            ], prologue='''
+                static int foocnt = 0;
+                static PyTypeObject* datetime_cls = NULL;
+                static PyObject * obj = NULL;
+                static PyObject*
+                _timestamp_new(PyTypeObject* t, PyObject* a, PyObject* k)
+                {
+                    foocnt ++;
+                    return datetime_cls->tp_new(t, a, k);
+                }
+
+                static PyObject*
+                timestamp_new(PyTypeObject* t, PyObject* a, PyObject* k)
+                {
+                    return datetime_cls->tp_new(t, a, k);
+                }
+
+                static void 
+                _timestamp_dealloc(PyObject *op)
+                {
+                    foocnt --;
+                    datetime_cls->tp_dealloc(op);
+                }
+                 
+
+                static PyTypeObject _Timestamp = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,                            /* ob_size */
+                    "foo._Timestamp",   /* tp_name*/
+                    0,                  /* tp_basicsize*/
+                    0,                  /* tp_itemsize */
+                    _timestamp_dealloc  /* tp_dealloc  */
+                };
+                static PyTypeObject Timestamp = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,                            /* ob_size */
+                    "foo.Timestamp",   /* tp_name*/
+                    0,                  /* tp_basicsize*/
+                    0                  /* tp_itemsize */
+                };
+            ''', more_init='''
+                PyObject * mod = PyImport_ImportModule("datetime");
+                if (mod == NULL) INITERROR;
+                PyObject * dt = PyString_FromString("datetime");
+                datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod, dt); 
+                if (datetime_cls == NULL) INITERROR;
+                _Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                _Timestamp.tp_base = datetime_cls;
+                _Timestamp.tp_new = _timestamp_new;
+                Py_DECREF(mod);
+                Py_DECREF(dt);
+                if (PyType_Ready(&_Timestamp) < 0) INITERROR;
+
+                Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                Timestamp.tp_base = &_Timestamp;
+                Timestamp.tp_new = timestamp_new;
+                Timestamp.tp_dealloc = datetime_cls->tp_dealloc;
+                if (PyType_Ready(&Timestamp) < 0) INITERROR;
+            ''')
+        # _Timestamp has __new__, __del__ and 
+        #      inherits from datetime.datetime
+        # Timestamp has __new__, default __del__ (subtype_dealloc) and
+        #      inherits from _Timestamp
+        import gc, sys
+        cnt = module.get_cnt()
+        assert cnt == 0
+        obj = module.get__timestamp() #_Timestamp
+        cnt = module.get_cnt()
+        assert cnt == 1
+        assert obj.year == 1
+        del obj
+        self.debug_collect()
+        cnt = module.get_cnt()
+        assert cnt == 0
+
+        obj = module.get_timestamp() #Timestamp
+        cnt = module.get_cnt()
+        assert cnt == 0
+        assert obj.year == 1
+        # XXX calling Timestamp.tp_dealloc which is subtype_dealloc
+        #     causes infinite recursion
+        del obj
+        self.debug_collect()
+        cnt = module.get_cnt()
+        assert cnt == 0
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -1,7 +1,7 @@
 import os
 
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
 
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
@@ -32,12 +32,12 @@
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
 from pypy.module.cpyext.typeobjectdefs import (
-    PyGetSetDef, PyMemberDef,
+    PyGetSetDef, PyMemberDef, PyMappingMethods,
     PyNumberMethods, PySequenceMethods, PyBufferProcs)
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 
-WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
+#WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
@@ -250,6 +250,7 @@
             dict_w[name] = w_descr
             i += 1
 
+missing_slots={}
 def update_all_slots(space, w_type, pto):
     # fill slots in pto
     # Not very sure about it, but according to
@@ -257,22 +258,46 @@
     # overwrite slots that are already set: these ones are probably
     # coming from a parent C type.
 
-    typedef = w_type.layout.typedef
+    if w_type.is_heaptype():
+        typedef = None
+        search_dict_w = w_type.dict_w
+    else:
+        typedef = w_type.layout.typedef
+        search_dict_w = None
+
     for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots:
-        w_descr = w_type.lookup(method_name)
-        if w_descr is None:
-            # XXX special case iternext
-            continue
+        slot_func_helper = None
+        if search_dict_w is None:
+            # built-in types: expose as many slots as possible, even
+            # if it happens to come from some parent class
+            slot_apifunc = None # use get_slot_tp_function
+        else:
+            # For heaptypes, w_type.layout.typedef will be object's typedef, and
+            # get_slot_tp_function will fail
+            w_descr = search_dict_w.get(method_name, None)
+            if w_descr:
+                # use the slot_apifunc (userslots) to lookup at runtime
+                pass
+            elif len(slot_names) ==1:
+                # 'inherit' from tp_base
+                slot_func_helper = getattr(pto.c_tp_base, slot_names[0])
+            else:
+                struct = getattr(pto.c_tp_base, slot_names[0])
+                if struct:
+                    slot_func_helper = getattr(struct, slot_names[1])
 
-        if slot_apifunc is None and typedef is not None:
-            slot_apifunc = get_slot_tp_function(space, typedef, slot_name)
-        if not slot_apifunc:
-            if WARN_ABOUT_MISSING_SLOT_FUNCTIONS:
-                os.write(2,
-                    "%s defined by %s but no slot function defined!\n" % (
-                        method_name, w_type.getname(space)))
-            continue
-        slot_func_helper = slot_apifunc.get_llhelper(space)
+        if not slot_func_helper:
+            if typedef is not None:
+                if slot_apifunc is None:
+                    slot_apifunc = get_slot_tp_function(space, typedef, slot_name)
+            if not slot_apifunc:
+                if not we_are_translated():
+                    if slot_name not in missing_slots:
+                        missing_slots[slot_name] = w_type.getname(space)
+                        print "missing slot %r/%r, discovered on %r" % (
+                            method_name, slot_name, w_type.getname(space))
+                continue
+            slot_func_helper = slot_apifunc.get_llhelper(space)
 
         # XXX special case wrapper-functions and use a "specific" slot func
 
@@ -300,6 +325,8 @@
                     STRUCT_TYPE = PySequenceMethods
                 elif slot_names[0] == 'c_tp_as_buffer':
                     STRUCT_TYPE = PyBufferProcs
+                elif slot_names[0] == 'c_tp_as_mapping':
+                    STRUCT_TYPE = PyMappingMethods
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
@@ -396,13 +423,6 @@
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
-    flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
-    base_object_pyo = make_ref(space, space.w_object)
-    base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
-    if base_pto != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
-        if not pto.c_tp_new:
-            pto.c_tp_new = base_pto.c_tp_new
-    Py_DecRef(space, base_object_pyo)
 
 def check_descr(space, w_self, w_type):
     if not space.isinstance_w(w_self, w_type):
@@ -502,9 +522,25 @@
     pto = obj.c_ob_type
     base = pto
     this_func_ptr = llslot(space, subtype_dealloc)
+    w_obj = from_ref(space, rffi.cast(PyObject, base))
+    # This wrapper is created on a specific type, call it w_A.
+    # We wish to call the dealloc function from one of the base classes of w_A,
+    # the first of which is not this function itself.
+    # w_obj is an instance of w_A or one of its subclasses. So climb up the
+    # inheritance chain until base.c_tp_dealloc is exactly this_func, and then
+    # continue on up until they differ.
+    #print 'subtype_dealloc, start from', rffi.charp2str(base.c_tp_name)
+    while base.c_tp_dealloc != this_func_ptr:
+        base = base.c_tp_base
+        assert base
+        #print '                 ne move to', rffi.charp2str(base.c_tp_name)
+        w_obj = from_ref(space, rffi.cast(PyObject, base))
     while base.c_tp_dealloc == this_func_ptr:
         base = base.c_tp_base
         assert base
+        #print '                 eq move to', rffi.charp2str(base.c_tp_name)
+        w_obj = from_ref(space, rffi.cast(PyObject, base))
+    #print '                   end with', rffi.charp2str(base.c_tp_name)
     dealloc = base.c_tp_dealloc
     # XXX call tp_del if necessary
     generic_cpy_call(space, dealloc, obj)
@@ -676,13 +712,6 @@
 
     typedescr = get_typedescr(w_type.layout.typedef)
 
-    # dealloc
-    if space.gettypeobject(w_type.layout.typedef) is w_type:
-        # only for the exact type, like 'space.w_tuple' or 'space.w_list'
-        pto.c_tp_dealloc = typedescr.get_dealloc().get_llhelper(space)
-    else:
-        # for all subtypes, use subtype_dealloc()
-        pto.c_tp_dealloc = llslot(space, subtype_dealloc)
     if space.is_w(w_type, space.w_str):
         pto.c_tp_itemsize = 1
     elif space.is_w(w_type, space.w_tuple):
@@ -713,6 +742,17 @@
     w_base = best_base(space, w_type.bases_w)
     pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base))
 
+    # dealloc
+    if space.gettypeobject(w_type.layout.typedef) is w_type:
+        # only for the exact type, like 'space.w_tuple' or 'space.w_list'
+        pto.c_tp_dealloc = typedescr.get_dealloc().get_llhelper(space)
+    else:
+        # for all subtypes, use base's dealloc (requires sorting in attach_all)
+        pto.c_tp_dealloc = pto.c_tp_base.c_tp_dealloc
+        if not pto.c_tp_dealloc:
+            # strange, but happens (ABCMeta)
+            pto.c_tp_dealloc = llslot(space, subtype_dealloc)
+
     if builder.cpyext_type_init is not None:
         builder.cpyext_type_init.append((pto, w_type))
     else:
@@ -726,11 +766,18 @@
         if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
             pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
-    # will be filled later on with the correct value
-    # may not be 0
     if space.is_w(w_type, space.w_object):
+        # will be filled later on with the correct value
+        # may not be 0
         pto.c_tp_new = cts.cast('newfunc', 1)
     update_all_slots(space, w_type, pto)
+    if not pto.c_tp_new:
+        base_object_pyo = make_ref(space, space.w_object)
+        base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
+        flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
+        if pto.c_tp_base != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
+                pto.c_tp_new = pto.c_tp_base.c_tp_new
+        Py_DecRef(space, base_object_pyo)
     pto.c_tp_flags |= Py_TPFLAGS_READY
     return pto
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -65,9 +65,10 @@
 def unicode_attach(space, py_obj, w_obj, w_userdata=None):
     "Fills a newly allocated PyUnicodeObject with a unicode string"
     py_unicode = rffi.cast(PyUnicodeObject, py_obj)
-    py_unicode.c_length = len(space.unicode_w(w_obj))
+    s = space.unicode_w(w_obj)
+    py_unicode.c_length = len(s)
     py_unicode.c_str = lltype.nullptr(rffi.CWCHARP.TO)
-    py_unicode.c_hash = space.hash_w(w_obj)
+    py_unicode.c_hash = space.hash_w(space.newunicode(s))
     py_unicode.c_defenc = lltype.nullptr(PyObject.TO)
 
 def unicode_realize(space, py_obj):
@@ -80,7 +81,7 @@
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
     w_obj.__init__(s)
-    py_uni.c_hash = space.hash_w(w_obj)
+    py_uni.c_hash = space.hash_w(space.newunicode(s))
     track_reference(space, py_obj, w_obj)
     return w_obj
 
@@ -205,9 +206,8 @@
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PyUnicode_GET_SIZE(space, w_obj):
-    """Return the size of the object.  o has to be a PyUnicodeObject (not
+    """Return the size of the object.  obj is a PyUnicodeObject (not
     checked)."""
-    assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
     return space.len_w(w_obj)
 
 @cpython_api([rffi.VOIDP], rffi.CWCHARP, error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/userslot.py
@@ -0,0 +1,112 @@
+"""
+These are the default implementation for type slots that we put
+in user-defined app-level Python classes, if the class implements
+the corresponding '__xxx__' special method.  It should mostly just
+call back the general version of the space operation.
+
+This is only approximately correct.  One problem is that some
+details are likely subtly wrong.  Another problem is that we don't
+track changes to an app-level Python class (addition or removal of
+'__xxx__' special methods) after initalization of the PyTypeObject.
+"""
+
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.argument import Arguments
+from pypy.module.cpyext.api import slot_function, PyObject, Py_ssize_t
+from pypy.module.cpyext.api import PyTypeObjectPtr
+from rpython.rtyper.lltypesystem import rffi, lltype
+
+ at slot_function([PyObject], Py_ssize_t, error=-1)
+def slot_sq_length(space, w_obj):
+    return space.int_w(space.len(w_obj))
+
+ at slot_function([PyObject], lltype.Signed, error=-1)
+def slot_tp_hash(space, w_obj):
+    return space.int_w(space.hash(w_obj))
+
+ at slot_function([PyObject, Py_ssize_t], PyObject)
+def slot_sq_item(space, w_obj, index):
+    return space.getitem(w_obj, space.wrap(index))
+
+ at slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
+def slot_tp_new(space, w_type, w_args, w_kwds):
+    # XXX problem - we need to find the actual __new__ function to call.
+    #     but we have no 'self' argument. Theoretically, self will be
+    #     w_type, but if w_type is a subclass of self, and w_type has a
+    #     __new__ function that calls super().__new__, and that call ends
+    #     up here, we will get infinite recursion. Prevent the recursion
+    #     in the simple case (from cython) where w_type is a cpytype, but
+    #     we know (since we are in this function) that self is not a cpytype
+    from pypy.module.cpyext.typeobject import W_PyCTypeObject
+    w_type0 = w_type
+    mro_w = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
+    for w_m in mro_w[1:]:
+        if not w_type0.is_cpytype():
+            break
+        w_type0 = w_m
+    w_impl = space.getattr(w_type0, space.wrap('__new__'))
+    args = Arguments(space, [w_type],
+                     w_stararg=w_args, w_starstararg=w_kwds)
+    return space.call_args(w_impl, args)
+
+# unary functions
+
+ at slot_function([PyObject], PyObject)
+def slot_tp_str(space, w_obj):
+    return space.str(w_obj)
+
+ at slot_function([PyObject], PyObject)
+def slot_tp_repr(space, w_obj):
+    return space.repr(w_obj)
+
+#binary functions
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_add(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_subtract(space, w_obj1, w_obj2):
+    return space.sub(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_multiply(space, w_obj1, w_obj2):
+    return space.mul(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_divide(space, w_obj1, w_obj2):
+    return space.div(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_add(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_subtract(space, w_obj1, w_obj2):
+    return space.sub(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_multiply(space, w_obj1, w_obj2):
+    return space.mul(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_divide(space, w_obj1, w_obj2):
+    return space.div(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_sq_concat(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_sq_inplace_concat(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_mp_subscript(space, w_obj1, w_obj2):
+    return space.getitem(w_obj1, w_obj2)
+
+ at slot_function([PyObject, PyObject], PyObject)
+def slot_tp_getattr(space, w_obj1, w_obj2):
+    return space.getattr(w_obj1, w_obj2)
+
+
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -1,5 +1,6 @@
 """The builtin str implementation"""
 
+from rpython.rlib import jit
 from rpython.rlib.jit import we_are_jitted
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin)
@@ -950,6 +951,7 @@
 W_BytesObject.typedef.flag_sequence_bug_compat = True
 
 
+ at jit.elidable
 def string_escape_encode(s, quote):
     buf = StringBuilder(len(s) + 2)
 
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1329,3 +1329,8 @@
             pass
         assert X.__dict__['__dict__'].__objclass__ is X
         assert X.__dict__['__weakref__'].__objclass__ is X
+        assert object.__dict__['__class__'].__objclass__ is object
+        assert int.__dict__['imag'].__objclass__ is int
+        assert file.closed.__objclass__ is file
+        assert type.__dict__['__name__'].__objclass__ is type
+        assert type.__dict__['__doc__'].__objclass__ is type
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -958,7 +958,7 @@
     __base__ = GetSetProperty(descr__base),
     __mro__ = GetSetProperty(descr_get__mro__),
     __dict__=GetSetProperty(type_get_dict),
-    __doc__ = GetSetProperty(descr__doc),
+    __doc__ = GetSetProperty(descr__doc, cls=W_TypeObject),
     mro = gateway.interp2app(descr_mro),
     __flags__ = GetSetProperty(descr__flags),
     __module__ = GetSetProperty(descr_get__module, descr_set__module),
@@ -1312,10 +1312,13 @@
     def build(self, typedef):
         "NOT_RPYTHON: initialization-time only."
         from pypy.objspace.std.objectobject import W_ObjectObject
+        from pypy.interpreter.typedef import GetSetProperty
+        from rpython.rlib.objectmodel import instantiate
 
         space = self.space
         rawdict = typedef.rawdict
         lazyloaders = {}
+        w_type = instantiate(W_TypeObject)
 
         # compute the bases
         if typedef is W_ObjectObject.typedef:
@@ -1327,13 +1330,16 @@
         # wrap everything
         dict_w = {}
         for descrname, descrvalue in rawdict.items():
+            # special case for GetSetProperties' __objclass__:
+            if isinstance(descrvalue, GetSetProperty):
+                descrvalue = descrvalue.copy_for_type(w_type)
             dict_w[descrname] = space.wrap(descrvalue)
 
         if typedef.applevel_subclasses_base is not None:
             overridetypedef = typedef.applevel_subclasses_base.typedef
         else:
             overridetypedef = typedef
-        w_type = W_TypeObject(space, typedef.name, bases_w, dict_w,
+        w_type.__init__(space, typedef.name, bases_w, dict_w,
                               overridetypedef=overridetypedef,
                               is_heaptype=overridetypedef.heaptype)
         if typedef is not overridetypedef:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -405,7 +405,8 @@
                 i += 1
                 self.optimizer.send_extra_operation(op)
             # force all of them except the virtuals
-            for arg in args_no_virtuals + short_jump_args:
+            for arg in (args_no_virtuals +
+                        self._map_args(mapping, short_jump_args)):
                 self.optimizer.force_box(self.get_box_replacement(arg))
             self.optimizer.flush()
             # done unless "short" has grown again
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4620,3 +4620,19 @@
             time.clock()
             return 0
         self.interp_operations(g, [])
+
+    def test_issue2465(self):
+        driver = JitDriver(greens=[], reds=['i', 'a', 'b'])
+        class F(object):
+            def __init__(self, floatval):
+                self.floatval = floatval
+        def f(i):
+            a = F(0.0)
+            b = None
+            while i > 0:
+                driver.jit_merge_point(i=i, a=a, b=b)
+                b = F(a.floatval / 1.)
+                i -= 1
+            return i
+
+        self.meta_interp(f, [10])
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1601,6 +1601,8 @@
                                       resulttype=llmemory.Address)
             hop.genop("cast_adr_to_ptr", [v_ret],
                       resultvar = hop.spaceop.result)
+        else:
+            hop.rename("same_as")
 
 
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -112,6 +112,7 @@
 
     def get_raw_address(self):
         from rpython.rtyper.lltypesystem import rffi
+        # may still raise ValueError on some GCs
         return rffi.get_raw_address_of_string(self.value)
 
 class SubBuffer(Buffer):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -542,8 +542,13 @@
         become identical to the one returned.
 
         NOTE: Only use for immutable objects!
+
+        NOTE: Might fail on some GCs!  You have to check again
+        can_move() afterwards.  It should always work with the default
+        GC.  With Boehm, can_move() is always False so
+        move_out_of_nursery() should never be called in the first place.
     """
-    pass
+    return obj
 
 class MoveOutOfNurseryEntry(ExtRegistryEntry):
     _about_ = move_out_of_nursery
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1378,7 +1378,12 @@
         CHR = chr
 
     def unicode_escape(s, size, errors, errorhandler=None):
-        # errorhandler is not used: this function cannot cause Unicode errors
+        # errors and errorhandler are not used: this function cannot cause
+        # Unicode errors
+        return _unicode_escape(s, size)
+
+    @jit.elidable
+    def _unicode_escape(s, size):
         result = STRING_BUILDER(size)
 
         if quotes:
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1336,6 +1336,8 @@
     as key is alive. If key goes out of scope, the buffer will eventually
     be freed. `string` cannot go out of scope until the RawBytes object
     referencing it goes out of scope.
+
+    NOTE: may raise ValueError on some GCs, but not the default one.
     """
     assert isinstance(string, str)
     from rpython.rtyper.annlowlevel import llstr
@@ -1346,6 +1348,8 @@
     if we_are_translated():
         if rgc.can_move(string):
             string = rgc.move_out_of_nursery(string)
+            if rgc.can_move(string):
+                raise ValueError("cannot make string immovable")
 
         # string cannot move now! return the address
         lldata = llstr(string)
diff --git a/rpython/rtyper/lltypesystem/test/test_ztranslated.py b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
--- a/rpython/rtyper/lltypesystem/test/test_ztranslated.py
+++ b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
@@ -37,7 +37,10 @@
     return ptr
 
 def main(argv=[]):
-    use_str()
+    try:
+        use_str()
+    except ValueError:
+        return 42
     gc.collect()
     mystr = b"12341234aa"*4096*10
     #debug_assert(not rgc.can_move(mystr), "long string can move... why?")
@@ -56,4 +59,15 @@
 
 def test_compiled_incminimark():
     fn = compile(main, [], gcpolicy="incminimark")
-    fn()
+    res = fn()
+    assert res == 0
+
+def test_compiled_semispace():
+    fn = compile(main, [], gcpolicy="semispace")
+    res = fn()
+    assert res == 42    # get_raw_address_of_string() raises ValueError
+
+def test_compiled_boehm():
+    fn = compile(main, [], gcpolicy="boehm")
+    res = fn()
+    assert res == 0

From pypy.commits at gmail.com  Mon Jan 16 20:46:44 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 16 Jan 2017 17:46:44 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Define tp_name and tp_doc as
 const char*
Message-ID: <587d7784.8a9a1c0a.46f5d.c3a9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89624:ca94f56f64e0
Date: 2017-01-17 01:46 +0000
http://bitbucket.org/pypy/pypy/changeset/ca94f56f64e0/

Log:	Define tp_name and tp_doc as const char*

diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -226,7 +226,7 @@
 
 typedef struct _typeobject {
 	PyObject_VAR_HEAD
-	/* const */ char *tp_name; /* For printing, in format "<module>.<name>" */
+	const char *tp_name; /* For printing, in format "<module>.<name>" */
 	Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
 
 	/* Methods to implement standard operations */
@@ -258,7 +258,7 @@
 	/* Flags to define presence of optional/expanded features */
 	long tp_flags;
 
-	/*const*/ char *tp_doc; /* Documentation string */
+	const char *tp_doc; /* Documentation string */
 
 	/* Assigned meaning in release 2.0 */
 	/* call function for all accessible objects */
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -732,9 +732,10 @@
         heaptype = cts.cast('PyHeapTypeObject*', pto)
         heaptype.c_ht_name = make_ref(space, w_typename)
         from pypy.module.cpyext.bytesobject import PyString_AsString
-        pto.c_tp_name = PyString_AsString(space, heaptype.c_ht_name)
+        pto.c_tp_name = cts.cast('const char *',
+            PyString_AsString(space, heaptype.c_ht_name))
     else:
-        pto.c_tp_name = rffi.str2charp(w_type.name)
+        pto.c_tp_name = cts.cast('const char*', rffi.str2charp(w_type.name))
     # uninitialized fields:
     # c_tp_print
     # XXX implement

From pypy.commits at gmail.com  Tue Jan 17 00:11:25 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Mon, 16 Jan 2017 21:11:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix translation
Message-ID: <587da77d.849c1c0a.dd8a.6f32@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89625:340c549c6f11
Date: 2017-01-16 21:10 -0800
http://bitbucket.org/pypy/pypy/changeset/340c549c6f11/

Log:	fix translation

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -544,7 +544,7 @@
             except OperationError as e:
                 if not e.match(space, space.w_TypeError):
                     raise
-                name = '?'
+                name = u'?'
         objrepr = space.unicode_w(space.repr(self.w_instance))
         s = u'<bound method %s of %s>' % (name, objrepr)
         return space.wrap(s)

From pypy.commits at gmail.com  Tue Jan 17 04:34:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 01:34:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <587de538.e4361c0a.5e57d.0a12@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89626:99326d8d01a0
Date: 2017-01-17 10:34 +0100
http://bitbucket.org/pypy/pypy/changeset/99326d8d01a0/

Log:	fix test

diff --git a/pypy/module/test_lib_pypy/test_collections.py b/pypy/module/test_lib_pypy/test_collections.py
--- a/pypy/module/test_lib_pypy/test_collections.py
+++ b/pypy/module/test_lib_pypy/test_collections.py
@@ -219,9 +219,10 @@
                 self.default_factory = self._factory
             def _factory(self):
                 return []
+        sub._factory.__qualname__ = "FACTORY"
         d = sub()
         assert repr(d).startswith(
-            "defaultdict(<bound method sub._factory of defaultdict(...")
+            "defaultdict(<bound method FACTORY of defaultdict(...")
 
     def test_copy(self):
         collections = self.collections

From pypy.commits at gmail.com  Tue Jan 17 05:25:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 02:25:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <587df10e.54b31c0a.39a7b.06b3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89627:a3becf0c9331
Date: 2017-01-17 11:24 +0100
http://bitbucket.org/pypy/pypy/changeset/a3becf0c9331/

Log:	fix test

diff --git a/lib-python/3/test/test_json/__init__.py b/lib-python/3/test/test_json/__init__.py
--- a/lib-python/3/test/test_json/__init__.py
+++ b/lib-python/3/test/test_json/__init__.py
@@ -10,7 +10,8 @@
 cjson = support.import_fresh_module('json', fresh=['_json'])
 pyjson = support.import_fresh_module('json', blocked=['_json'])
 # JSONDecodeError is cached inside the _json module
-cjson.JSONDecodeError = cjson.decoder.JSONDecodeError = json.JSONDecodeError
+if cjson is not None:
+    cjson.JSONDecodeError = cjson.decoder.JSONDecodeError = json.JSONDecodeError
 
 # create two base classes that will be used by the other tests
 class PyTest(unittest.TestCase):

From pypy.commits at gmail.com  Tue Jan 17 05:30:05 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 17 Jan 2017 02:30:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: change comment,
 still needs cffi binding for COMP_get_type
Message-ID: <587df22d.09bb1c0a.27a9b.2c25@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89629:478e858ada9e
Date: 2017-01-17 11:27 +0100
http://bitbucket.org/pypy/pypy/changeset/478e858ada9e/

Log:	change comment, still needs cffi binding for COMP_get_type

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -527,7 +527,7 @@
             return None
 
         comp_method = lib.SSL_get_current_compression(self.ssl);
-        if comp_method == ffi.NULL: # or comp_method.type == lib.NID_undef:
+        if comp_method == ffi.NULL: # or lib.SSL_COMP_get_type(comp_method) == lib.NID_undef:
             return None
         short_name = lib.SSL_COMP_get_name(comp_method)
         if short_name == ffi.NULL:

From pypy.commits at gmail.com  Tue Jan 17 05:30:07 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 17 Jan 2017 02:30:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <587df22f.878f1c0a.ab837.344c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89630:ae346d1e5a98
Date: 2017-01-17 11:29 +0100
http://bitbucket.org/pypy/pypy/changeset/ae346d1e5a98/

Log:	merge heads

diff --git a/lib-python/3/test/test_json/__init__.py b/lib-python/3/test/test_json/__init__.py
--- a/lib-python/3/test/test_json/__init__.py
+++ b/lib-python/3/test/test_json/__init__.py
@@ -10,7 +10,8 @@
 cjson = support.import_fresh_module('json', fresh=['_json'])
 pyjson = support.import_fresh_module('json', blocked=['_json'])
 # JSONDecodeError is cached inside the _json module
-cjson.JSONDecodeError = cjson.decoder.JSONDecodeError = json.JSONDecodeError
+if cjson is not None:
+    cjson.JSONDecodeError = cjson.decoder.JSONDecodeError = json.JSONDecodeError
 
 # create two base classes that will be used by the other tests
 class PyTest(unittest.TestCase):

From pypy.commits at gmail.com  Tue Jan 17 05:30:02 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 17 Jan 2017 02:30:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: _fs_decode(_str_from_buf(ffistr)) will
 never work (thanks armin)
Message-ID: <587df22a.0b561c0a.ea73d.1b6a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89628:7952b451d86c
Date: 2017-01-17 11:15 +0100
http://bitbucket.org/pypy/pypy/changeset/7952b451d86c/

Log:	_fs_decode(_str_from_buf(ffistr)) will never work (thanks armin)

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -532,7 +532,7 @@
         short_name = lib.SSL_COMP_get_name(comp_method)
         if short_name == ffi.NULL:
             return None
-        return _fs_decode(_str_from_buf(short_name))
+        return _cstr_decode_fs(short_name)
 
     def version(self):
         if self.ssl == ffi.NULL:

From pypy.commits at gmail.com  Tue Jan 17 09:15:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 06:15:31 -0800 (PST)
Subject: [pypy-commit] cffi default: Add '#pragma GCC visibility
 push(default)' in case the user
Message-ID: <587e2703.958c1c0a.6a8d6.148c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2861:08a1e3c711a3
Date: 2017-01-17 15:14 +0100
http://bitbucket.org/cffi/cffi/changeset/08a1e3c711a3/

Log:	Add '#pragma GCC visibility push(default)' in case the user
	specifies -fvisibility=hidden

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -391,6 +391,10 @@
         prnt()
         #
         # the init function
+        prnt('#ifdef __GNUC__')
+        prnt('#  pragma GCC visibility push(default)  /* for -fvisibility= */')
+        prnt('#endif')
+        prnt()
         prnt('#ifdef PYPY_VERSION')
         prnt('PyMODINIT_FUNC')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
@@ -429,6 +433,10 @@
             self.module_name, version))
         prnt('}')
         prnt('#endif')
+        prnt()
+        prnt('#ifdef __GNUC__')
+        prnt('#  pragma GCC visibility pop')
+        prnt('#endif')
 
     def _to_py(self, x):
         if isinstance(x, str):
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -31,6 +31,7 @@
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
     else:
+        # add '-Werror' to the existing 'extra_compile_args' flags
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
                                       ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
@@ -2175,3 +2176,15 @@
         "  Such structs are only supported as return value if the function is "
         "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()"
         "+ffibuilder.set_source() and not taking a final '...' argument)")
+
+def test_gcc_visibility_hidden():
+    if sys.platform == 'win32':
+        py.test.skip("test for gcc/clang")
+    ffi = FFI()
+    ffi.cdef("""
+    int f(int);
+    """)
+    lib = verify(ffi, "test_gcc_visibility_hidden", """
+    int f(int a) { return a + 40; }
+    """, extra_compile_args=['-fvisibility=hidden'])
+    assert lib.f(2) == 42

From pypy.commits at gmail.com  Tue Jan 17 10:32:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 07:32:24 -0800 (PST)
Subject: [pypy-commit] pypy default: * unify the error messages raised by
 decode_string() and
Message-ID: <587e3908.4e9d1c0a.5f1a1.9788@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89631:081a08995764
Date: 2017-01-17 16:31 +0100
http://bitbucket.org/pypy/pypy/changeset/081a08995764/

Log:	* unify the error messages raised by decode_string() and
	decode_string_escaped(), which both contained only half of the logic

	* avoid one string copy

	* use StringBuilder.append(char) instead of
	StringBuilder.append_multiple_chars(char, 1)

	* test and fix the position reported in some messages

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -106,7 +106,7 @@
             return self.decode_numeric(i)
         else:
             self._raise("No JSON object could be decoded: unexpected '%s' at char %d",
-                        ch, self.pos)
+                        ch, i)
 
     def decode_null(self, i):
         if (self.ll_chars[i]   == 'u' and
@@ -244,7 +244,7 @@
                 self._raise("Unterminated array starting at char %d", start)
             else:
                 self._raise("Unexpected '%s' when decoding array (char %d)",
-                            ch, self.pos)
+                            ch, i-1)
 
     def decode_object(self, i):
         start = i
@@ -282,7 +282,7 @@
                 self._raise("Unterminated object starting at char %d", start)
             else:
                 self._raise("Unexpected '%s' when decoding object (char %d)",
-                            ch, self.pos)
+                            ch, i-1)
 
 
     def decode_string(self, i):
@@ -307,18 +307,17 @@
                 self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.wrap(content_unicode)
-            elif ch == '\\':
-                content_so_far = self.getslice(start, i-1)
+            elif ch == '\\' or ch < '\x20':
                 self.pos = i-1
-                return self.decode_string_escaped(start, content_so_far)
-            elif ch < '\x20':
-                self._raise("Invalid control character at char %d", self.pos-1)
+                return self.decode_string_escaped(start)
 
 
-    def decode_string_escaped(self, start, content_so_far):
-        builder = StringBuilder(len(content_so_far)*2) # just an estimate
-        builder.append(content_so_far)
+    def decode_string_escaped(self, start):
         i = self.pos
+        builder = StringBuilder((i - start) * 2) # just an estimate
+        assert start >= 0
+        assert i >= 0
+        builder.append_slice(self.s, start, i)
         while True:
             ch = self.ll_chars[i]
             i += 1
@@ -330,27 +329,31 @@
                 return self.space.wrap(content_unicode)
             elif ch == '\\':
                 i = self.decode_escape_sequence(i, builder)
-            elif ch == '\0':
-                self._raise("Unterminated string starting at char %d", start)
+            elif ch < '\x20':
+                if ch == '\0':
+                    self._raise("Unterminated string starting at char %d",
+                                start - 1)
+                else:
+                    self._raise("Invalid control character at char %d", i-1)
             else:
-                builder.append_multiple_char(ch, 1) # we should implement append_char
+                builder.append(ch)
 
     def decode_escape_sequence(self, i, builder):
         ch = self.ll_chars[i]
         i += 1
-        put = builder.append_multiple_char
-        if ch == '\\':  put('\\', 1)
-        elif ch == '"': put('"' , 1)
-        elif ch == '/': put('/' , 1)
-        elif ch == 'b': put('\b', 1)
-        elif ch == 'f': put('\f', 1)
-        elif ch == 'n': put('\n', 1)
-        elif ch == 'r': put('\r', 1)
-        elif ch == 't': put('\t', 1)
+        put = builder.append
+        if ch == '\\':  put('\\')
+        elif ch == '"': put('"' )
+        elif ch == '/': put('/' )
+        elif ch == 'b': put('\b')
+        elif ch == 'f': put('\f')
+        elif ch == 'n': put('\n')
+        elif ch == 'r': put('\r')
+        elif ch == 't': put('\t')
         elif ch == 'u':
             return self.decode_escape_sequence_unicode(i, builder)
         else:
-            self._raise("Invalid \\escape: %s (char %d)", ch, self.pos-1)
+            self._raise("Invalid \\escape: %s (char %d)", ch, i-1)
         return i
 
     def decode_escape_sequence_unicode(self, i, builder):
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -214,3 +214,20 @@
         assert check("a\"c") == "a\\\"c"
         assert check("\\\"\b\f\n\r\t") == '\\\\\\"\\b\\f\\n\\r\\t'
         assert check("\x07") == "\\u0007"
+
+    def test_error_position(self):
+        import _pypyjson
+        test_cases = [
+            ('[,', "No JSON object could be decoded: unexpected ',' at char 1"),
+            ('{"spam":[}', "No JSON object could be decoded: unexpected '}' at char 9"),
+            ('[42:', "Unexpected ':' when decoding array (char 3)"),
+            ('[42 "spam"', "Unexpected '\"' when decoding array (char 4)"),
+            ('[42,]', "No JSON object could be decoded: unexpected ']' at char 4"),
+            ('{"spam":[42}', "Unexpected '}' when decoding array (char 11)"),
+            ('["]', 'Unterminated string starting at char 1'),
+            ('["spam":', "Unexpected ':' when decoding array (char 7)"),
+            ('[{]', "No JSON object could be decoded: unexpected ']' at char 2"),
+        ]
+        for inputtext, errmsg in test_cases:
+            exc = raises(ValueError, _pypyjson.loads, inputtext)
+            assert str(exc.value) == errmsg

From pypy.commits at gmail.com  Tue Jan 17 10:32:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 07:32:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <587e3929.4e9d1c0a.5f1a1.97d3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89633:f94e694e3ca5
Date: 2017-01-17 16:32 +0100
http://bitbucket.org/pypy/pypy/changeset/f94e694e3ca5/

Log:	merge heads

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -527,12 +527,12 @@
             return None
 
         comp_method = lib.SSL_get_current_compression(self.ssl);
-        if comp_method == ffi.NULL: # or comp_method.type == lib.NID_undef:
+        if comp_method == ffi.NULL: # or lib.SSL_COMP_get_type(comp_method) == lib.NID_undef:
             return None
         short_name = lib.SSL_COMP_get_name(comp_method)
         if short_name == ffi.NULL:
             return None
-        return _fs_decode(_str_from_buf(short_name))
+        return _cstr_decode_fs(short_name)
 
     def version(self):
         if self.ssl == ffi.NULL:

From pypy.commits at gmail.com  Tue Jan 17 10:32:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 07:32:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Move the error reporting from
 json.decode() in line with Python 3's
Message-ID: <587e3927.43e61c0a.451d3.d0fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89632:b947344ca26e
Date: 2017-01-17 12:33 +0100
http://bitbucket.org/pypy/pypy/changeset/b947344ca26e/

Log:	Move the error reporting from json.decode() in line with Python 3's
	expectations

diff --git a/lib-python/3/json/__init__.py b/lib-python/3/json/__init__.py
--- a/lib-python/3/json/__init__.py
+++ b/lib-python/3/json/__init__.py
@@ -322,7 +322,8 @@
     if (cls is None and object_hook is None and
             parse_int is None and parse_float is None and
             parse_constant is None and object_pairs_hook is None and not kw):
-        return _pypyjson.loads(s) if _pypyjson else _default_decoder.decode(s)
+        return (_pypyjson.loads(s, JSONDecodeError)
+                if _pypyjson else _default_decoder.decode(s))
     if cls is None:
         cls = JSONDecoder
     if object_hook is not None:
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -3,7 +3,7 @@
 from rpython.rlib.objectmodel import specialize, always_inline
 from rpython.rlib import rfloat, runicode
 from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import oefmt, OperationError
 from pypy.interpreter import unicodehelper
 
 OVF_DIGITS = len(str(sys.maxint))
@@ -42,6 +42,11 @@
         ll_res.chars[i] = cast_primitive(UniChar, ch)
     return hlunicode(ll_res)
 
+class DecoderError(Exception):
+    def __init__(self, msg, pos):
+        self.msg = msg
+        self.pos = pos
+
 TYPE_UNKNOWN = 0
 TYPE_STRING = 1
 class JSONDecoder(object):
@@ -76,10 +81,6 @@
                 break
         return i
 
-    @specialize.arg(1)
-    def _raise(self, msg, *args):
-        raise oefmt(self.space.w_ValueError, msg, *args)
-
     def decode_any(self, i):
         i = self.skip_whitespace(i)
         ch = self.ll_chars[i]
@@ -106,8 +107,9 @@
         elif ch.isdigit():
             return self.decode_numeric(i)
         else:
-            self._raise("No JSON object could be decoded: unexpected '%s' at char %d",
-                        ch, self.pos)
+            raise DecoderError(
+                "No JSON object could be decoded: unexpected '%s' at" % ch,
+                self.pos)
 
     def decode_null(self, i):
         if (self.ll_chars[i]   == 'u' and
@@ -115,7 +117,7 @@
             self.ll_chars[i+2] == 'l'):
             self.pos = i+3
             return self.space.w_None
-        self._raise("Error when decoding null at char %d", i)
+        raise DecoderError("Error when decoding null at", i)
 
     def decode_true(self, i):
         if (self.ll_chars[i]   == 'r' and
@@ -123,7 +125,7 @@
             self.ll_chars[i+2] == 'e'):
             self.pos = i+3
             return self.space.w_True
-        self._raise("Error when decoding true at char %d", i)
+        raise DecoderError("Error when decoding true at", i)
 
     def decode_false(self, i):
         if (self.ll_chars[i]   == 'a' and
@@ -132,7 +134,7 @@
             self.ll_chars[i+3] == 'e'):
             self.pos = i+4
             return self.space.w_False
-        self._raise("Error when decoding false at char %d", i)
+        raise DecoderError("Error when decoding false at", i)
 
     def decode_infinity(self, i, sign=1):
         if (self.ll_chars[i]   == 'n' and
@@ -144,14 +146,14 @@
             self.ll_chars[i+6] == 'y'):
             self.pos = i+7
             return self.space.wrap(rfloat.INFINITY * sign)
-        self._raise("Error when decoding Infinity at char %d", i)
+        raise DecoderError("Error when decoding Infinity at", i)
 
     def decode_nan(self, i):
         if (self.ll_chars[i]   == 'a' and
             self.ll_chars[i+1] == 'N'):
             self.pos = i+2
             return self.space.wrap(rfloat.NAN)
-        self._raise("Error when decoding NaN at char %d", i)
+        raise DecoderError("Error when decoding NaN at", i)
 
     def decode_numeric(self, i):
         start = i
@@ -161,7 +163,7 @@
         ch = self.ll_chars[i]
         if ch == '.':
             if not self.ll_chars[i+1].isdigit():
-                self._raise("Expected digit at char %d", i+1)
+                raise DecoderError("Expected digit at", i+1)
             return self.decode_float(start)
         elif ch == 'e' or ch == 'E':
             return self.decode_float(start)
@@ -215,7 +217,7 @@
                 break
         count = i - start
         if count == 0:
-            self._raise("Expected digit at char %d", i)
+            raise DecoderError("Expected digit at", i)
         # if the number has more digits than OVF_DIGITS, it might have
         # overflowed
         ovf_maybe = (count >= OVF_DIGITS)
@@ -242,10 +244,10 @@
             elif ch == ',':
                 pass
             elif ch == '\0':
-                self._raise("Unterminated array starting at char %d", start)
+                raise DecoderError("Unterminated array starting at", start)
             else:
-                self._raise("Unexpected '%s' when decoding array (char %d)",
-                            ch, self.pos)
+                raise DecoderError("Unexpected '%s' when decoding array" % ch,
+                                   self.pos)
 
     def decode_object(self, i):
         start = i
@@ -261,13 +263,13 @@
             self.last_type = TYPE_UNKNOWN
             w_name = self.decode_any(i)
             if self.last_type != TYPE_STRING:
-                self._raise("Key name must be string for object starting at char %d", start)
+                raise DecoderError("Key name must be string for object starting at", start)
             w_name = self.memo.setdefault(self.space.unicode_w(w_name), w_name)
 
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             if ch != ':':
-                self._raise("No ':' found at char %d", i)
+                raise DecoderError("No ':' found at", i)
             i += 1
             i = self.skip_whitespace(i)
             #
@@ -282,10 +284,10 @@
             elif ch == ',':
                 pass
             elif ch == '\0':
-                self._raise("Unterminated object starting at char %d", start)
+                raise DecoderError("Unterminated object starting at", start)
             else:
-                self._raise("Unexpected '%s' when decoding object (char %d)",
-                            ch, self.pos)
+                raise DecoderError("Unexpected '%s' when decoding object" % ch,
+                                   self.pos)
 
 
     def decode_string(self, i):
@@ -315,7 +317,7 @@
                 self.pos = i-1
                 return self.decode_string_escaped(start, content_so_far)
             elif ch < '\x20':
-                self._raise("Invalid control character at char %d", self.pos-1)
+                raise DecoderError("Invalid control character at", self.pos-1)
 
 
     def decode_string_escaped(self, start, content_so_far):
@@ -335,7 +337,7 @@
             elif ch == '\\':
                 i = self.decode_escape_sequence(i, builder)
             elif ch == '\0':
-                self._raise("Unterminated string starting at char %d", start)
+                raise DecoderError("Unterminated string starting at", start)
             else:
                 builder.append_multiple_char(ch, 1) # we should implement append_char
 
@@ -354,7 +356,7 @@
         elif ch == 'u':
             return self.decode_escape_sequence_unicode(i, builder)
         else:
-            self._raise("Invalid \\escape: %s (char %d)", ch, self.pos-1)
+            raise DecoderError("Invalid \\escape: %s" % ch, self.pos-1)
         return i
 
     def decode_escape_sequence_unicode(self, i, builder):
@@ -370,9 +372,7 @@
                     val = self.decode_surrogate_pair(i, val)
                     i += 6
         except ValueError:
-            self._raise("Invalid \uXXXX escape (char %d)", i-1)
-            return # help the annotator to know that we'll never go beyond
-                   # this point
+            raise DecoderError("Invalid \\uXXXX escape", i-1)
         #
         uchr = runicode.code_to_unichr(val)     # may be a surrogate pair again
         utf8_ch = unicodehelper.encode_utf8(
@@ -389,7 +389,7 @@
         lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller
         return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00))
 
-def loads(space, w_s):
+def loads(space, w_s, w_errorcls=None):
     if space.isinstance_w(w_s, space.w_bytes):
         raise oefmt(space.w_TypeError, "Expected string, got %T", w_s)
     s = space.str_w(w_s)
@@ -399,9 +399,13 @@
         i = decoder.skip_whitespace(decoder.pos)
         if i < len(s):
             start = i
-            end = len(s) - 1
-            raise oefmt(space.w_ValueError,
-                        "Extra data: char %d - %d", start, end)
+            raise DecoderError('Extra data', start)
         return w_res
+    except DecoderError as e:
+        if w_errorcls is None:
+            w_errorcls = space.w_ValueError
+        w_e = space.call_function(w_errorcls, space.wrap(e.msg), w_s,
+                                  space.wrap(e.pos))
+        raise OperationError(w_errorcls, w_e)
     finally:
         decoder.close()
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -222,3 +222,10 @@
         (a, b), (c, d) = sorted(rval[0]), sorted(rval[1])
         assert a is c
         assert b is d
+
+    def test_custom_error_class(self):
+        import _pypyjson
+        class MyError(Exception):
+            pass
+        exc = raises(MyError, _pypyjson.loads, 'nul', MyError)
+        assert exc.value.args == ('Error when decoding null at', 'nul', 1)

From pypy.commits at gmail.com  Tue Jan 17 11:47:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 08:47:16 -0800 (PST)
Subject: [pypy-commit] pypy default: Backport changes from py3.5
Message-ID: <587e4a94.07941c0a.3db71.f01d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89634:97d138151271
Date: 2017-01-17 17:46 +0100
http://bitbucket.org/pypy/pypy/changeset/97d138151271/

Log:	Backport changes from py3.5

diff --git a/rpython/rlib/rStringIO.py b/rpython/rlib/rStringIO.py
--- a/rpython/rlib/rStringIO.py
+++ b/rpython/rlib/rStringIO.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.objectmodel import we_are_translated
 
 AT_END = -1
 
@@ -154,8 +155,11 @@
         assert p >= 0
         self.__copy_into_bigbuffer()
         end = len(self.__bigbuffer)
-        if size >= 0 and size < end - p:
+        count = end - p
+        if size >= 0 and size < count:
             end = p + size
+        if count <= 0:
+            return ''
         i = p
         while i < end:
             finished = self.__bigbuffer[i] == '\n'
@@ -163,6 +167,11 @@
             if finished:
                 break
         self.__pos = i
+        if not we_are_translated():
+            # assert that we read within the bounds!
+            bl = len(self.__bigbuffer)
+            assert p <= bl
+            assert i <= bl
         return ''.join(self.__bigbuffer[p:i])
 
     def truncate(self, size):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1474,6 +1474,8 @@
 # This function is also used by _codecs/interp_codecs.py
 (unicode_encode_unicode_escape, raw_unicode_escape_helper
  ) = make_unicode_escape_function()
+(_, raw_unicode_escape_helper_unicode
+) = make_unicode_escape_function(unicode_output=True)
 
 # ____________________________________________________________
 # Raw unicode escape
diff --git a/rpython/rlib/test/test_rStringIO.py b/rpython/rlib/test/test_rStringIO.py
--- a/rpython/rlib/test/test_rStringIO.py
+++ b/rpython/rlib/test/test_rStringIO.py
@@ -91,6 +91,10 @@
     assert f.readline() == 'baz'
     assert f.readline() == ''
 
+    f.seek(100000, 0)
+    assert f.tell() == 100000
+    assert f.readline() == ''
+
 def test_truncate():
     f = RStringIO()
     f.truncate(20)

From pypy.commits at gmail.com  Tue Jan 17 11:58:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 08:58:18 -0800 (PST)
Subject: [pypy-commit] pypy default: fix reference leak
Message-ID: <587e4d2a.07941c0a.3db71.f71c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89635:37645aa1733f
Date: 2017-01-17 17:57 +0100
http://bitbucket.org/pypy/pypy/changeset/37645aa1733f/

Log:	fix reference leak

diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -137,7 +137,7 @@
     PyObject *mod, *d;
     PyObject *module = NULL;
     module = PyImport_ImportModule("datetime");
-    typ = PyObject_GetAttr(module, PyString_FromString("datetime"));
+    typ = PyObject_GetAttrString(module, "datetime");
     Py_DECREF(module);
     if (!PyType_Check(typ)) {
         PyErr_Format(PyExc_TypeError, "datetime.datetime is not a type object");

From pypy.commits at gmail.com  Tue Jan 17 12:03:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 09:03:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <587e4e7b.e4361c0a.5e57d.fbea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89637:91192b388a6e
Date: 2017-01-17 18:03 +0100
http://bitbucket.org/pypy/pypy/changeset/91192b388a6e/

Log:	hg merge default


From pypy.commits at gmail.com  Tue Jan 17 12:03:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 09:03:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <587e4e79.85e11c0a.9bfc0.e62d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89636:6d7e97a686d7
Date: 2017-01-17 18:03 +0100
http://bitbucket.org/pypy/pypy/changeset/6d7e97a686d7/

Log:	hg merge default

diff too long, truncating to 2000 out of 3163 lines

diff --git a/lib-python/2.7/sqlite3/test/regression.py b/lib-python/2.7/sqlite3/test/regression.py
--- a/lib-python/2.7/sqlite3/test/regression.py
+++ b/lib-python/2.7/sqlite3/test/regression.py
@@ -351,10 +351,7 @@
         self.assertRaises(ValueError, cur.execute, " \0select 2")
         self.assertRaises(ValueError, cur.execute, "select 2\0")
 
-    @test_support.impl_detail(pypy=False)
     def CheckCommitCursorReset(self):
-        # This test is for logic added in 2.7.13 which PyPy doesn't
-        # implement.  See http://bugs.python.org/issue29006
         """
         Connection.commit() did reset cursors, which made sqlite3
         to return rows multiple times when fetched from cursors
diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py
--- a/lib-python/2.7/test/test_capi.py
+++ b/lib-python/2.7/test/test_capi.py
@@ -26,7 +26,6 @@
 skips = []
 if support.check_impl_detail(pypy=True):
     skips += [
-            'test_broken_memoryview',
             'test_buildvalue_N',
             'test_capsule',
             'test_lazy_hash_inheritance',
diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py
--- a/lib-python/2.7/test/test_multiprocessing.py
+++ b/lib-python/2.7/test/test_multiprocessing.py
@@ -1969,9 +1969,10 @@
         if not gc.isenabled():
             gc.enable()
             self.addCleanup(gc.disable)
-        #thresholds = gc.get_threshold()
-        #self.addCleanup(gc.set_threshold, *thresholds)
-        #gc.set_threshold(10)
+        if test_support.check_impl_detail(cpython=True):
+            thresholds = gc.get_threshold()
+            self.addCleanup(gc.set_threshold, *thresholds)
+            gc.set_threshold(10)
 
         # perform numerous block allocations, with cyclic references to make
         # sure objects are collected asynchronously by the gc
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -231,6 +231,7 @@
         self.__statements_counter = 0
         self.__rawstatements = set()
         self._statement_cache = _StatementCache(self, cached_statements)
+        self.__statements_already_committed = []
 
         self.__func_cache = {}
         self.__aggregates = {}
@@ -374,6 +375,14 @@
                 if cursor is not None:
                     cursor._reset = True
 
+    def _reset_already_committed_statements(self):
+        lst = self.__statements_already_committed
+        self.__statements_already_committed = []
+        for weakref in lst:
+            statement = weakref()
+            if statement is not None:
+                statement._reset()
+
     @_check_thread_wrap
     @_check_closed_wrap
     def __call__(self, sql):
@@ -429,15 +438,18 @@
         if not self._in_transaction:
             return
 
-        # The following line is a KNOWN DIFFERENCE with CPython 2.7.13.
-        # More precisely, the corresponding line was removed in the
-        # version 2.7.13 of CPython, but this is causing troubles for
-        # PyPy (and potentially for CPython too):
-        #
-        #     http://bugs.python.org/issue29006
-        #
-        # So for now, we keep this line.
-        self.__do_all_statements(Statement._reset, False)
+        # PyPy fix for non-refcounting semantics: since 2.7.13 (and in
+        # <= 2.6.x), the statements are not automatically reset upon
+        # commit.  However, if this is followed by some specific SQL
+        # operations like "drop table", these open statements come in
+        # the way and cause the "drop table" to fail.  On CPython the
+        # problem is much less important because typically all the old
+        # statements are freed already by reference counting.  So here,
+        # we copy all the still-alive statements to another list which
+        # is usually ignored, except if we get SQLITE_LOCKED
+        # afterwards---at which point we reset all statements in this
+        # list.
+        self.__statements_already_committed = self.__statements[:]
 
         statement_star = _ffi.new('sqlite3_stmt **')
         ret = _lib.sqlite3_prepare_v2(self._db, b"COMMIT", -1,
@@ -866,8 +878,18 @@
                 self.__statement._set_params(params)
 
                 # Actually execute the SQL statement
+
                 ret = _lib.sqlite3_step(self.__statement._statement)
 
+                # PyPy: if we get SQLITE_LOCKED, it's probably because
+                # one of the cursors created previously is still alive
+                # and not reset and the operation we're trying to do
+                # makes Sqlite unhappy about that.  In that case, we
+                # automatically reset all old cursors and try again.
+                if ret == _lib.SQLITE_LOCKED:
+                    self.__connection._reset_already_committed_statements()
+                    ret = _lib.sqlite3_step(self.__statement._statement)
+
                 if ret == _lib.SQLITE_ROW:
                     if multiple:
                         raise ProgrammingError("executemany() can only execute DML statements.")
diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -2856,7 +2856,7 @@
     m = PyModule_Create(&_testcapimodule);
     if (m == NULL)
         return NULL;
-
+    
     Py_TYPE(&_HashInheritanceTester_Type)=&PyType_Type;
 
     Py_TYPE(&test_structmembersType)=&PyType_Type;
diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -374,7 +374,7 @@
 
     sample_count = _sample_count(cp, size)
 
-    rv = ffi.new("unsigned char[]", len(cp) * 2)
+    rv = ffi.new("char[]", len(cp) * 2)
     lib.tostereo(rv, ffi.from_buffer(cp), len(cp), size, fac1, fac2)
     return ffi.buffer(rv)[:]
 
@@ -385,7 +385,7 @@
     if len(cp1) != len(cp2):
         raise error("Lengths should be the same")
 
-    rv = ffi.new("unsigned char[]", len(cp1))
+    rv = ffi.new("char[]", len(cp1))
     lib.add(rv, ffi.from_buffer(cp1), ffi.from_buffer(cp2), len(cp1), size)
     return ffi.buffer(rv)[:]
 
@@ -488,7 +488,7 @@
     ceiling = (q + 1) * outrate
     nbytes = ceiling * bytes_per_frame
 
-    rv = ffi.new("unsigned char[]", nbytes)
+    rv = ffi.new("char[]", nbytes)
     trim_index = lib.ratecv(rv, cp, frame_count, size,
                             nchannels, inrate, outrate,
                             state_d, prev_i, cur_i,
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -496,11 +496,6 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* The ``sqlite`` module was updated on 2.7.13 to no longer reset all
-  cursors when there is a commit.  This causes troubles for PyPy (and
-  potentially for CPython too), and so for now we didn't port this change:
-  see http://bugs.python.org/issue29006.
-
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -26,16 +26,16 @@
 
 Almost!
 
-The mostly likely stumbling block for any given project is support for
+The most likely stumbling block for any given project is support for
 :ref:`extension modules <extension-modules>`.  PyPy supports a continually growing
 number of extension modules, but so far mostly only those found in the
 standard library.
 
 The language features (including builtin types and functions) are very
-complete and well tested, so if your project does not use many
+refined and well tested, so if your project doesn't use many
 extension modules there is a good chance that it will work with PyPy.
 
-We list the differences we know about in :doc:`cpython differences <cpython_differences>`.
+We list the known differences in :doc:`cpython differences <cpython_differences>`.
 
 
 Module xyz does not work with PyPy: ImportError
@@ -76,10 +76,10 @@
 
 You cannot import *any* extension module in a `sandboxed PyPy`_,
 sorry.  Even the built-in modules available are very limited.
-Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
-it is only a proof of concept.  It seriously requires someone working
-on it.  Before this occurs, it can only be used it for "pure Python"
-examples: programs that import mostly nothing (or only pure Python
+Sandboxing in PyPy is a good proof of concept, and is without a doubt
+safe IMHO, however it is only a proof of concept.  It currently requires 
+some work from a motivated developer. However, until then it can only be used for "pure Python"
+example: programs that import mostly nothing (or only pure Python
 modules, recursively).
 
 .. _`sandboxed PyPy`: sandbox.html
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -336,6 +336,6 @@
    that fixes some bugs and is translatable.
 
 *  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the files ``xxxtype.py`` and
-   ``xxxobject.py`` contain respectively the definition of the type and its
-   (default) implementation.
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -110,3 +110,21 @@
 
 Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to
 the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak.
+
+.. branch: issue2464
+
+Give (almost?) all GetSetProperties a valid __objclass__.
+
+.. branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419
+.. branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033
+
+.. branch: missing-tp_new
+
+Improve mixing app-level classes in c-extensions, especially if the app-level
+class has a ``tp_new`` or ``tp_dealloc``. The issue is that c-extensions expect
+all the method slots to be filled with a function pointer, where app-level will
+search up the mro for an appropriate function at runtime. With this branch we
+now fill many more slots in the c-extenion type objects.
+Also fix for c-extension type that calls ``tp_hash`` during initialization
+(str, unicode types), and fix instantiating c-extension types from built-in
+classes by enforcing an order of instaniation.
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -143,7 +143,11 @@
             pass
         def fget(self, space, w_self):
             assert self is prop
-        prop = typedef.GetSetProperty(fget, use_closure=True)
+        # NB. this GetSetProperty is not copied when creating the
+        # W_TypeObject because of 'cls'.  Without it, a duplicate of the
+        # GetSetProperty is taken and it is given the w_objclass that is
+        # the W_TypeObject
+        prop = typedef.GetSetProperty(fget, use_closure=True, cls=W_SomeType)
         W_SomeType.typedef = typedef.TypeDef(
             'some_type',
             x=prop)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -277,12 +277,15 @@
         self.use_closure = use_closure
 
     def copy_for_type(self, w_objclass):
-        new = instantiate(GetSetProperty)
-        new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
-                  None, self.use_closure)
-        new.name = self.name
-        new.w_objclass = w_objclass
-        return new
+        if self.objclass_getter is None:
+            new = instantiate(GetSetProperty)
+            new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
+                      None, self.use_closure)
+            new.name = self.name
+            new.w_objclass = w_objclass
+            return new
+        else:
+            return self
 
     @unwrap_spec(w_cls = WrappedDefault(None))
     def descr_property_get(self, space, w_obj, w_cls=None):
@@ -513,7 +516,7 @@
     return lifeline.get_any_weakref(space)
 
 dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
-                            doc="dictionary for instance variables")
+                            doc="dictionary for instance variables (if defined)")
 dict_descr.name = '__dict__'
 
 
@@ -548,7 +551,7 @@
     return space.newtuple([w_docstring])
 
 weakref_descr = GetSetProperty(descr_get_weakref,
-                               doc="list of weak references to the object")
+                    doc="list of weak references to the object (if defined)")
 weakref_descr.name = '__weakref__'
 
 def make_weakref_descr(cls):
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -109,7 +109,7 @@
         else:
             raise DecoderError(
                 "No JSON object could be decoded: unexpected '%s' at" % ch,
-                self.pos)
+                i)
 
     def decode_null(self, i):
         if (self.ll_chars[i]   == 'u' and
@@ -247,7 +247,7 @@
                 raise DecoderError("Unterminated array starting at", start)
             else:
                 raise DecoderError("Unexpected '%s' when decoding array" % ch,
-                                   self.pos)
+                                   i-1)
 
     def decode_object(self, i):
         start = i
@@ -287,7 +287,7 @@
                 raise DecoderError("Unterminated object starting at", start)
             else:
                 raise DecoderError("Unexpected '%s' when decoding object" % ch,
-                                   self.pos)
+                                   i-1)
 
 
     def decode_string(self, i):
@@ -312,18 +312,17 @@
                 self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.wrap(content_unicode)
-            elif ch == '\\':
-                content_so_far = self.getslice(start, i-1)
+            elif ch == '\\' or ch < '\x20':
                 self.pos = i-1
-                return self.decode_string_escaped(start, content_so_far)
-            elif ch < '\x20':
-                raise DecoderError("Invalid control character at", self.pos-1)
+                return self.decode_string_escaped(start)
 
 
-    def decode_string_escaped(self, start, content_so_far):
-        builder = StringBuilder(len(content_so_far)*2) # just an estimate
-        builder.append(content_so_far)
+    def decode_string_escaped(self, start):
         i = self.pos
+        builder = StringBuilder((i - start) * 2) # just an estimate
+        assert start >= 0
+        assert i >= 0
+        builder.append_slice(self.s, start, i)
         while True:
             ch = self.ll_chars[i]
             i += 1
@@ -336,27 +335,31 @@
                 return self.space.wrap(content_unicode)
             elif ch == '\\':
                 i = self.decode_escape_sequence(i, builder)
-            elif ch == '\0':
-                raise DecoderError("Unterminated string starting at", start)
+            elif ch < '\x20':
+                if ch == '\0':
+                    raise DecoderError("Unterminated string starting at",
+                                       start - 1)
+                else:
+                    raise DecoderError("Invalid control character at", i-1)
             else:
-                builder.append_multiple_char(ch, 1) # we should implement append_char
+                builder.append(ch)
 
     def decode_escape_sequence(self, i, builder):
         ch = self.ll_chars[i]
         i += 1
-        put = builder.append_multiple_char
-        if ch == '\\':  put('\\', 1)
-        elif ch == '"': put('"' , 1)
-        elif ch == '/': put('/' , 1)
-        elif ch == 'b': put('\b', 1)
-        elif ch == 'f': put('\f', 1)
-        elif ch == 'n': put('\n', 1)
-        elif ch == 'r': put('\r', 1)
-        elif ch == 't': put('\t', 1)
+        put = builder.append
+        if ch == '\\':  put('\\')
+        elif ch == '"': put('"' )
+        elif ch == '/': put('/' )
+        elif ch == 'b': put('\b')
+        elif ch == 'f': put('\f')
+        elif ch == 'n': put('\n')
+        elif ch == 'r': put('\r')
+        elif ch == 't': put('\t')
         elif ch == 'u':
             return self.decode_escape_sequence_unicode(i, builder)
         else:
-            raise DecoderError("Invalid \\escape: %s" % ch, self.pos-1)
+            raise DecoderError("Invalid \\escape: %s" % ch, i-1)
         return i
 
     def decode_escape_sequence_unicode(self, i, builder):
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -215,6 +215,23 @@
         assert check("\\\"\b\f\n\r\t") == '\\\\\\"\\b\\f\\n\\r\\t'
         assert check("\x07") == "\\u0007"
 
+    def test_error_position(self):
+        import _pypyjson
+        test_cases = [
+            ('[,', "No JSON object could be decoded: unexpected ',' at", 1),
+            ('{"spam":[}', "No JSON object could be decoded: unexpected '}' at", 9),
+            ('[42:', "Unexpected ':' when decoding array", 3),
+            ('[42 "spam"', "Unexpected '\"' when decoding array", 4),
+            ('[42,]', "No JSON object could be decoded: unexpected ']' at", 4),
+            ('{"spam":[42}', "Unexpected '}' when decoding array", 11),
+            ('["]', 'Unterminated string starting at', 1),
+            ('["spam":', "Unexpected ':' when decoding array", 7),
+            ('[{]', "No JSON object could be decoded: unexpected ']' at", 2),
+        ]
+        for inputtext, errmsg, errpos in test_cases:
+            exc = raises(ValueError, _pypyjson.loads, inputtext)
+            assert exc.value.args == (errmsg, inputtext, errpos)
+
     def test_keys_reuse(self):
         import _pypyjson
         s = '[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'
diff --git a/pypy/module/_ssl/test/allsans.pem b/pypy/module/_ssl/test/allsans.pem
deleted file mode 100644
--- a/pypy/module/_ssl/test/allsans.pem
+++ /dev/null
@@ -1,37 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAOoy7/QOtTjQ0niE
-6uDcTwtkC0R2Tvy1AjVnXohCntZfdzbTGDoYTgXSOLsP8A697jUiJ8VCePGH50xG
-Z4DKnAF3a9O3a9nr2pLXb0iY3XOMv+YEBii7CfI+3oxFYgCl0sMgHzDD2ZTVYAsm
-DWgLUVsE2gHEccRwrM2tPf2EgR+FAgMBAAECgYEA3qyfyYVSeTrTYxO93x6ZaVMu
-A2IZp9zSxMQL9bKiI2GRj+cV2ebSCGbg2btFnD6qBor7FWsmYz+8g6FNN/9sY4az
-61rMqMtQvLBe+7L8w70FeTze4qQ4Y1oQri0qD6tBWhDVlpnbI5Py9bkZKD67yVUk
-elcEA/5x4PrYXkuqsAECQQD80NjT0mDvaY0JOOaQFSEpMv6QiUA8GGX8Xli7IoKb
-tAolPG8rQBa+qSpcWfDMTrWw/aWHuMEEQoP/bVDH9W4FAkEA7SYQbBAKnojZ5A3G
-kOHdV7aeivRQxQk/JN8Fb8oKB9Csvpv/BsuGxPKXHdhFa6CBTTsNRtHQw/szPo4l
-xMIjgQJAPoMxqibR+0EBM6+TKzteSL6oPXsCnBl4Vk/J5vPgkbmR7KUl4+7j8N8J
-b2554TrxKEN/w7CGYZRE6UrRd7ATNQJAWD7Yz41sli+wfPdPU2xo1BHljyl4wMk/
-EPZYbI/PCbdyAH/F935WyQTIjNeEhZc1Zkq6FwdOWw8ns3hrv3rKgQJAHXv1BqUa
-czGPIFxX2TNoqtcl6/En4vrxVB1wzsfzkkDAg98kBl7qsF+S3qujSzKikjeaVbI2
-/CyWR2P3yLtOmA==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIIDcjCCAtugAwIBAgIJAN5dc9TOWjB7MA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV
-BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
-IFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2FsbHNhbnMwHhcNMTYwODA1
-MTAyMTExWhcNMjYwODAzMTAyMTExWjBdMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwO
-Q2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0
-aW9uMRAwDgYDVQQDDAdhbGxzYW5zMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
-gQDqMu/0DrU40NJ4hOrg3E8LZAtEdk78tQI1Z16IQp7WX3c20xg6GE4F0ji7D/AO
-ve41IifFQnjxh+dMRmeAypwBd2vTt2vZ69qS129ImN1zjL/mBAYouwnyPt6MRWIA
-pdLDIB8ww9mU1WALJg1oC1FbBNoBxHHEcKzNrT39hIEfhQIDAQABo4IBODCCATQw
-ggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBvdGhlciBp
-ZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2hFTAToAMC
-AQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4YW1wbGUu
-b3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMw
-IQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UEAwwPZGly
-bmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAAAYcQAAAA
-AAAAAAAAAAAAAAAAAYgEKgMEBTANBgkqhkiG9w0BAQsFAAOBgQAy16h+F+nOmeiT
-VWR0fc8F/j6FcadbLseAUaogcC15OGxCl4UYpLV88HBkABOoGCpP155qwWTwOrdG
-iYPGJSusf1OnJEbvzFejZf6u078bPd9/ZL4VWLjv+FPGkjd+N+/OaqMvgj8Lu99f
-3Y/C4S7YbHxxwff6C6l2Xli+q6gnuQ==
------END CERTIFICATE-----
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -39,6 +39,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 DEBUG_WRAPPER = True
 
@@ -903,6 +904,7 @@
         retval = fatal_value
         boxed_args = ()
         tb = None
+        state = space.fromcache(State)
         try:
             if not we_are_translated() and DEBUG_WRAPPER:
                 print >>sys.stderr, callable,
@@ -921,7 +923,6 @@
                 boxed_args += (arg_conv, )
             if pygilstate_ensure:
                 boxed_args += (args[-1], )
-            state = space.fromcache(State)
             try:
                 result = callable(space, *boxed_args)
                 if not we_are_translated() and DEBUG_WRAPPER:
@@ -967,6 +968,7 @@
         except Exception as e:
             unexpected_exception(pname, e, tb)
             _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
+            state.check_and_raise_exception(always=True)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
@@ -1127,6 +1129,34 @@
     setup_init_functions(eci, prefix)
     return modulename.new(ext='')
 
+def attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i):
+    # Start at i but make sure all the base classes are already attached
+    from pypy.module.cpyext.pyobject import get_typedescr, make_ref
+    if i in attached_objs:
+        return
+    py_obj = static_pyobjs[i]
+    w_obj = static_objs_w[i]
+    w_base = None
+    # w_obj can be NotImplemented, which is not a W_TypeObject
+    if isinstance(w_obj, W_TypeObject):
+        bases_w = w_obj.bases_w
+        if bases_w:
+            w_base = find_best_base(bases_w)
+        if w_base:
+            try:
+                j = static_objs_w.index(w_base)
+            except ValueError:
+                j = -1
+            if j >=0 and j not in attached_objs:
+                attach_recursively(space, static_pyobjs, static_objs_w,
+                                                 attached_objs, j)
+    w_type = space.type(w_obj)
+    typedescr = get_typedescr(w_type.layout.typedef)
+    py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
+                                 make_ref(space, w_type))
+    typedescr.attach(space, py_obj, w_obj)
+    attached_objs.append(i)
+
 
 class StaticObjectBuilder(object):
     def __init__(self):
@@ -1147,7 +1177,6 @@
 
     def attach_all(self, space):
         # this is RPython, called once in pypy-c when it imports cpyext
-        from pypy.module.cpyext.pyobject import get_typedescr, make_ref
         from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
         from pypy.module.cpyext.pyobject import track_reference
         #
@@ -1157,14 +1186,9 @@
             track_reference(space, static_pyobjs[i], static_objs_w[i])
         #
         self.cpyext_type_init = []
+        attached_objs = []
         for i in range(len(static_objs_w)):
-            py_obj = static_pyobjs[i]
-            w_obj = static_objs_w[i]
-            w_type = space.type(w_obj)
-            typedescr = get_typedescr(w_type.layout.typedef)
-            py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
-                                         make_ref(space, w_type))
-            typedescr.attach(space, py_obj, w_obj)
+            attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i)
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -80,14 +80,18 @@
     """
     py_str = rffi.cast(PyBytesObject, py_obj)
     s = space.str_w(w_obj)
-    if py_str.c_ob_size  < len(s):
+    len_s = len(s)
+    if py_str.c_ob_size  < len_s:
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
-            py_str.c_ob_size, len(s))
+            py_str.c_ob_size, len_s)
     with rffi.scoped_nonmovingbuffer(s) as s_ptr:
-        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
-    py_str.c_ob_sval[len(s)] = '\0'
-    py_str.c_ob_shash = space.hash_w(w_obj)
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len_s)
+    py_str.c_ob_sval[len_s] = '\0'
+    # if py_obj has a tp_hash, this will try to call it, but the objects are
+    # not fully linked yet
+    #py_str.c_ob_shash = space.hash_w(w_obj)
+    py_str.c_ob_shash = space.hash_w(space.newbytes(s))
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
 def bytes_realize(space, py_obj):
@@ -100,7 +104,9 @@
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(W_BytesObject, w_type)
     w_obj.__init__(s)
-    py_str.c_ob_shash = space.hash_w(w_obj)
+    # if py_obj has a tp_hash, this will try to call it but the object is
+    # not realized yet
+    py_str.c_ob_shash = space.hash_w(space.newbytes(s))
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
     return w_obj
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -90,11 +90,10 @@
     return 0
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
-def PyList_GET_SIZE(space, w_list):
+def PyList_GET_SIZE(space, w_obj):
     """Macro form of PyList_Size() without error checking.
     """
-    assert isinstance(w_list, W_ListObject)
-    return w_list.length()
+    return space.len_w(w_obj)
 
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -18,6 +18,7 @@
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
+from pypy.module.cpyext import userslot
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
 from rpython.rlib.buffer import Buffer
@@ -506,6 +507,10 @@
 
 @specialize.memo()
 def get_slot_tp_function(space, typedef, name):
+    """Return a description of the slot C function to use for the built-in
+    type for 'typedef'.  The 'name' is the slot name.  This is a memo
+    function that, after translation, returns one of a built-in finite set.
+    """
     key = (typedef, name)
     try:
         return SLOTS[key]
@@ -543,6 +548,19 @@
                 return space.call_function(slot_fn, w_self)
             handled = True
 
+    for tp_name, attr in [('tp_hash', '__hash__'),
+                         ]:
+        if name == tp_name:
+            slot_fn = w_type.getdictvalue(space, attr)
+            if slot_fn is None:
+                return
+            @slot_function([PyObject], lltype.Signed, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_obj):
+                return space.int_w(space.call_function(slot_fn, w_obj))
+            handled = True
+
+
     # binary functions
     for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'),
                           ('tp_as_number.c_nb_subtract', '__sub__'),
@@ -748,7 +766,7 @@
         else:
             assert False
 
-    function = globals().get(FUNCTION, None)
+    function = getattr(userslot, FUNCTION or '!missing', None)
     assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS
     if FLAGS:
         if wrapper is Ellipsis:
@@ -811,7 +829,7 @@
 static slotdef slotdefs[] = {
         SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc,
                "x.__len__() <==> len(x)"),
-        SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc,
+        SQSLOT("__add__", sq_concat, slot_sq_concat, wrap_binaryfunc,
           "x.__add__(y) <==> x+y"),
         SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc,
           "x.__mul__(n) <==> x*n"),
@@ -928,9 +946,7 @@
                "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS),
         TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook,
                wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"),
-        TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""),
-        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""),
-        TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""),
+        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""),
         TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr,
                "x.__setattr__('name', value) <==> x.name = value"),
         TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""),
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -8,6 +8,24 @@
     return newType;
 }
 
+PyObject * typ = NULL;
+PyObject* datetimetype_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
+{
+    PyObject* newType;
+    if (typ == NULL)
+    {
+        PyErr_Format(PyExc_TypeError, "could not import datetime.datetime");
+        return NULL;
+    }
+    newType = ((PyTypeObject*)typ)->tp_new(metatype, args, kwds);
+    return newType;
+}
+
+void datetimetype_tp_dealloc(PyObject* self)
+{
+    return ((PyTypeObject*)typ)->tp_dealloc(self);
+}
+
 #define BASEFLAGS Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
 
 PyTypeObject footype = {
@@ -58,6 +76,54 @@
     /*tp_weaklist*/         0
 };
 
+PyTypeObject datetimetype = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    /*tp_name*/             "foo3.datetimetype",
+    /*tp_basicsize*/        sizeof(PyTypeObject),
+    /*tp_itemsize*/         0,
+    /*tp_dealloc*/          datetimetype_tp_dealloc,
+    /*tp_print*/            0,
+    /*tp_getattr*/          0,
+    /*tp_setattr*/          0,
+    /*tp_compare*/          0,
+    /*tp_repr*/             0,
+    /*tp_as_number*/        0,
+    /*tp_as_sequence*/      0,
+    /*tp_as_mapping*/       0,
+    /*tp_hash*/             0,
+    /*tp_call*/             0,
+    /*tp_str*/              0,
+    /*tp_getattro*/         0,
+    /*tp_setattro*/         0,
+    /*tp_as_buffer*/        0,
+    /*tp_flags*/            BASEFLAGS,
+    /*tp_doc*/              0,
+    /*tp_traverse*/         0,
+    /*tp_clear*/            0,
+    /*tp_richcompare*/      0,
+    /*tp_weaklistoffset*/   0,
+    /*tp_iter*/             0,
+    /*tp_iternext*/         0,
+    /*tp_methods*/          0,
+    /*tp_members*/          0,
+    /*tp_getset*/           0,
+    /*tp_base*/             0,  //  set to &PyType_Type in module init function (why can it not be done here?)
+    /*tp_dict*/             0,
+    /*tp_descr_get*/        0,
+    /*tp_descr_set*/        0,
+    /*tp_dictoffset*/       0,
+    /*tp_init*/             0,
+    /*tp_alloc*/            0,
+    /*tp_new*/              datetimetype_tp_new,
+    /*tp_free*/             0,
+    /*tp_is_gc*/            0,
+    /*tp_bases*/            0,
+    /*tp_mro*/              0,
+    /*tp_cache*/            0,
+    /*tp_subclasses*/       0,
+    /*tp_weaklist*/         0
+};
+
 static PyMethodDef sbkMethods[] = {{NULL, NULL, 0, NULL}};
 
 static struct PyModuleDef moduledef = {
@@ -81,6 +147,16 @@
 PyInit_foo3(void)
 {
     PyObject *mod, *d;
+    PyObject *module = NULL;
+    module = PyImport_ImportModule("datetime");
+    typ = PyObject_GetAttrString(module, "datetime");
+    Py_DECREF(module);
+    if (!PyType_Check(typ)) {
+        PyErr_Format(PyExc_TypeError, "datetime.datetime is not a type object");
+        return NULL;
+    }
+    datetimetype.tp_base = (PyTypeObject*)typ;
+    PyType_Ready(&datetimetype);
     footype.tp_base = &PyType_Type;
     PyType_Ready(&footype);
     mod = PyModule_Create(&moduledef);
@@ -91,6 +167,8 @@
         return NULL;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return NULL;
+    if (PyDict_SetItemString(d, "datetimetype", (PyObject *)&datetimetype) < 0)
+        return NULL;
     Py_INCREF(&footype);
     return mod;
 }
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -1,83 +1,82 @@
 import py
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP, PyTypeObjectPtr
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.dictproxyobject import *
+from pypy.module.cpyext.dictobject import *
+from pypy.module.cpyext.pyobject import decref
 
 class TestDictObject(BaseApiTest):
-    def test_dict(self, space, api):
-        d = api.PyDict_New()
+    def test_dict(self, space):
+        d = PyDict_New(space)
         assert space.eq_w(d, space.newdict())
 
-        assert space.eq_w(api.PyDict_GetItem(space.wrap({"a": 72}),
+        assert space.eq_w(PyDict_GetItem(space, space.wrap({"a": 72}),
                                              space.wrap("a")),
                           space.wrap(72))
 
-        assert api.PyDict_SetItem(d, space.wrap("c"), space.wrap(42)) >= 0
+        PyDict_SetItem(space, d, space.wrap("c"), space.wrap(42))
         assert space.eq_w(space.getitem(d, space.wrap("c")),
                           space.wrap(42))
 
         space.setitem(d, space.wrap("name"), space.wrap(3))
-        assert space.eq_w(api.PyDict_GetItem(d, space.wrap("name")),
+        assert space.eq_w(PyDict_GetItem(space, d, space.wrap("name")),
                           space.wrap(3))
 
         space.delitem(d, space.wrap("name"))
-        assert not api.PyDict_GetItem(d, space.wrap("name"))
-        assert not api.PyErr_Occurred()
+        assert not PyDict_GetItem(space, d, space.wrap("name"))
 
         buf = rffi.str2charp("name")
-        assert not api.PyDict_GetItemString(d, buf)
+        assert not PyDict_GetItemString(space, d, buf)
         rffi.free_charp(buf)
-        assert not api.PyErr_Occurred()
 
-        assert api.PyDict_Contains(d, space.wrap("c"))
-        assert not api.PyDict_Contains(d, space.wrap("z"))
+        assert PyDict_Contains(space, d, space.wrap("c"))
+        assert not PyDict_Contains(space, d, space.wrap("z"))
 
-        assert api.PyDict_DelItem(d, space.wrap("c")) == 0
-        assert api.PyDict_DelItem(d, space.wrap("name")) < 0
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
-        assert api.PyDict_Size(d) == 0
+        PyDict_DelItem(space, d, space.wrap("c"))
+        with raises_w(space, KeyError):
+            PyDict_DelItem(space, d, space.wrap("name"))
+        assert PyDict_Size(space, d) == 0
 
         space.setitem(d, space.wrap("some_key"), space.wrap(3))
         buf = rffi.str2charp("some_key")
-        assert api.PyDict_DelItemString(d, buf) == 0
-        assert api.PyDict_Size(d) == 0
-        assert api.PyDict_DelItemString(d, buf) < 0
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
+        PyDict_DelItemString(space, d, buf)
+        assert PyDict_Size(space, d) == 0
+        with raises_w(space, KeyError):
+            PyDict_DelItemString(space, d, buf)
         rffi.free_charp(buf)
 
         d = space.wrap({'a': 'b'})
-        api.PyDict_Clear(d)
-        assert api.PyDict_Size(d) == 0
+        PyDict_Clear(space, d)
+        assert PyDict_Size(space, d) == 0
 
-    def test_check(self, space, api):
-        d = api.PyDict_New()
-        assert api.PyDict_Check(d)
-        assert api.PyDict_CheckExact(d)
+    def test_check(self, space):
+        d = PyDict_New(space, )
+        assert PyDict_Check(space, d)
+        assert PyDict_CheckExact(space, d)
         sub = space.appexec([], """():
             class D(dict):
                 pass
             return D""")
         d = space.call_function(sub)
-        assert api.PyDict_Check(d)
-        assert not api.PyDict_CheckExact(d)
+        assert PyDict_Check(space, d)
+        assert not PyDict_CheckExact(space, d)
         i = space.wrap(2)
-        assert not api.PyDict_Check(i)
-        assert not api.PyDict_CheckExact(i)
+        assert not PyDict_Check(space, i)
+        assert not PyDict_CheckExact(space, i)
 
-    def test_keys(self, space, api):
+    def test_keys(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
-        assert space.eq_w(api.PyDict_Keys(w_d), space.wrap(["a"]))
-        assert space.eq_w(api.PyDict_Values(w_d), space.wrap(["b"]))
-        assert space.eq_w(api.PyDict_Items(w_d), space.wrap([("a", "b")]))
+        assert space.eq_w(PyDict_Keys(space, w_d), space.wrap(["a"]))
+        assert space.eq_w(PyDict_Values(space, w_d), space.wrap(["b"]))
+        assert space.eq_w(PyDict_Items(space, w_d), space.wrap([("a", "b")]))
 
-    def test_merge(self, space, api):
+    def test_merge(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
@@ -86,35 +85,34 @@
         space.setitem(w_d2, space.wrap("c"), space.wrap("d"))
         space.setitem(w_d2, space.wrap("e"), space.wrap("f"))
 
-        api.PyDict_Merge(w_d, w_d2, 0)
+        PyDict_Merge(space, w_d, w_d2, 0)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
-        api.PyDict_Merge(w_d, w_d2, 1)
+        PyDict_Merge(space, w_d, w_d2, 1)
         assert space.unwrap(w_d) == dict(a='c', c='d', e='f')
 
-    def test_update(self, space, api):
+    def test_update(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
-        w_d2 = api.PyDict_Copy(w_d)
+        w_d2 = PyDict_Copy(space, w_d)
         assert not space.is_w(w_d2, w_d)
         space.setitem(w_d, space.wrap("c"), space.wrap("d"))
         space.setitem(w_d2, space.wrap("e"), space.wrap("f"))
 
-        api.PyDict_Update(w_d, w_d2)
+        PyDict_Update(space, w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
-    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+    def test_update_doesnt_accept_list_of_tuples(self, space):
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("a"), space.wrap("b"))
 
         w_d2 = space.wrap([("c", "d"), ("e", "f")])
 
-        api.PyDict_Update(w_d, w_d2)
-        assert api.PyErr_Occurred() is space.w_AttributeError
-        api.PyErr_Clear()
+        with raises_w(space, AttributeError):
+            PyDict_Update(space, w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b') # unchanged
 
-    def test_iter(self, space, api):
+    def test_iter(self, space):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
 
@@ -125,7 +123,7 @@
 
         try:
             w_copy = space.newdict()
-            while api.PyDict_Next(w_dict, ppos, pkey, pvalue):
+            while PyDict_Next(space, w_dict, ppos, pkey, pvalue):
                 w_key = from_ref(space, pkey[0])
                 w_value = from_ref(space, pvalue[0])
                 space.setitem(w_copy, w_key, w_value)
@@ -134,12 +132,12 @@
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
 
-        api.Py_DecRef(py_dict) # release borrowed references
+        decref(space, py_dict) # release borrowed references
 
         assert space.eq_w(space.len(w_copy), space.len(w_dict))
         assert space.eq_w(w_copy, w_dict)
 
-    def test_iterkeys(self, space, api):
+    def test_iterkeys(self, space):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
 
@@ -151,11 +149,11 @@
         values_w = []
         try:
             ppos[0] = 0
-            while api.PyDict_Next(w_dict, ppos, pkey, None):
+            while PyDict_Next(space, w_dict, ppos, pkey, None):
                 w_key = from_ref(space, pkey[0])
                 keys_w.append(w_key)
             ppos[0] = 0
-            while api.PyDict_Next(w_dict, ppos, None, pvalue):
+            while PyDict_Next(space, w_dict, ppos, None, pvalue):
                 w_value = from_ref(space, pvalue[0])
                 values_w.append(w_value)
         finally:
@@ -163,7 +161,7 @@
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
 
-        api.Py_DecRef(py_dict) # release borrowed references
+        decref(space, py_dict) # release borrowed references
 
         assert space.eq_w(space.newlist(keys_w),
                           space.call_function(
@@ -174,18 +172,18 @@
                              space.w_list,
                              space.call_method(w_dict, "values")))
 
-    def test_dictproxy(self, space, api):
+    def test_dictproxy(self, space):
         w_dict = space.sys.get('modules')
-        w_proxy = api.PyDictProxy_New(w_dict)
+        w_proxy = PyDictProxy_New(space, w_dict)
         assert space.contains_w(w_proxy, space.wrap('sys'))
         raises(OperationError, space.setitem,
                w_proxy, space.wrap('sys'), space.w_None)
         raises(OperationError, space.delitem,
                w_proxy, space.wrap('sys'))
         raises(OperationError, space.call_method, w_proxy, 'clear')
-        assert api.PyDictProxy_Check(w_proxy)
+        assert PyDictProxy_Check(space, w_proxy)
 
-    def test_typedict1(self, space, api):
+    def test_typedict1(self, space):
         py_type = make_ref(space, space.w_int)
         py_dict = rffi.cast(PyTypeObjectPtr, py_type).c_tp_dict
         ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
@@ -195,7 +193,7 @@
         pvalue = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         try:
             w_copy = space.newdict()
-            while api.PyDict_Next(py_dict, ppos, pkey, pvalue):
+            while PyDict_Next(space, py_dict, ppos, pkey, pvalue):
                 w_key = from_ref(space, pkey[0])
                 w_value = from_ref(space, pvalue[0])
                 space.setitem(w_copy, w_key, w_value)
@@ -203,7 +201,7 @@
             lltype.free(ppos, flavor='raw')
             lltype.free(pkey, flavor='raw')
             lltype.free(pvalue, flavor='raw')
-        api.Py_DecRef(py_type) # release borrowed references
+        decref(space, py_type) # release borrowed references
         # do something with w_copy ?
 
 class AppTestDictObject(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -1,17 +1,26 @@
+import sys
+import os
+import pytest
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.object import PyObject_Size, PyObject_GetItem
+from pypy.module.cpyext.pythonrun import Py_AtExit
 from pypy.module.cpyext.eval import (
-    Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import (c_fopen, c_fclose, c_fileno, 
-                Py_ssize_tP, is_valid_fd)
+    Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags,
+    PyEval_CallObjectWithKeywords, PyObject_CallObject, PyEval_EvalCode,
+    PyRun_SimpleString, PyRun_String, PyRun_StringFlags, PyRun_File,
+    PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals,
+    _PyEval_SliceIndex)
+from pypy.module.cpyext.api import (
+    c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd)
 from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
-import sys, os
 
 class TestEval(BaseApiTest):
-    def test_eval(self, space, api):
+    def test_eval(self, space):
         w_l, w_f = space.fixedview(space.appexec([], """():
         l = []
         def f(arg1, arg2):
@@ -22,7 +31,7 @@
         """))
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, None)
+        w_res = PyEval_CallObjectWithKeywords(space, w_f, w_t, None)
         assert space.int_w(w_res) == 2
         assert space.len_w(w_l) == 2
         w_f = space.appexec([], """():
@@ -35,10 +44,10 @@
         w_t = space.newtuple([space.w_None, space.w_None])
         w_d = space.newdict()
         space.setitem(w_d, space.wrap("xyz"), space.wrap(3))
-        w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, w_d)
+        w_res = PyEval_CallObjectWithKeywords(space, w_f, w_t, w_d)
         assert space.int_w(w_res) == 21
 
-    def test_call_object(self, space, api):
+    def test_call_object(self, space):
         w_l, w_f = space.fixedview(space.appexec([], """():
         l = []
         def f(arg1, arg2):
@@ -49,7 +58,7 @@
         """))
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyObject_CallObject(w_f, w_t)
+        w_res = PyObject_CallObject(space, w_f, w_t)
         assert space.int_w(w_res) == 2
         assert space.len_w(w_l) == 2
 
@@ -61,11 +70,11 @@
             """)
 
         w_t = space.newtuple([space.wrap(1), space.wrap(2)])
-        w_res = api.PyObject_CallObject(w_f, w_t)
+        w_res = PyObject_CallObject(space, w_f, w_t)
 
         assert space.int_w(w_res) == 10
 
-    def test_evalcode(self, space, api):
+    def test_evalcode(self, space):
         w_f = space.appexec([], """():
             def f(*args):
                 assert isinstance(args, tuple)
@@ -77,84 +86,78 @@
         w_globals = space.newdict()
         w_locals = space.newdict()
         space.setitem(w_locals, space.wrap("args"), w_t)
-        w_res = api.PyEval_EvalCode(w_f.code, w_globals, w_locals)
+        w_res = PyEval_EvalCode(space, w_f.code, w_globals, w_locals)
 
         assert space.int_w(w_res) == 10
 
-    def test_run_simple_string(self, space, api):
+    def test_run_simple_string(self, space):
         def run(code):
             buf = rffi.str2charp(code)
             try:
-                return api.PyRun_SimpleString(buf)
+                return PyRun_SimpleString(space, buf)
             finally:
                 rffi.free_charp(buf)
 
-        assert 0 == run("42 * 43")
+        assert run("42 * 43") == 0  # no error
+        with pytest.raises(OperationError):
+            run("4..3 * 43")
 
-        assert -1 == run("4..3 * 43")
-
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
-
-    def test_run_string(self, space, api):
+    def test_run_string(self, space):
         def run(code, start, w_globals, w_locals):
             buf = rffi.str2charp(code)
             try:
-                return api.PyRun_String(buf, start, w_globals, w_locals)
+                return PyRun_String(space, buf, start, w_globals, w_locals)
             finally:
                 rffi.free_charp(buf)
 
         w_globals = space.newdict()
         assert 42 * 43 == space.unwrap(
             run("42 * 43", Py_eval_input, w_globals, w_globals))
-        assert api.PyObject_Size(w_globals) == 0
+        assert PyObject_Size(space, w_globals) == 0
 
         assert run("a = 42 * 43", Py_single_input,
                    w_globals, w_globals) == space.w_None
         assert 42 * 43 == space.unwrap(
-            api.PyObject_GetItem(w_globals, space.wrap("a")))
+            PyObject_GetItem(space, w_globals, space.wrap("a")))
 
-    def test_run_string_flags(self, space, api):
+    def test_run_string_flags(self, space):
         flags = lltype.malloc(PyCompilerFlags, flavor='raw')
         flags.c_cf_flags = rffi.cast(rffi.INT, consts.PyCF_SOURCE_IS_UTF8)
         w_globals = space.newdict()
         buf = rffi.str2charp("a = 'caf\xc3\xa9'")
         try:
-            api.PyRun_StringFlags(buf, Py_single_input,
-                                  w_globals, w_globals, flags)
+            PyRun_StringFlags(space, buf, Py_single_input, w_globals,
+                              w_globals, flags)
         finally:
             rffi.free_charp(buf)
         w_a = space.getitem(w_globals, space.wrap("a"))
         assert space.unwrap(w_a) == u'caf\xe9'
         lltype.free(flags, flavor='raw')
 
-    def test_run_file(self, space, api):
+    def test_run_file(self, space):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
         fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
-        api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
+        with raises_w(space, ZeroDivisionError):
+            PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
         c_fclose(fp)
-        assert api.PyErr_Occurred() is space.w_ZeroDivisionError
-        api.PyErr_Clear()
 
         # try again, but with a closed file
         fp = c_fopen(str(filepath), "rb")
         os.close(c_fileno(fp))
-        api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
+        with raises_w(space, IOError):
+            PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
         if is_valid_fd(c_fileno(fp)):
             c_fclose(fp)
-        assert api.PyErr_Occurred() is space.w_IOError
-        api.PyErr_Clear()
-
         rffi.free_charp(filename)
 
-    def test_getbuiltins(self, space, api):
-        assert api.PyEval_GetBuiltins() is space.builtin.w_dict
+    def test_getbuiltins(self, space):
+        assert PyEval_GetBuiltins(space) is space.builtin.w_dict
 
         def cpybuiltins(space):
-            return api.PyEval_GetBuiltins()
+            return PyEval_GetBuiltins(space)
         w_cpybuiltins = space.wrap(interp2app(cpybuiltins))
 
         w_result = space.appexec([w_cpybuiltins], """(cpybuiltins):
@@ -168,13 +171,13 @@
         """)
         assert space.len_w(w_result) == 1
 
-    def test_getglobals(self, space, api):
-        assert api.PyEval_GetLocals() is None
-        assert api.PyEval_GetGlobals() is None
+    def test_getglobals(self, space):
+        assert PyEval_GetLocals(space) is None
+        assert PyEval_GetGlobals(space) is None
 
         def cpyvars(space):
-            return space.newtuple([api.PyEval_GetGlobals(),
-                                   api.PyEval_GetLocals()])
+            return space.newtuple([PyEval_GetGlobals(space),
+                                   PyEval_GetLocals(space)])
         w_cpyvars = space.wrap(interp2app(cpyvars))
 
         w_result = space.appexec([w_cpyvars], """(cpyvars):
@@ -186,26 +189,26 @@
         assert sorted(locals) == ['cpyvars', 'x']
         assert sorted(globals) == ['__builtins__', 'anonymous', 'y']
 
-    def test_sliceindex(self, space, api):
+    def test_sliceindex(self, space):
         pi = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
-        assert api._PyEval_SliceIndex(space.w_None, pi) == 0
-        api.PyErr_Clear()
+        with pytest.raises(OperationError):
+            _PyEval_SliceIndex(space, space.w_None, pi)
 
-        assert api._PyEval_SliceIndex(space.wrap(123), pi) == 1
+        assert _PyEval_SliceIndex(space, space.wrap(123), pi) == 1
         assert pi[0] == 123
 
-        assert api._PyEval_SliceIndex(space.wrap(1 << 66), pi) == 1
+        assert _PyEval_SliceIndex(space, space.wrap(1 << 66), pi) == 1
         assert pi[0] == sys.maxint
 
         lltype.free(pi, flavor='raw')
 
-    def test_atexit(self, space, api):
+    def test_atexit(self, space):
         lst = []
         def func():
             lst.append(42)
-        api.Py_AtExit(func)
+        Py_AtExit(space, func)
         cpyext = space.getbuiltinmodule('cpyext')
-        cpyext.shutdown(space) # simulate shutdown
+        cpyext.shutdown(space)  # simulate shutdown
         assert lst == [42]
 
 class AppTestCall(AppTestCpythonExtensionBase):
@@ -269,6 +272,7 @@
                 return res;
              """),
             ])
+
         def f(*args):
             return args
         assert module.call_func(f) == (None,)
@@ -323,7 +327,7 @@
             ])
         assert module.get_flags() == (0, 0)
 
-        ns = {'module':module}
+        ns = {'module': module}
         if not hasattr(sys, 'pypy_version_info'):  # no barry_as_FLUFL on pypy
             exec("""from __future__ import barry_as_FLUFL    \nif 1:
                     def nested_flags():
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ b/pypy/module/cpyext/test/test_floatobject.py
@@ -1,36 +1,40 @@
+import pytest
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.floatobject import (
+    PyFloat_FromDouble, PyFloat_AsDouble, PyFloat_AS_DOUBLE, PyNumber_Float,
+    _PyFloat_Unpack4, _PyFloat_Unpack8)
 
 class TestFloatObject(BaseApiTest):
-    def test_floatobject(self, space, api):
-        assert space.unwrap(api.PyFloat_FromDouble(3.14)) == 3.14
-        assert api.PyFloat_AsDouble(space.wrap(23.45)) == 23.45
-        assert api.PyFloat_AS_DOUBLE(space.wrap(23.45)) == 23.45
+    def test_floatobject(self, space):
+        assert space.unwrap(PyFloat_FromDouble(space, 3.14)) == 3.14
+        assert PyFloat_AsDouble(space, space.wrap(23.45)) == 23.45
+        assert PyFloat_AS_DOUBLE(space, space.wrap(23.45)) == 23.45
+        with pytest.raises(OperationError):
+            PyFloat_AsDouble(space, space.w_None)
 
-        assert api.PyFloat_AsDouble(space.w_None) == -1
-        api.PyErr_Clear()
-
-    def test_coerce(self, space, api):
-        assert space.type(api.PyNumber_Float(space.wrap(3))) is space.w_float
-        assert space.type(api.PyNumber_Float(space.wrap("3"))) is space.w_float
+    def test_coerce(self, space):
+        assert space.type(PyNumber_Float(space, space.wrap(3))) is space.w_float
+        assert space.type(PyNumber_Float(space, space.wrap("3"))) is space.w_float
 
         w_obj = space.appexec([], """():
             class Coerce(object):
                 def __float__(self):
                     return 42.5
             return Coerce()""")
-        assert space.eq_w(api.PyNumber_Float(w_obj), space.wrap(42.5))
+        assert space.eq_w(PyNumber_Float(space, w_obj), space.wrap(42.5))
 
-    def test_unpack(self, space, api):
+    def test_unpack(self, space):
         with rffi.scoped_str2charp("\x9a\x99\x99?") as ptr:
-            assert abs(api._PyFloat_Unpack4(ptr, 1) - 1.2) < 1e-7
+            assert abs(_PyFloat_Unpack4(space, ptr, 1) - 1.2) < 1e-7
         with rffi.scoped_str2charp("?\x99\x99\x9a") as ptr:
-            assert abs(api._PyFloat_Unpack4(ptr, 0) - 1.2) < 1e-7
+            assert abs(_PyFloat_Unpack4(space, ptr, 0) - 1.2) < 1e-7
         with rffi.scoped_str2charp("\x1f\x85\xebQ\xb8\x1e\t@") as ptr:
-            assert abs(api._PyFloat_Unpack8(ptr, 1) - 3.14) < 1e-15
+            assert abs(_PyFloat_Unpack8(space, ptr, 1) - 3.14) < 1e-15
         with rffi.scoped_str2charp("@\t\x1e\xb8Q\xeb\x85\x1f") as ptr:
-            assert abs(api._PyFloat_Unpack8(ptr, 0) - 3.14) < 1e-15
+            assert abs(_PyFloat_Unpack8(space, ptr, 0) - 3.14) < 1e-15
 
 class AppTestFloatObject(AppTestCpythonExtensionBase):
     def test_fromstring(self):
@@ -79,8 +83,6 @@
         assert math.isinf(neginf)
 
     def test_macro_accepts_wrong_pointer_type(self):
-        import math
-
         module = self.import_extension('foo', [
             ("test_macros", "METH_NOARGS",
              """
diff --git a/pypy/module/cpyext/test/test_funcobject.py b/pypy/module/cpyext/test/test_funcobject.py
--- a/pypy/module/cpyext/test/test_funcobject.py
+++ b/pypy/module/cpyext/test/test_funcobject.py
@@ -1,16 +1,18 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
+from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, decref
+from pypy.module.cpyext.methodobject import PyClassMethod_New
 from pypy.module.cpyext.funcobject import (
-    PyFunctionObject, PyCodeObject, CODE_FLAGS)
-from pypy.interpreter.function import Function, Method
+    PyFunctionObject, PyCodeObject, CODE_FLAGS, PyMethod_Function,
+    PyMethod_Self, PyMethod_New, PyFunction_GetCode,
+    PyCode_NewEmpty, PyCode_GetNumFree)
+from pypy.interpreter.function import Function
 from pypy.interpreter.pycode import PyCode
 
 globals().update(CODE_FLAGS)
 
 class TestFunctionObject(BaseApiTest):
-    def test_function(self, space, api):
+    def test_function(self, space):
         w_function = space.appexec([], """():
             def f(): pass
             return f
@@ -19,10 +21,10 @@
         assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is
                 space.gettypeobject(Function.typedef))
         assert "f" == space.unwrap(
-           from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name))
-        api.Py_DecRef(ref)
+            from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name))
+        decref(space, ref)
 
-    def test_method(self, space, api):
+    def test_method(self, space):
         w_method = space.appexec([], """():
             class C(list):
                 def method(self): pass
@@ -32,29 +34,29 @@
         w_function = space.getattr(w_method, space.wrap("__func__"))
         w_self = space.getattr(w_method, space.wrap("__self__"))
 
-        assert space.is_w(api.PyMethod_Function(w_method), w_function)
-        assert space.is_w(api.PyMethod_Self(w_method), w_self)
+        assert space.is_w(PyMethod_Function(space, w_method), w_function)
+        assert space.is_w(PyMethod_Self(space, w_method), w_self)
 
-        w_method2 = api.PyMethod_New(w_function, w_self)
+        w_method2 = PyMethod_New(space, w_function, w_self)
         assert space.eq_w(w_method, w_method2)
 
-    def test_getcode(self, space, api):
+    def test_getcode(self, space):
         w_function = space.appexec([], """():
             def func(x, y, z): return x
             return func
         """)
-        w_code = api.PyFunction_GetCode(w_function)
+        w_code = PyFunction_GetCode(space, w_function)
         assert w_code.co_name == "func"
 
         ref = make_ref(space, w_code)
         assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is
                 space.gettypeobject(PyCode.typedef))
         assert "func" == space.unwrap(
-           from_ref(space, rffi.cast(PyCodeObject, ref).c_co_name))
+            from_ref(space, rffi.cast(PyCodeObject, ref).c_co_name))
         assert 3 == rffi.cast(PyCodeObject, ref).c_co_argcount
-        api.Py_DecRef(ref)
+        decref(space, ref)
 
-    def test_co_flags(self, space, api):
+    def test_co_flags(self, space):
         def get_flags(signature, body="pass"):
             w_code = space.appexec([], """():
                 def func(%s): %s
@@ -62,36 +64,36 @@
             """ % (signature, body))
             ref = make_ref(space, w_code)
             co_flags = rffi.cast(PyCodeObject, ref).c_co_flags
-            api.Py_DecRef(ref)
+            decref(space, ref)
             return co_flags
         assert get_flags("x") == CO_NESTED | CO_OPTIMIZED | CO_NEWLOCALS
         assert get_flags("x, *args") & CO_VARARGS
         assert get_flags("x, **kw") & CO_VARKEYWORDS
         assert get_flags("x", "yield x") & CO_GENERATOR
 
-    def test_newcode(self, space, api):
+    def test_newcode(self, space):
         filename = rffi.str2charp('filename')
         funcname = rffi.str2charp('funcname')
-        w_code = api.PyCode_NewEmpty(filename, funcname, 3)
+        w_code = PyCode_NewEmpty(space, filename, funcname, 3)
         assert w_code.co_filename == 'filename'
         assert w_code.co_firstlineno == 3
 
         ref = make_ref(space, w_code)
         assert "filename" == space.unwrap(
             from_ref(space, rffi.cast(PyCodeObject, ref).c_co_filename))
-        api.Py_DecRef(ref)
+        decref(space, ref)
         rffi.free_charp(filename)
         rffi.free_charp(funcname)
 
-    def test_getnumfree(self, space, api):
+    def test_getnumfree(self, space):
         w_function = space.appexec([], """():
             a = 5
             def method(x): return a, x
             return method
         """)
-        assert api.PyCode_GetNumFree(w_function.code) == 1
+        assert PyCode_GetNumFree(space, w_function.code) == 1
 
-    def test_classmethod(self, space, api):
+    def test_classmethod(self, space):
         w_function = space.appexec([], """():
             def method(x): return x
             return method
@@ -103,6 +105,7 @@
         space.setattr(w_class, space.wrap("method"), w_function)
         assert space.is_w(space.call_method(w_instance, "method"), w_instance)
         # now a classmethod
-        w_classmethod = api.PyClassMethod_New(w_function)
+        w_classmethod = PyClassMethod_New(space, w_function)
         space.setattr(w_class, space.wrap("classmethod"), w_classmethod)
-        assert space.is_w(space.call_method(w_instance, "classmethod"), w_class)
+        assert space.is_w(
+            space.call_method(w_instance, "classmethod"), w_class)
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -1,48 +1,51 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.import_ import *
+from pypy.module.cpyext.import_ import (
+    _PyImport_AcquireLock, _PyImport_ReleaseLock)
 from rpython.rtyper.lltypesystem import rffi
 
 class TestImport(BaseApiTest):
-    def test_import(self, space, api):
-        stat = api.PyImport_Import(space.wrap("stat"))
+    def test_import(self, space):
+        stat = PyImport_Import(space, space.wrap("stat"))
         assert stat
         assert space.getattr(stat, space.wrap("S_IMODE"))
 
-    def test_addmodule(self, space, api):
+    def test_addmodule(self, space):
         with rffi.scoped_str2charp("sys") as modname:
-            w_sys = api.PyImport_AddModule(modname)
+            w_sys = PyImport_AddModule(space, modname)
         assert w_sys is space.sys
 
         with rffi.scoped_str2charp("foobar") as modname:
-            w_foobar = api.PyImport_AddModule(modname)
+            w_foobar = PyImport_AddModule(space, modname)
         assert space.str_w(space.getattr(w_foobar,
                                          space.wrap('__name__'))) == 'foobar'
 
     def test_getmoduledict(self, space, api):
         testmod = "contextlib"
-        w_pre_dict = api.PyImport_GetModuleDict()
+        w_pre_dict = PyImport_GetModuleDict(space, )
         assert not space.contains_w(w_pre_dict, space.wrap(testmod))
 
         with rffi.scoped_str2charp(testmod) as modname:
-            w_module = api.PyImport_ImportModule(modname)
+            w_module = PyImport_ImportModule(space, modname)
             print w_module
             assert w_module
 
-        w_dict = api.PyImport_GetModuleDict()
+        w_dict = PyImport_GetModuleDict(space, )
         assert space.contains_w(w_dict, space.wrap(testmod))
 
-    def test_reload(self, space, api):
-        stat = api.PyImport_Import(space.wrap("stat"))
+    def test_reload(self, space):
+        stat = PyImport_Import(space, space.wrap("stat"))
         space.delattr(stat, space.wrap("S_IMODE"))
-        stat = api.PyImport_ReloadModule(stat)
+        stat = PyImport_ReloadModule(space, stat)
         assert space.getattr(stat, space.wrap("S_IMODE"))
 
-    def test_lock(self, space, api):
+    def test_lock(self, space):
         # "does not crash"
-        api._PyImport_AcquireLock()
-        api._PyImport_AcquireLock()
-        api._PyImport_ReleaseLock()
-        api._PyImport_ReleaseLock()
+        _PyImport_AcquireLock(space, )
+        _PyImport_AcquireLock(space, )
+        _PyImport_ReleaseLock(space, )
+        _PyImport_ReleaseLock(space, )
 
 
 class AppTestImportLogic(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -30,12 +30,14 @@
         assert view.c_len == 5
         o = rffi.charp2str(view.c_buf)
         assert o == 'hello'
-        w_mv = from_ref(space, api.PyMemoryView_FromBuffer(view))
+        ref = api.PyMemoryView_FromBuffer(view)
+        w_mv = from_ref(space, ref)
         for f in ('format', 'itemsize', 'ndim', 'readonly',
                   'shape', 'strides', 'suboffsets'):
             w_f = space.wrap(f)
             assert space.eq_w(space.getattr(w_mv, w_f),
                               space.getattr(w_memoryview, w_f))
+        api.Py_DecRef(ref)
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     def test_fillWithObject(self):
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -90,7 +90,7 @@
         api.PyErr_WriteUnraisable(w_where)
         space.call_method(space.sys.get('stderr'), "flush")
         out, err = capfd.readouterr()
-        assert "Exception ValueError: 'message' in 'location' ignored" == err.strip()
+        assert "Exception ignored in: 'location'\nValueError: message" == err.strip()
 
     @pytest.mark.skipif(True, reason='not implemented yet')
     def test_interrupt_occurred(self, space, api):
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.api import generic_cpy_call
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 
@@ -634,26 +635,6 @@
         assert module.tp_init(list, x, ("hi",)) is None
         assert x == ["h", "i"]
 
-    def test_tp_str(self):
-        module = self.import_extension('foo', [
-           ("tp_str", "METH_VARARGS",
-            '''
-                 PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0);
-                 PyObject *obj = PyTuple_GET_ITEM(args, 1);
-                 if (!type->tp_str)
-                 {
-                     PyErr_SetNone(PyExc_ValueError);
-                     return NULL;
-                 }
-                 return type->tp_str(obj);
-             '''
-             )
-            ])
-        class D(int):
-            def __str__(self):
-                return "more text"
-        assert module.tp_str(int, D(42)) == "42"
-
     def test_mp_ass_subscript(self):
         module = self.import_extension('foo', [
            ("new_obj", "METH_NOARGS",
@@ -942,9 +923,12 @@
         assert (d + a) == 5
         assert pow(d,b) == 16
 
-    def test_tp_new_in_subclass_of_type(self):
+    def test_tp_new_in_subclass(self):
+        import datetime
         module = self.import_module(name='foo3')
         module.footype("X", (object,), {})
+        a = module.datetimetype(1, 1, 1)
+        assert isinstance(a, module.datetimetype)
 
     def test_app_subclass_of_c_type(self):
         import sys
@@ -1130,7 +1114,6 @@
         # class X(object, metaclass=FooType): pass
         X = FooType('X', (object,), {})
 
-        print(repr(X))
         X()
 
     def test_multiple_inheritance3(self):
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -1,5 +1,6 @@
 # encoding: utf-8
-from pypy.module.cpyext.test.test_api import BaseApiTest
+import pytest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.unicodeobject import (
     Py_UNICODE, PyUnicodeObject, new_empty_unicode)
@@ -7,6 +8,7 @@
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref
 from rpython.rtyper.lltypesystem import rffi, lltype
 import sys, py
+from pypy.module.cpyext.unicodeobject import *
 
 class AppTestUnicodeObject(AppTestCpythonExtensionBase):
     def test_unicodeobject(self):
@@ -209,44 +211,45 @@
         assert module.test_macro_invocations() == u''
 
 class TestUnicode(BaseApiTest):
-    def test_unicodeobject(self, space, api):
-        assert api.PyUnicode_GET_SIZE(space.wrap(u'sp&#228;m')) == 4
-        assert api.PyUnicode_GetSize(space.wrap(u'sp&#228;m')) == 4
+    def test_unicodeobject(self, space):
+        assert PyUnicode_GET_SIZE(space, space.wrap(u'sp&#228;m')) == 4
+        assert PyUnicode_GetSize(space, space.wrap(u'sp&#228;m')) == 4
         unichar = rffi.sizeof(Py_UNICODE)
-        assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp&#228;m')) == 4 * unichar
+        assert PyUnicode_GET_DATA_SIZE(space, space.wrap(u'sp&#228;m')) == 4 * unichar
 
-        encoding = rffi.charp2str(api.PyUnicode_GetDefaultEncoding())
+        encoding = rffi.charp2str(PyUnicode_GetDefaultEncoding(space, ))
         w_default_encoding = space.call_function(
             space.sys.get('getdefaultencoding')
         )
         assert encoding == space.unwrap(w_default_encoding)
 
-    def test_AS(self, space, api):
+    def test_AS(self, space):
         word = space.wrap(u'spam')
-        array = rffi.cast(rffi.CWCHARP, api.PyUnicode_AS_DATA(word))
-        array2 = api.PyUnicode_AS_UNICODE(word)
-        array3 = api.PyUnicode_AsUnicode(word)
+        array = rffi.cast(rffi.CWCHARP, PyUnicode_AS_DATA(space, word))
+        array2 = PyUnicode_AS_UNICODE(space, word)
+        array3 = PyUnicode_AsUnicode(space, word)
         for (i, char) in enumerate(space.unwrap(word)):
             assert array[i] == char
             assert array2[i] == char
             assert array3[i] == char
-        self.raises(space, api, TypeError, api.PyUnicode_AsUnicode,
-                    space.newbytes('spam'))
+        with raises_w(space, TypeError):
+            PyUnicode_AsUnicode(space, space.newbytes('spam'))
 
         utf_8 = rffi.str2charp('utf-8')
-        encoded = api.PyUnicode_AsEncodedString(space.wrap(u'sp&#228;m'),
+        encoded = PyUnicode_AsEncodedString(space, space.wrap(u'sp&#228;m'),
                                                 utf_8, None)
         assert space.unwrap(encoded) == 'sp\xc3\xa4m'
-        encoded_obj = api.PyUnicode_AsEncodedObject(space.wrap(u'sp&#228;m'),
+        encoded_obj = PyUnicode_AsEncodedObject(space, space.wrap(u'sp&#228;m'),
                                                 utf_8, None)
         assert space.eq_w(encoded, encoded_obj)
-        self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString,
-               space.newtuple([1, 2, 3]), None, None)
-        self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString,
-               space.newbytes(''), None, None)
+        with raises_w(space, TypeError):
+            PyUnicode_AsEncodedString(
+               space, space.newtuple([1, 2, 3]), None, None)
+        with raises_w(space, TypeError):
+            PyUnicode_AsEncodedString(space, space.newbytes(''), None, None)
         ascii = rffi.str2charp('ascii')
         replace = rffi.str2charp('replace')
-        encoded = api.PyUnicode_AsEncodedString(space.wrap(u'sp&#228;m'),
+        encoded = PyUnicode_AsEncodedString(space, space.wrap(u'sp&#228;m'),
                                                 ascii, replace)
         assert space.unwrap(encoded) == 'sp?m'
         rffi.free_charp(utf_8)
@@ -254,45 +257,45 @@
         rffi.free_charp(ascii)
 
         buf = rffi.unicode2wcharp(u"12345")
-        api.PyUnicode_AsWideChar(space.wrap(u'longword'), buf, 5)
+        PyUnicode_AsWideChar(space, space.wrap(u'longword'), buf, 5)
         assert rffi.wcharp2unicode(buf) == 'longw'
-        api.PyUnicode_AsWideChar(space.wrap(u'a'), buf, 5)
+        PyUnicode_AsWideChar(space, space.wrap(u'a'), buf, 5)
         assert rffi.wcharp2unicode(buf) == 'a'
         rffi.free_wcharp(buf)
 
-    def test_fromstring(self, space, api):
+    def test_fromstring(self, space):
         s = rffi.str2charp(u'sp\x09m'.encode("utf-8"))
-        w_res = api.PyUnicode_FromString(s)
+        w_res = PyUnicode_FromString(space, s)
         assert space.unwrap(w_res) == u'sp\x09m'
 
-        res = api.PyUnicode_FromStringAndSize(s, 4)
+        res = PyUnicode_FromStringAndSize(space, s, 4)
         w_res = from_ref(space, res)
-        api.Py_DecRef(res)
+        Py_DecRef(space, res)
         assert space.unwrap(w_res) == u'sp\x09m'
         rffi.free_charp(s)
 
-    def test_internfromstring(self, space, api):
+    def test_internfromstring(self, space):
         with rffi.scoped_str2charp('foo') as s:
-            w_res = api.PyUnicode_InternFromString(s)
+            w_res = PyUnicode_InternFromString(space, s)
             assert space.unwrap(w_res) == u'foo'
-            w_res2 = api.PyUnicode_InternFromString(s)
+            w_res2 = PyUnicode_InternFromString(space, s)
             assert w_res is w_res2
 
-    def test_unicode_resize(self, space, api):
+    def test_unicode_resize(self, space):
         py_uni = new_empty_unicode(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         py_uni.c_buffer[0] = u'a'
         py_uni.c_buffer[1] = u'b'
         py_uni.c_buffer[2] = u'c'
         ar[0] = rffi.cast(PyObject, py_uni)
-        api.PyUnicode_Resize(ar, 3)
+        PyUnicode_Resize(space, ar, 3)
         py_uni = rffi.cast(PyUnicodeObject, ar[0])
         assert py_uni.c_length == 3
         assert py_uni.c_buffer[1] == u'b'
         assert py_uni.c_buffer[3] == u'\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_uni)
-        api.PyUnicode_Resize(ar, 10)
+        PyUnicode_Resize(space, ar, 10)
         py_uni = rffi.cast(PyUnicodeObject, ar[0])
         assert py_uni.c_length == 10
         assert py_uni.c_buffer[1] == 'b'
@@ -300,47 +303,46 @@
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
-    def test_AsUTF8String(self, space, api):
+    def test_AsUTF8String(self, space):
         w_u = space.wrap(u'sp\x09m')
-        w_res = api.PyUnicode_AsUTF8String(w_u)
+        w_res = PyUnicode_AsUTF8String(space, w_u)
         assert space.type(w_res) is space.w_str
         assert space.unwrap(w_res) == 'sp\tm'
 
-    def test_decode_utf8(self, space, api):
+    def test_decode_utf8(self, space):
         u = rffi.str2charp(u'sp\x134m'.encode("utf-8"))
-        w_u = api.PyUnicode_DecodeUTF8(u, 5, None)
+        w_u = PyUnicode_DecodeUTF8(space, u, 5, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == u'sp\x134m'
 
-        w_u = api.PyUnicode_DecodeUTF8(u, 2, None)
+        w_u = PyUnicode_DecodeUTF8(space, u, 2, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == 'sp'
         rffi.free_charp(u)
 
-    def test_encode_utf8(self, space, api):
+    def test_encode_utf8(self, space):
         u = rffi.unicode2wcharp(u'sp\x09m')
-        w_s = api.PyUnicode_EncodeUTF8(u, 4, None)
+        w_s = PyUnicode_EncodeUTF8(space, u, 4, None)
         assert space.unwrap(w_s) == u'sp\x09m'.encode('utf-8')
         rffi.free_wcharp(u)
 
-    def test_encode_decimal(self, space, api):
+    def test_encode_decimal(self, space):
         with rffi.scoped_unicode2wcharp(u' (12, 35 ABC)') as u:
             with rffi.scoped_alloc_buffer(20) as buf:
-                res = api.PyUnicode_EncodeDecimal(u, 13, buf.raw, None)
+                res = PyUnicode_EncodeDecimal(space, u, 13, buf.raw, None)
                 s = rffi.charp2str(buf.raw)
         assert res == 0
         assert s == ' (12, 35 ABC)'
 
         with rffi.scoped_unicode2wcharp(u' (12, \u1234\u1235)') as u:
             with rffi.scoped_alloc_buffer(20) as buf:
-                res = api.PyUnicode_EncodeDecimal(u, 9, buf.raw, None)
-        assert res == -1
-        api.PyErr_Clear()
+                with pytest.raises(OperationError):
+                    PyUnicode_EncodeDecimal(space, u, 9, buf.raw, None)
 
         with rffi.scoped_unicode2wcharp(u' (12, \u1234\u1235)') as u:
             with rffi.scoped_alloc_buffer(20) as buf:
                 with rffi.scoped_str2charp("replace") as errors:
-                    res = api.PyUnicode_EncodeDecimal(u, 9, buf.raw,
+                    res = PyUnicode_EncodeDecimal(space, u, 9, buf.raw,
                                                       errors)
                 s = rffi.charp2str(buf.raw)
         assert res == 0
@@ -349,117 +351,117 @@
         with rffi.scoped_unicode2wcharp(u'12\u1234') as u:
             with rffi.scoped_alloc_buffer(20) as buf:
                 with rffi.scoped_str2charp("xmlcharrefreplace") as errors:
-                    res = api.PyUnicode_EncodeDecimal(u, 3, buf.raw,
+                    res = PyUnicode_EncodeDecimal(space, u, 3, buf.raw,
                                                       errors)
                 s = rffi.charp2str(buf.raw)
         assert res == 0
         assert s == "12&#4660;"
 
-    def test_encode_fsdefault(self, space, api):
+    def test_encode_fsdefault(self, space):
         w_u = space.wrap(u'sp&#228;m')
-        w_s = api.PyUnicode_EncodeFSDefault(w_u)
+        w_s = PyUnicode_EncodeFSDefault(space, w_u)
         if w_s is None:
-            api.PyErr_Clear()
+            PyErr_Clear(space)
             py.test.skip("Requires a unicode-aware fsencoding")
         with rffi.scoped_str2charp(space.str_w(w_s)) as encoded:
-            w_decoded = api.PyUnicode_DecodeFSDefaultAndSize(encoded, space.len_w(w_s))
+            w_decoded = PyUnicode_DecodeFSDefaultAndSize(space, encoded, space.len_w(w_s))
             assert space.eq_w(w_decoded, w_u)
-            w_decoded = api.PyUnicode_DecodeFSDefault(encoded)
+            w_decoded = PyUnicode_DecodeFSDefault(space, encoded)
             assert space.eq_w(w_decoded, w_u)
 
-    def test_fsconverter(self, space, api):
+    def test_fsconverter(self, space):
         # Input is bytes
         w_input = space.newbytes("test")
         with lltype.scoped_alloc(PyObjectP.TO, 1) as result:
             # Decoder
-            ret = api.PyUnicode_FSDecoder(w_input, result)
+            ret = PyUnicode_FSDecoder(space, w_input, result)
             assert ret == Py_CLEANUP_SUPPORTED
             assert space.isinstance_w(from_ref(space, result[0]), space.w_unicode)
-            assert api.PyUnicode_FSDecoder(None, result) == 1
+            assert PyUnicode_FSDecoder(space, None, result) == 1
             # Converter
-            ret = api.PyUnicode_FSConverter(w_input, result)
+            ret = PyUnicode_FSConverter(space, w_input, result)
             assert ret == Py_CLEANUP_SUPPORTED
             assert space.eq_w(from_ref(space, result[0]), w_input)
-            assert api.PyUnicode_FSDecoder(None, result) == 1
+            assert PyUnicode_FSDecoder(space, None, result) == 1
         # Input is unicode
         w_input = space.wrap("test")
         with lltype.scoped_alloc(PyObjectP.TO, 1) as result:
             # Decoder
-            ret = api.PyUnicode_FSDecoder(w_input, result)
+            ret = PyUnicode_FSDecoder(space, w_input, result)
             assert ret == Py_CLEANUP_SUPPORTED
             assert space.eq_w(from_ref(space, result[0]), w_input)
-            assert api.PyUnicode_FSDecoder(None, result) == 1
+            assert PyUnicode_FSDecoder(space, None, result) == 1
             # Converter
-            ret = api.PyUnicode_FSConverter(w_input, result)
+            ret = PyUnicode_FSConverter(space, w_input, result)
             assert ret == Py_CLEANUP_SUPPORTED
             assert space.isinstance_w(from_ref(space, result[0]), space.w_bytes)
-            assert api.PyUnicode_FSDecoder(None, result) == 1
+            assert PyUnicode_FSDecoder(space, None, result) == 1
 
-    def test_IS(self, space, api):
+    def test_IS(self, space):
         for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f,
                      0x20, 0x85, 0xa0, 0x1680, 0x2000, 0x2001, 0x2002,
                      0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008,
                      0x2009, 0x200a,
                      #0x200b is in Other_Default_Ignorable_Code_Point in 4.1.0
                      0x2028, 0x2029, 0x202f, 0x205f, 0x3000]:
-            assert api.Py_UNICODE_ISSPACE(unichr(char))
-        assert not api.Py_UNICODE_ISSPACE(u'a')
+            assert Py_UNICODE_ISSPACE(space, unichr(char))
+        assert not Py_UNICODE_ISSPACE(space, u'a')
 
-        assert api.Py_UNICODE_ISALPHA(u'a')
-        assert not api.Py_UNICODE_ISALPHA(u'0')
-        assert api.Py_UNICODE_ISALNUM(u'a')
-        assert api.Py_UNICODE_ISALNUM(u'0')
-        assert not api.Py_UNICODE_ISALNUM(u'+')
+        assert Py_UNICODE_ISALPHA(space, u'a')
+        assert not Py_UNICODE_ISALPHA(space, u'0')
+        assert Py_UNICODE_ISALNUM(space, u'a')
+        assert Py_UNICODE_ISALNUM(space, u'0')
+        assert not Py_UNICODE_ISALNUM(space, u'+')
 
-        assert api.Py_UNICODE_ISDECIMAL(u'\u0660')
-        assert not api.Py_UNICODE_ISDECIMAL(u'a')
-        assert api.Py_UNICODE_ISDIGIT(u'9')
-        assert not api.Py_UNICODE_ISDIGIT(u'@')
-        assert api.Py_UNICODE_ISNUMERIC(u'9')
-        assert not api.Py_UNICODE_ISNUMERIC(u'@')
+        assert Py_UNICODE_ISDECIMAL(space, u'\u0660')
+        assert not Py_UNICODE_ISDECIMAL(space, u'a')
+        assert Py_UNICODE_ISDIGIT(space, u'9')
+        assert not Py_UNICODE_ISDIGIT(space, u'@')
+        assert Py_UNICODE_ISNUMERIC(space, u'9')
+        assert not Py_UNICODE_ISNUMERIC(space, u'@')
 
         for char in [0x0a, 0x0d, 0x1c, 0x1d, 0x1e, 0x85, 0x2028, 0x2029]:
-            assert api.Py_UNICODE_ISLINEBREAK(unichr(char))
+            assert Py_UNICODE_ISLINEBREAK(space, unichr(char))
 
-        assert api.Py_UNICODE_ISLOWER(u'\xdf') # sharp s
-        assert api.Py_UNICODE_ISUPPER(u'\xde') # capital thorn
-        assert api.Py_UNICODE_ISLOWER(u'a')
-        assert not api.Py_UNICODE_ISUPPER(u'a')
-        assert not api.Py_UNICODE_ISTITLE(u'\xce')
-        assert api.Py_UNICODE_ISTITLE(
+        assert Py_UNICODE_ISLOWER(space, u'\xdf') # sharp s

From pypy.commits at gmail.com  Tue Jan 17 12:17:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 09:17:44 -0800 (PST)
Subject: [pypy-commit] pypy default: issue2464: another missing obscure case
Message-ID: <587e51b8.c4811c0a.28521.e788@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89638:f7660e563f57
Date: 2017-01-17 18:15 +0100
http://bitbucket.org/pypy/pypy/changeset/f7660e563f57/

Log:	issue2464: another missing obscure case

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -251,12 +251,11 @@
 
 class GetSetProperty(W_Root):
     _immutable_fields_ = ["fget", "fset", "fdel"]
-    name = '<generic property>'
     w_objclass = None
 
     @specialize.arg(7)
     def __init__(self, fget, fset=None, fdel=None, doc=None,
-                 cls=None, use_closure=False, tag=None):
+                 cls=None, use_closure=False, tag=None, name=None):
         objclass_getter, cls = make_objclass_getter(tag, fget, cls)
         fget = make_descr_typecheck_wrapper((tag, 0), fget,
                                             cls=cls, use_closure=use_closure)
@@ -264,9 +263,11 @@
                                             cls=cls, use_closure=use_closure)
         fdel = make_descr_typecheck_wrapper((tag, 2), fdel,
                                             cls=cls, use_closure=use_closure)
-        self._init(fget, fset, fdel, doc, cls, objclass_getter, use_closure)
+        self._init(fget, fset, fdel, doc, cls, objclass_getter, use_closure,
+                   name)
 
-    def _init(self, fget, fset, fdel, doc, cls, objclass_getter, use_closure):
+    def _init(self, fget, fset, fdel, doc, cls, objclass_getter, use_closure,
+              name):
         self.fget = fget
         self.fset = fset
         self.fdel = fdel
@@ -274,13 +275,13 @@
         self.reqcls = cls
         self.objclass_getter = objclass_getter
         self.use_closure = use_closure
+        self.name = name if name is not None else '<generic property>'
 
     def copy_for_type(self, w_objclass):
         if self.objclass_getter is None:
             new = instantiate(GetSetProperty)
             new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
-                      None, self.use_closure)
-            new.name = self.name
+                      None, self.use_closure, self.name)
             new.w_objclass = w_objclass
             return new
         else:
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1334,3 +1334,6 @@
         assert file.closed.__objclass__ is file
         assert type.__dict__['__name__'].__objclass__ is type
         assert type.__dict__['__doc__'].__objclass__ is type
+        #
+        assert type.__dict__['__name__'].__name__ == '__name__'
+        assert type.__dict__['__doc__'].__name__ == '__doc__'
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -958,7 +958,7 @@
     __base__ = GetSetProperty(descr__base),
     __mro__ = GetSetProperty(descr_get__mro__),
     __dict__=GetSetProperty(type_get_dict),
-    __doc__ = GetSetProperty(descr__doc, cls=W_TypeObject),
+    __doc__ = GetSetProperty(descr__doc, cls=W_TypeObject, name='__doc__'),
     mro = gateway.interp2app(descr_mro),
     __flags__ = GetSetProperty(descr__flags),
     __module__ = GetSetProperty(descr_get__module, descr_set__module),

From pypy.commits at gmail.com  Tue Jan 17 12:43:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 09:43:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix or skip remaining tests in json
Message-ID: <587e57b6.8d071c0a.e2f3e.15b6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89639:f11fffc36423
Date: 2017-01-17 17:42 +0000
http://bitbucket.org/pypy/pypy/changeset/f11fffc36423/

Log:	fix or skip remaining tests in json

diff --git a/lib-python/3/test/test_json/test_fail.py b/lib-python/3/test/test_json/test_fail.py
--- a/lib-python/3/test/test_json/test_fail.py
+++ b/lib-python/3/test/test_json/test_fail.py
@@ -1,4 +1,5 @@
 from test.test_json import PyTest, CTest
+from test import support
 import re
 
 # 2007-10-05
@@ -127,7 +128,16 @@
             with self.assertRaises(self.JSONDecodeError) as cm:
                 self.loads(data)
             err = cm.exception
-            self.assertEqual(err.msg, msg)
+            if support.check_impl_detail():
+                self.assertEqual(err.msg, msg)
+            else:
+                if data in ['[42',         # skip these tests, PyPy gets
+                            '["spam"',     # another error which makes sense
+                            '{"spam":42',  # too: unterminated array
+                           ]:
+                    continue
+                msg = err.msg   # ignore the message provided in the test,
+                                # only check for position
             self.assertEqual(err.pos, idx)
             self.assertEqual(err.lineno, 1)
             self.assertEqual(err.colno, idx + 1)
@@ -163,7 +173,11 @@
             with self.assertRaises(self.JSONDecodeError) as cm:
                 self.loads(data)
             err = cm.exception
-            self.assertEqual(err.msg, msg)
+            if support.check_impl_detail():
+                self.assertEqual(err.msg, msg)
+            else:
+                msg = err.msg   # ignore the message provided in the test,
+                                # only check for position
             self.assertEqual(err.pos, idx)
             self.assertEqual(err.lineno, 1)
             self.assertEqual(err.colno, idx + 1)
@@ -205,13 +219,18 @@
             with self.assertRaises(self.JSONDecodeError) as cm:
                 self.loads(data)
             err = cm.exception
-            self.assertEqual(err.msg, 'Expecting value')
+            if support.check_impl_detail():
+                msg = 'Expecting value'
+                self.assertEqual(err.msg, msg)
+            else:
+                msg = err.msg   # ignore the message provided in the test,
+                                # only check for position
             self.assertEqual(err.pos, idx)
             self.assertEqual(err.lineno, line)
             self.assertEqual(err.colno, col)
             self.assertEqual(str(err),
-                             'Expecting value: line %s column %d (char %d)' %
-                             (line, col, idx))
+                             '%s: line %s column %d (char %d)' %
+                             (msg, line, col, idx))
 
 class TestPyFail(TestFail, PyTest): pass
 class TestCFail(TestFail, CTest): pass
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -107,9 +107,7 @@
         elif ch.isdigit():
             return self.decode_numeric(i)
         else:
-            raise DecoderError(
-                "No JSON object could be decoded: unexpected '%s' at" % ch,
-                i)
+            raise DecoderError("Unexpected '%s' at" % ch, i)
 
     def decode_null(self, i):
         if (self.ll_chars[i]   == 'u' and
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -218,15 +218,15 @@
     def test_error_position(self):
         import _pypyjson
         test_cases = [
-            ('[,', "No JSON object could be decoded: unexpected ',' at", 1),
-            ('{"spam":[}', "No JSON object could be decoded: unexpected '}' at", 9),
+            ('[,', "Unexpected ',' at", 1),
+            ('{"spam":[}', "Unexpected '}' at", 9),
             ('[42:', "Unexpected ':' when decoding array", 3),
             ('[42 "spam"', "Unexpected '\"' when decoding array", 4),
-            ('[42,]', "No JSON object could be decoded: unexpected ']' at", 4),
+            ('[42,]', "Unexpected ']' at", 4),
             ('{"spam":[42}', "Unexpected '}' when decoding array", 11),
             ('["]', 'Unterminated string starting at', 1),
             ('["spam":', "Unexpected ':' when decoding array", 7),
-            ('[{]', "No JSON object could be decoded: unexpected ']' at", 2),
+            ('[{]', "Unexpected ']' at", 2),
         ]
         for inputtext, errmsg, errpos in test_cases:
             exc = raises(ValueError, _pypyjson.loads, inputtext)

From pypy.commits at gmail.com  Tue Jan 17 12:47:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jan 2017 09:47:33 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <587e58b5.ce941c0a.4a05c.f679@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r847:cbf84a3ab462
Date: 2017-01-17 18:47 +0100
http://bitbucket.org/pypy/pypy.org/changeset/cbf84a3ab462/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66545 of $105000 (63.4%)
+   $66560 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2377 left
+    we have $2390 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Tue Jan 17 12:58:05 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 09:58:05 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: document branch
Message-ID: <587e5b2d.ce841c0a.6102b.2287@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89640:861483e6c971
Date: 2017-01-17 17:57 +0000
http://bitbucket.org/pypy/pypy/changeset/861483e6c971/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -119,3 +119,9 @@
 Also fix for c-extension type that calls ``tp_hash`` during initialization
 (str, unicode types), and fix instantiating c-extension types from built-in
 classes by enforcing an order of instaniation.
+
+.. branch: rffi-parser-2
+
+rffi structures in cpyext can now be created by parsing simple C headers.
+Additionally, the cts object that holds the parsed information can act like
+cffi's ffi objects, with the methods cts.cast() and cts.gettype().

From pypy.commits at gmail.com  Tue Jan 17 12:59:11 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 09:59:11 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser-2: Close branch rffi-parser-2
Message-ID: <587e5b6f.d5091c0a.81af2.272b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser-2
Changeset: r89641:716c24634b02
Date: 2017-01-17 17:58 +0000
http://bitbucket.org/pypy/pypy/changeset/716c24634b02/

Log:	Close branch rffi-parser-2


From pypy.commits at gmail.com  Tue Jan 17 12:59:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 09:59:30 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in rffi-parser-2 (pull request
 #509)
Message-ID: <587e5b82.cb911c0a.619ed.d1e9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89642:721ea0b51de4
Date: 2017-01-17 17:58 +0000
http://bitbucket.org/pypy/pypy/changeset/721ea0b51de4/

Log:	Merged in rffi-parser-2 (pull request #509)

	A branch to use a cffi-style C parser to create rffi objects in
	cpyext.

diff too long, truncating to 2000 out of 2652 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -119,3 +119,9 @@
 Also fix for c-extension type that calls ``tp_hash`` during initialization
 (str, unicode types), and fix instantiating c-extension types from built-in
 classes by enforcing an order of instaniation.
+
+.. branch: rffi-parser-2
+
+rffi structures in cpyext can now be created by parsing simple C headers.
+Additionally, the cts object that holds the parsed information can act like
+cffi's ffi objects, with the methods cts.cast() and cts.gettype().
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -41,37 +41,35 @@
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
+from pypy.module.cpyext.cparser import CTypeSpace
 
 DEBUG_WRAPPER = True
 
-# update these for other platforms
-Py_ssize_t = lltype.Typedef(rffi.SSIZE_T, 'Py_ssize_t')
-Py_ssize_tP = rffi.CArrayPtr(Py_ssize_t)
-size_t = rffi.ULONG
-ADDR = lltype.Signed
-
 pypydir = py.path.local(pypydir)
 include_dir = pypydir / 'module' / 'cpyext' / 'include'
+parse_dir = pypydir / 'module' / 'cpyext' / 'parse'
 source_dir = pypydir / 'module' / 'cpyext' / 'src'
 translator_c_dir = py.path.local(cdir)
 include_dirs = [
     include_dir,
+    parse_dir,
     translator_c_dir,
     udir,
     ]
 
-class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
+configure_eci = ExternalCompilationInfo(
         include_dirs=include_dirs,
         includes=['Python.h', 'stdarg.h', 'structmember.h'],
-        compile_extra=['-DPy_BUILD_CORE'],
-        )
+        compile_extra=['-DPy_BUILD_CORE'])
+
+class CConfig:
+    _compilation_info_ = configure_eci
 
 class CConfig2:
-    _compilation_info_ = CConfig._compilation_info_
+    _compilation_info_ = configure_eci
 
 class CConfig_constants:
-    _compilation_info_ = CConfig._compilation_info_
+    _compilation_info_ = configure_eci
 
 CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
                                        hints={'nolength': True}),
@@ -118,6 +116,9 @@
     return is_valid_fd(c_fileno(fp))
 
 pypy_decl = 'pypy_decl.h'
+udir.join(pypy_decl).write("/* Will be filled later */\n")
+udir.join('pypy_structmember_decl.h').write("/* Will be filled later */\n")
+udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
@@ -129,9 +130,6 @@
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
-udir.join(pypy_decl).write("/* Will be filled later */\n")
-udir.join('pypy_structmember_decl.h').write("/* Will be filled later */\n")
-udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 globals().update(rffi_platform.configure(CConfig_constants))
 
 def _copy_header_files(headers, dstdir):
@@ -145,12 +143,14 @@
         target.chmod(0444) # make the file read-only, to make sure that nobody
                            # edits it by mistake
 
-def copy_header_files(dstdir, copy_numpy_headers):
+def copy_header_files(cts, dstdir, copy_numpy_headers):
     # XXX: 20 lines of code to recursively copy a directory, really??
     assert dstdir.check(dir=True)
     headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl')
     for name in ["pypy_macros.h"] + FUNCTIONS_BY_HEADER.keys():
         headers.append(udir.join(name))
+    for path in cts.parsed_headers:
+        headers.append(path)
     _copy_header_files(headers, dstdir)
 
     if copy_numpy_headers:
@@ -251,13 +251,15 @@
 
 class ApiFunction(object):
     def __init__(self, argtypes, restype, callable, error=CANNOT_FAIL,
-                 c_name=None, gil=None, result_borrowed=False, result_is_ll=False):
+                 c_name=None, cdecl=None, gil=None,
+                 result_borrowed=False, result_is_ll=False):
         self.argtypes = argtypes
         self.restype = restype
         self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
         self.callable = callable
         self.error_value = error
         self.c_name = c_name
+        self.cdecl = cdecl
 
         # extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
@@ -272,8 +274,6 @@
         self.gil = gil
         self.result_borrowed = result_borrowed
         self.result_is_ll = result_is_ll
-        if result_is_ll:    # means 'returns a low-level PyObject pointer'
-            assert is_PyObject(restype)
         #
         def get_llhelper(space):
             return llhelper(self.functype, self.get_wrapper(space))
@@ -378,6 +378,8 @@
         return unwrapper
 
     def get_c_restype(self, c_writer):
+        if self.cdecl:
+            return self.cdecl.split(self.c_name)[0].strip()
         return c_writer.gettype(self.restype).replace('@', '').strip()
 
     def get_c_args(self, c_writer):
@@ -477,6 +479,20 @@
         return unwrapper
     return decorate
 
+def api_decl(cdecl, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+    def decorate(func):
+        func._always_inline_ = 'try'
+        name, FUNC = cts.parse_func(cdecl)
+        api_function = ApiFunction(
+            FUNC.ARGS, FUNC.RESULT, func,
+            error=_compute_error(error, FUNC.RESULT), cdecl=cdecl)
+        FUNCTIONS_BY_HEADER[header][name] = api_function
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
+
 def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
     def decorate(func):
         func._always_inline_ = 'try'
@@ -655,41 +671,31 @@
                              % (cpyname, ))
 build_exported_objects()
 
+cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h'])
+cts.parse_header(parse_dir / 'cpyext_object.h')
+
+Py_ssize_t = cts.gettype('Py_ssize_t')
+Py_ssize_tP = cts.gettype('Py_ssize_t *')
+size_t = rffi.ULONG
+ADDR = lltype.Signed
+
 # Note: as a special case, "PyObject" is the pointer type in RPython,
 # corresponding to "PyObject *" in C.  We do that only for PyObject.
 # For example, "PyTypeObject" is the struct type even in RPython.
-PyTypeObject = lltype.ForwardReference()
-PyTypeObjectPtr = lltype.Ptr(PyTypeObject)
-PyObjectStruct = lltype.ForwardReference()
-PyObject = lltype.Ptr(PyObjectStruct)
+PyTypeObject = cts.gettype('PyTypeObject')
+PyTypeObjectPtr = cts.gettype('PyTypeObject *')
+PyObjectStruct = cts.gettype('PyObject')
+PyObject = cts.gettype('PyObject *')
 PyObjectFields = (("ob_refcnt", lltype.Signed),
                   ("ob_pypy_link", lltype.Signed),
                   ("ob_type", PyTypeObjectPtr))
 PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), )
-cpython_struct('PyObject', PyObjectFields, PyObjectStruct)
-PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields)
-PyVarObject = lltype.Ptr(PyVarObjectStruct)
+PyVarObjectStruct = cts.gettype('PyVarObject')
+PyVarObject = cts.gettype('PyVarObject *')
 
-Py_buffer = cpython_struct(
-    "Py_buffer", (
-        ('buf', rffi.VOIDP),
-        ('obj', PyObject),
-        ('len', Py_ssize_t),
-        ('itemsize', Py_ssize_t),
+Py_buffer = cts.gettype('Py_buffer')
+Py_bufferP = cts.gettype('Py_buffer *')
 
-        ('readonly', rffi.INT_real),
-        ('ndim', rffi.INT_real),
-        ('format', rffi.CCHARP),
-        ('shape', Py_ssize_tP),
-        ('strides', Py_ssize_tP),
-        ('suboffsets', Py_ssize_tP),
-        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
-        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
-        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
-        #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
-        ('internal', rffi.VOIDP),
-))
-Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
@@ -1409,7 +1415,7 @@
         include_lines.append('RPY_EXPORTED %s %s;\n' % (typ, name))
 
     lines.append('};\n')
-    eci2 = CConfig._compilation_info_.merge(ExternalCompilationInfo(
+    eci2 = configure_eci.merge(ExternalCompilationInfo(
         separate_module_sources = [''.join(lines)],
         post_include_bits = [''.join(include_lines)],
         ))
@@ -1427,7 +1433,7 @@
 
     setup_init_functions(eci, prefix)
     trunk_include = pypydir.dirpath() / 'include'
-    copy_header_files(trunk_include, use_micronumpy)
+    copy_header_files(cts, trunk_include, use_micronumpy)
 
 
 def _load_from_cffi(space, name, path, initptr):
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/cparser.py
@@ -0,0 +1,879 @@
+from collections import OrderedDict
+from cffi import api, model
+from cffi.commontypes import COMMON_TYPES, resolve_common_type
+try:
+    from cffi import _pycparser as pycparser
+except ImportError:
+    import pycparser
+import weakref, re
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib.rfile import FILEP
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.tool import rfficache, rffi_platform
+from rpython.flowspace.model import Constant, const
+from rpython.flowspace.specialcase import register_flow_sc
+from rpython.flowspace.flowcontext import FlowingError
+
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
+_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
+_r_words = re.compile(r"\w+|\S")
+_parser_cache = None
+_r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE)
+_r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b")
+_r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b")
+_r_cdecl = re.compile(r"\b__cdecl\b")
+_r_star_const_space = re.compile(       # matches "* const "
+    r"[*]\s*((const|volatile|restrict)\b\s*)+")
+
+def _get_parser():
+    global _parser_cache
+    if _parser_cache is None:
+        _parser_cache = pycparser.CParser()
+    return _parser_cache
+
+def _preprocess(csource, macros):
+    # Remove comments.  NOTE: this only work because the cdef() section
+    # should not contain any string literal!
+    csource = _r_comment.sub(' ', csource)
+    # Remove the "#define FOO x" lines
+    for match in _r_define.finditer(csource):
+        macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
+        macros[macroname] = macrovalue
+    csource = _r_define.sub('', csource)
+    #
+    # BIG HACK: replace WINAPI or __stdcall with "volatile const".
+    # It doesn't make sense for the return type of a function to be
+    # "volatile volatile const", so we abuse it to detect __stdcall...
+    # Hack number 2 is that "int(volatile *fptr)();" is not valid C
+    # syntax, so we place the "volatile" before the opening parenthesis.
+    csource = _r_stdcall2.sub(' volatile volatile const(', csource)
+    csource = _r_stdcall1.sub(' volatile volatile const ', csource)
+    csource = _r_cdecl.sub(' ', csource)
+
+    for name, value in reversed(macros.items()):
+        csource = re.sub(r'\b%s\b' % name, value, csource)
+
+    return csource, macros
+
+def _common_type_names(csource):
+    # Look in the source for what looks like usages of types from the
+    # list of common types.  A "usage" is approximated here as the
+    # appearance of the word, minus a "definition" of the type, which
+    # is the last word in a "typedef" statement.  Approximative only
+    # but should be fine for all the common types.
+    look_for_words = set(COMMON_TYPES)
+    look_for_words.add(';')
+    look_for_words.add(',')
+    look_for_words.add('(')
+    look_for_words.add(')')
+    look_for_words.add('typedef')
+    words_used = set()
+    is_typedef = False
+    paren = 0
+    previous_word = ''
+    for word in _r_words.findall(csource):
+        if word in look_for_words:
+            if word == ';':
+                if is_typedef:
+                    words_used.discard(previous_word)
+                    look_for_words.discard(previous_word)
+                    is_typedef = False
+            elif word == 'typedef':
+                is_typedef = True
+                paren = 0
+            elif word == '(':
+                paren += 1
+            elif word == ')':
+                paren -= 1
+            elif word == ',':
+                if is_typedef and paren == 0:
+                    words_used.discard(previous_word)
+                    look_for_words.discard(previous_word)
+            else:   # word in COMMON_TYPES
+                words_used.add(word)
+        previous_word = word
+    return words_used
+
+
+class Parser(object):
+
+    def __init__(self):
+        self._declarations = {}
+        self._included_declarations = set()
+        self._anonymous_counter = 0
+        self._structnode2type = weakref.WeakKeyDictionary()
+        self._options = {}
+        self._int_constants = {}
+        self._recomplete = []
+        self._macros = OrderedDict()
+
+    def _parse(self, csource):
+        # modifies self._macros in-place
+        csource, macros = _preprocess(csource, self._macros)
+        # XXX: for more efficiency we would need to poke into the
+        # internals of CParser...  the following registers the
+        # typedefs, because their presence or absence influences the
+        # parsing itself (but what they are typedef'ed to plays no role)
+        ctn = _common_type_names(csource)
+        typenames = []
+        for name in sorted(self._declarations):
+            if name.startswith('typedef '):
+                name = name[8:]
+                typenames.append(name)
+                ctn.discard(name)
+        typenames += sorted(ctn)
+        #
+        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines.append('typedef int __dotdotdot__;')
+        csourcelines.append(csource)
+        csource = '\n'.join(csourcelines)
+        try:
+            ast = _get_parser().parse(csource)
+        except pycparser.c_parser.ParseError as e:
+            self.convert_pycparser_error(e, csource)
+        # csource will be used to find buggy source text
+        return ast, macros, csource
+
+    def _convert_pycparser_error(self, e, csource):
+        # xxx look for ":NUM:" at the start of str(e) and try to interpret
+        # it as a line number
+        line = None
+        msg = str(e)
+        if msg.startswith(':') and ':' in msg[1:]:
+            linenum = msg[1:msg.find(':',1)]
+            if linenum.isdigit():
+                linenum = int(linenum, 10)
+                csourcelines = csource.splitlines()
+                if 1 <= linenum <= len(csourcelines):
+                    line = csourcelines[linenum-1]
+        return line
+
+    def convert_pycparser_error(self, e, csource):
+        line = self._convert_pycparser_error(e, csource)
+
+        msg = str(e)
+        if line:
+            msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
+        else:
+            msg = 'parse error\n%s' % (msg,)
+        raise api.CDefError(msg)
+
+    def parse(self, csource, override=False, packed=False, dllexport=False):
+        prev_options = self._options
+        try:
+            self._options = {'override': override,
+                             'packed': packed,
+                             'dllexport': dllexport}
+            self._internal_parse(csource)
+        finally:
+            self._options = prev_options
+
+    def _internal_parse(self, csource):
+        ast, macros, csource = self._parse(csource)
+        # add the macros
+        self._process_macros(macros)
+        # find the first "__dotdotdot__" and use that as a separator
+        # between the repeated typedefs and the real csource
+        iterator = iter(ast.ext)
+        for decl in iterator:
+            if decl.name == '__dotdotdot__':
+                break
+        #
+        try:
+            for decl in iterator:
+                if isinstance(decl, pycparser.c_ast.Decl):
+                    self._parse_decl(decl)
+                elif isinstance(decl, pycparser.c_ast.Typedef):
+                    if not decl.name:
+                        raise api.CDefError("typedef does not declare any name",
+                                            decl)
+                    quals = 0
+                    realtype, quals = self._get_type_and_quals(
+                        decl.type, name=decl.name, partial_length_ok=True)
+                    self._declare('typedef ' + decl.name, realtype, quals=quals)
+                elif decl.__class__.__name__ == 'Pragma':
+                    pass    # skip pragma, only in pycparser 2.15
+                else:
+                    raise api.CDefError("unrecognized construct", decl)
+        except api.FFIError as e:
+            msg = self._convert_pycparser_error(e, csource)
+            if msg:
+                e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
+            raise
+
+    def _add_constants(self, key, val):
+        if key in self._int_constants:
+            if self._int_constants[key] == val:
+                return     # ignore identical double declarations
+            raise api.FFIError(
+                "multiple declarations of constant: %s" % (key,))
+        self._int_constants[key] = val
+
+    def _add_integer_constant(self, name, int_str):
+        int_str = int_str.lower().rstrip("ul")
+        neg = int_str.startswith('-')
+        if neg:
+            int_str = int_str[1:]
+        # "010" is not valid oct in py3
+        if (int_str.startswith("0") and int_str != '0'
+                and not int_str.startswith("0x")):
+            int_str = "0o" + int_str[1:]
+        pyvalue = int(int_str, 0)
+        if neg:
+            pyvalue = -pyvalue
+        self._add_constants(name, pyvalue)
+        self._declare('macro ' + name, pyvalue)
+
+    def _process_macros(self, macros):
+        for key, value in macros.items():
+            value = value.strip()
+            if _r_int_literal.match(value):
+                self._add_integer_constant(key, value)
+            else:
+                self._declare('macro ' + key, value)
+
+    def _declare_function(self, tp, quals, decl):
+        tp = self._get_type_pointer(tp, quals)
+        if self._options.get('dllexport'):
+            tag = 'dllexport_python '
+        else:
+            tag = 'function '
+        self._declare(tag + decl.name, tp)
+
+    def _parse_decl(self, decl):
+        node = decl.type
+        if isinstance(node, pycparser.c_ast.FuncDecl):
+            tp, quals = self._get_type_and_quals(node, name=decl.name)
+            assert isinstance(tp, model.RawFunctionType)
+            self._declare_function(tp, quals, decl)
+        else:
+            if isinstance(node, pycparser.c_ast.Struct):
+                self._get_struct_union_enum_type('struct', node)
+            elif isinstance(node, pycparser.c_ast.Union):
+                self._get_struct_union_enum_type('union', node)
+            elif isinstance(node, pycparser.c_ast.Enum):
+                self._get_struct_union_enum_type('enum', node)
+            elif not decl.name:
+                raise api.CDefError("construct does not declare any variable",
+                                    decl)
+            #
+            if decl.name:
+                tp, quals = self._get_type_and_quals(node,
+                                                     partial_length_ok=True)
+                if tp.is_raw_function:
+                    self._declare_function(tp, quals, decl)
+                elif (tp.is_integer_type() and
+                        hasattr(decl, 'init') and
+                        hasattr(decl.init, 'value') and
+                        _r_int_literal.match(decl.init.value)):
+                    self._add_integer_constant(decl.name, decl.init.value)
+                elif (tp.is_integer_type() and
+                        isinstance(decl.init, pycparser.c_ast.UnaryOp) and
+                        decl.init.op == '-' and
+                        hasattr(decl.init.expr, 'value') and
+                        _r_int_literal.match(decl.init.expr.value)):
+                    self._add_integer_constant(decl.name,
+                                               '-' + decl.init.expr.value)
+                else:
+                    if (quals & model.Q_CONST) and not tp.is_array_type:
+                        self._declare('constant ' + decl.name, tp, quals=quals)
+                    else:
+                        self._declare('variable ' + decl.name, tp, quals=quals)
+
+    def parse_type(self, cdecl):
+        return self.parse_type_and_quals(cdecl)[0]
+
+    def parse_type_and_quals(self, cdecl):
+        ast, _, _ = self._parse('void __dummy(\n%s\n);' % cdecl)
+        exprnode = ast.ext[-1].type.args.params[0]
+        if isinstance(exprnode, pycparser.c_ast.ID):
+            raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
+        return self._get_type_and_quals(exprnode.type)
+
+    def _declare(self, name, obj, included=False, quals=0):
+        if name in self._declarations:
+            prevobj, prevquals = self._declarations[name]
+            if prevobj is obj and prevquals == quals:
+                return
+        self._declarations[name] = (obj, quals)
+        if included:
+            self._included_declarations.add(obj)
+
+    def _extract_quals(self, type):
+        quals = 0
+        if isinstance(type, (pycparser.c_ast.TypeDecl,
+                             pycparser.c_ast.PtrDecl)):
+            if 'const' in type.quals:
+                quals |= model.Q_CONST
+            if 'volatile' in type.quals:
+                quals |= model.Q_VOLATILE
+            if 'restrict' in type.quals:
+                quals |= model.Q_RESTRICT
+        return quals
+
+    def _get_type_pointer(self, type, quals, declname=None):
+        if isinstance(type, model.RawFunctionType):
+            return type.as_function_pointer()
+        if (isinstance(type, model.StructOrUnionOrEnum) and
+                type.name.startswith('$') and type.name[1:].isdigit() and
+                type.forcename is None and declname is not None):
+            return model.NamedPointerType(type, declname, quals)
+        return model.PointerType(type, quals)
+
+    def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False):
+        # first, dereference typedefs, if we have it already parsed, we're good
+        if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
+            isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
+            len(typenode.type.names) == 1 and
+            ('typedef ' + typenode.type.names[0]) in self._declarations):
+            tp, quals = self._declarations['typedef ' + typenode.type.names[0]]
+            quals |= self._extract_quals(typenode)
+            return tp, quals
+        #
+        if isinstance(typenode, pycparser.c_ast.ArrayDecl):
+            # array type
+            if typenode.dim is None:
+                length = None
+            else:
+                length = self._parse_constant(
+                    typenode.dim, partial_length_ok=partial_length_ok)
+            tp, quals = self._get_type_and_quals(typenode.type,
+                                partial_length_ok=partial_length_ok)
+            return model.ArrayType(tp, length), quals
+        #
+        if isinstance(typenode, pycparser.c_ast.PtrDecl):
+            # pointer type
+            itemtype, itemquals = self._get_type_and_quals(typenode.type)
+            tp = self._get_type_pointer(itemtype, itemquals, declname=name)
+            quals = self._extract_quals(typenode)
+            return tp, quals
+        #
+        if isinstance(typenode, pycparser.c_ast.TypeDecl):
+            quals = self._extract_quals(typenode)
+            type = typenode.type
+            if isinstance(type, pycparser.c_ast.IdentifierType):
+                # assume a primitive type.  get it from .names, but reduce
+                # synonyms to a single chosen combination
+                names = list(type.names)
+                if names != ['signed', 'char']:    # keep this unmodified
+                    prefixes = {}
+                    while names:
+                        name = names[0]
+                        if name in ('short', 'long', 'signed', 'unsigned'):
+                            prefixes[name] = prefixes.get(name, 0) + 1
+                            del names[0]
+                        else:
+                            break
+                    # ignore the 'signed' prefix below, and reorder the others
+                    newnames = []
+                    for prefix in ('unsigned', 'short', 'long'):
+                        for i in range(prefixes.get(prefix, 0)):
+                            newnames.append(prefix)
+                    if not names:
+                        names = ['int']    # implicitly
+                    if names == ['int']:   # but kill it if 'short' or 'long'
+                        if 'short' in prefixes or 'long' in prefixes:
+                            names = []
+                    names = newnames + names
+                ident = ' '.join(names)
+                if ident == 'void':
+                    return model.void_type, quals
+                tp0, quals0 = resolve_common_type(self, ident)
+                return tp0, (quals | quals0)
+            #
+            if isinstance(type, pycparser.c_ast.Struct):
+                # 'struct foobar'
+                tp = self._get_struct_union_enum_type('struct', type, name)
+                return tp, quals
+            #
+            if isinstance(type, pycparser.c_ast.Union):
+                # 'union foobar'
+                tp = self._get_struct_union_enum_type('union', type, name)
+                return tp, quals
+            #
+            if isinstance(type, pycparser.c_ast.Enum):
+                # 'enum foobar'
+                tp = self._get_struct_union_enum_type('enum', type, name)
+                return tp, quals
+        #
+        if isinstance(typenode, pycparser.c_ast.FuncDecl):
+            # a function type
+            return self._parse_function_type(typenode, name), 0
+        #
+        # nested anonymous structs or unions end up here
+        if isinstance(typenode, pycparser.c_ast.Struct):
+            return self._get_struct_union_enum_type('struct', typenode, name,
+                                                    nested=True), 0
+        if isinstance(typenode, pycparser.c_ast.Union):
+            return self._get_struct_union_enum_type('union', typenode, name,
+                                                    nested=True), 0
+        #
+        raise api.FFIError(":%d: bad or unsupported type declaration" %
+                typenode.coord.line)
+
+    def _parse_function_type(self, typenode, funcname=None):
+        params = list(getattr(typenode.args, 'params', []))
+        for i, arg in enumerate(params):
+            if not hasattr(arg, 'type'):
+                raise api.CDefError("%s arg %d: unknown type '%s'"
+                    " (if you meant to use the old C syntax of giving"
+                    " untyped arguments, it is not supported)"
+                    % (funcname or 'in expression', i + 1,
+                       getattr(arg, 'name', '?')))
+        ellipsis = (
+            len(params) > 0 and
+            isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and
+            isinstance(params[-1].type.type,
+                       pycparser.c_ast.IdentifierType) and
+            params[-1].type.type.names == ['__dotdotdot__'])
+        if ellipsis:
+            params.pop()
+            if not params:
+                raise api.CDefError(
+                    "%s: a function with only '(...)' as argument"
+                    " is not correct C" % (funcname or 'in expression'))
+        args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type))
+                for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
+        result, quals = self._get_type_and_quals(typenode.type)
+        # the 'quals' on the result type are ignored.  HACK: we absure them
+        # to detect __stdcall functions: we textually replace "__stdcall"
+        # with "volatile volatile const" above.
+        abi = None
+        if hasattr(typenode.type, 'quals'): # else, probable syntax error anyway
+            if typenode.type.quals[-3:] == ['volatile', 'volatile', 'const']:
+                abi = '__stdcall'
+        return model.RawFunctionType(tuple(args), result, ellipsis, abi)
+
+    def _as_func_arg(self, type, quals):
+        if isinstance(type, model.ArrayType):
+            return model.PointerType(type.item, quals)
+        elif isinstance(type, model.RawFunctionType):
+            return type.as_function_pointer()
+        else:
+            return type
+
+    def _get_struct_union_enum_type(self, kind, type, name=None, nested=False):
+        # First, a level of caching on the exact 'type' node of the AST.
+        # This is obscure, but needed because pycparser "unrolls" declarations
+        # such as "typedef struct { } foo_t, *foo_p" and we end up with
+        # an AST that is not a tree, but a DAG, with the "type" node of the
+        # two branches foo_t and foo_p of the trees being the same node.
+        # It's a bit silly but detecting "DAG-ness" in the AST tree seems
+        # to be the only way to distinguish this case from two independent
+        # structs.  See test_struct_with_two_usages.
+        try:
+            return self._structnode2type[type]
+        except KeyError:
+            pass
+        #
+        # Note that this must handle parsing "struct foo" any number of
+        # times and always return the same StructType object.  Additionally,
+        # one of these times (not necessarily the first), the fields of
+        # the struct can be specified with "struct foo { ...fields... }".
+        # If no name is given, then we have to create a new anonymous struct
+        # with no caching; in this case, the fields are either specified
+        # right now or never.
+        #
+        force_name = name
+        name = type.name
+        #
+        # get the type or create it if needed
+        if name is None:
+            # 'force_name' is used to guess a more readable name for
+            # anonymous structs, for the common case "typedef struct { } foo".
+            if force_name is not None:
+                explicit_name = '$%s' % force_name
+            else:
+                self._anonymous_counter += 1
+                explicit_name = '$%d' % self._anonymous_counter
+            tp = None
+        else:
+            explicit_name = name
+            key = '%s %s' % (kind, name)
+            tp, _ = self._declarations.get(key, (None, None))
+        #
+        if tp is None:
+            if kind == 'struct':
+                tp = model.StructType(explicit_name, None, None, None)
+            elif kind == 'union':
+                tp = model.UnionType(explicit_name, None, None, None)
+            elif kind == 'enum':
+                tp = self._build_enum_type(explicit_name, type.values)
+            else:
+                raise AssertionError("kind = %r" % (kind,))
+            if name is not None:
+                self._declare(key, tp)
+        else:
+            if kind == 'enum' and type.values is not None:
+                raise NotImplementedError(
+                    "enum %s: the '{}' declaration should appear on the first "
+                    "time the enum is mentioned, not later" % explicit_name)
+        if not tp.forcename:
+            tp.force_the_name(force_name)
+        if tp.forcename and '$' in tp.name:
+            self._declare('anonymous %s' % tp.forcename, tp)
+        #
+        self._structnode2type[type] = tp
+        #
+        # enums: done here
+        if kind == 'enum':
+            return tp
+        #
+        # is there a 'type.decls'?  If yes, then this is the place in the
+        # C sources that declare the fields.  If no, then just return the
+        # existing type, possibly still incomplete.
+        if type.decls is None:
+            return tp
+        #
+        if tp.fldnames is not None:
+            raise api.CDefError("duplicate declaration of struct %s" % name)
+        fldnames = []
+        fldtypes = []
+        fldbitsize = []
+        fldquals = []
+        for decl in type.decls:
+            if decl.bitsize is None:
+                bitsize = -1
+            else:
+                bitsize = self._parse_constant(decl.bitsize)
+            self._partial_length = False
+            type, fqual = self._get_type_and_quals(decl.type,
+                                                   partial_length_ok=True)
+            if self._partial_length:
+                self._make_partial(tp, nested)
+            if isinstance(type, model.StructType) and type.partial:
+                self._make_partial(tp, nested)
+            fldnames.append(decl.name or '')
+            fldtypes.append(type)
+            fldbitsize.append(bitsize)
+            fldquals.append(fqual)
+        tp.fldnames = tuple(fldnames)
+        tp.fldtypes = tuple(fldtypes)
+        tp.fldbitsize = tuple(fldbitsize)
+        tp.fldquals = tuple(fldquals)
+        if fldbitsize != [-1] * len(fldbitsize):
+            if isinstance(tp, model.StructType) and tp.partial:
+                raise NotImplementedError("%s: using both bitfields and '...;'"
+                                          % (tp,))
+        tp.packed = self._options.get('packed')
+        if tp.completed:    # must be re-completed: it is not opaque any more
+            tp.completed = 0
+            self._recomplete.append(tp)
+        return tp
+
+    def _make_partial(self, tp, nested):
+        if not isinstance(tp, model.StructOrUnion):
+            raise api.CDefError("%s cannot be partial" % (tp,))
+        if not tp.has_c_name() and not nested:
+            raise NotImplementedError("%s is partial but has no C name" %(tp,))
+        tp.partial = True
+
+    def _parse_constant(self, exprnode, partial_length_ok=False):
+        # for now, limited to expressions that are an immediate number
+        # or positive/negative number
+        if isinstance(exprnode, pycparser.c_ast.Constant):
+            s = exprnode.value
+            if s.startswith('0'):
+                if s.startswith('0x') or s.startswith('0X'):
+                    return int(s, 16)
+                return int(s, 8)
+            elif '1' <= s[0] <= '9':
+                return int(s, 10)
+            elif s[0] == "'" and s[-1] == "'" and (
+                    len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
+                return ord(s[-2])
+            else:
+                raise api.CDefError("invalid constant %r" % (s,))
+        #
+        if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
+                exprnode.op == '+'):
+            return self._parse_constant(exprnode.expr)
+        #
+        if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
+                exprnode.op == '-'):
+            return -self._parse_constant(exprnode.expr)
+        # load previously defined int constant
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
+                exprnode.name in self._int_constants):
+            return self._int_constants[exprnode.name]
+        #
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
+                    exprnode.name == '__dotdotdotarray__'):
+            if partial_length_ok:
+                self._partial_length = True
+                return '...'
+            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
+                               "the actual array length in this context"
+                               % exprnode.coord.line)
+        #
+        raise api.FFIError(":%d: unsupported expression: expected a "
+                           "simple numeric constant" % exprnode.coord.line)
+
+    def _build_enum_type(self, explicit_name, decls):
+        if decls is not None:
+            partial = False
+            enumerators = []
+            enumvalues = []
+            nextenumvalue = 0
+            for enum in decls.enumerators:
+                if enum.value is not None:
+                    nextenumvalue = self._parse_constant(enum.value)
+                enumerators.append(enum.name)
+                enumvalues.append(nextenumvalue)
+                self._add_constants(enum.name, nextenumvalue)
+                nextenumvalue += 1
+            enumerators = tuple(enumerators)
+            enumvalues = tuple(enumvalues)
+            tp = model.EnumType(explicit_name, enumerators, enumvalues)
+            tp.partial = partial
+        else:   # opaque enum
+            tp = model.EnumType(explicit_name, (), ())
+        return tp
+
+    def include(self, other):
+        for name, (tp, quals) in other._declarations.items():
+            if name.startswith('anonymous $enum_$'):
+                continue   # fix for test_anonymous_enum_include
+            kind = name.split(' ', 1)[0]
+            if kind in ('struct', 'union', 'enum', 'anonymous', 'typedef', 'macro'):
+                self._declare(name, tp, included=True, quals=quals)
+        for k, v in other._int_constants.items():
+            self._add_constants(k, v)
+        for k, v in other._macros.items():
+            self._macros[k] = v
+
+CNAME_TO_LLTYPE = {
+    'char': rffi.CHAR,
+    'double': rffi.DOUBLE, 'long double': rffi.LONGDOUBLE,
+    'float': rffi.FLOAT, 'FILE': FILEP.TO}
+
+def add_inttypes():
+    for name in rffi.TYPES:
+        if name.startswith('unsigned'):
+            rname = 'u' + name[9:]
+        else:
+            rname = name
+        rname = rname.replace(' ', '').upper()
+        CNAME_TO_LLTYPE[name] = rfficache.platform.types[rname]
+
+add_inttypes()
+CNAME_TO_LLTYPE['int'] = rffi.INT_real
+
+def cname_to_lltype(name):
+    return CNAME_TO_LLTYPE[name]
+
+class DelayedStruct(object):
+    def __init__(self, name, fields, TYPE):
+        self.struct_name = name
+        self.type_name = None
+        self.fields = fields
+        self.TYPE = TYPE
+
+    def get_type_name(self):
+        if self.type_name is not None:
+            return self.type_name
+        elif not self.struct_name.startswith('$'):
+            return 'struct %s' % self.struct_name
+        else:
+            raise ValueError('Anonymous struct')
+
+    def __repr__(self):
+        return "<struct {struct_name}>".format(**vars(self))
+
+
+class CTypeSpace(object):
+    def __init__(self, parser=None, definitions=None, macros=None,
+                 headers=None, includes=None):
+        self.definitions = definitions if definitions is not None else {}
+        self.macros = macros if macros is not None else {}
+        self.structs = {}
+        self.ctx = parser if parser else Parser()
+        self.headers = headers if headers is not None else ['sys/types.h']
+        self.parsed_headers = []
+        self.sources = []
+        self._Config = type('Config', (object,), {})
+        self._TYPES = {}
+        self.includes = []
+        self.struct_typedefs = {}
+        self._handled = set()
+        self._frozen = False
+        if includes is not None:
+            for header in includes:
+                self.include(header)
+
+    def include(self, other):
+        self.ctx.include(other.ctx)
+        self.structs.update(other.structs)
+        self.includes.append(other)
+
+    def parse_source(self, source):
+        self.sources.append(source)
+        self.ctx.parse(source)
+        self.configure_types()
+
+    def parse_header(self, header_path):
+        self.headers.append(str(header_path))
+        self.parsed_headers.append(header_path)
+        self.ctx.parse(header_path.read())
+        self.configure_types()
+
+    def add_typedef(self, name, obj, quals):
+        assert name not in self.definitions
+        tp = self.convert_type(obj, quals)
+        if isinstance(tp, DelayedStruct):
+            if tp.type_name is None:
+                tp.type_name = name
+            tp = self.realize_struct(tp)
+        self.definitions[name] = tp
+
+    def add_macro(self, name, value):
+        assert name not in self.macros
+        self.macros[name] = value
+
+    def new_struct(self, obj):
+        if obj.name == '_IO_FILE':  # cffi weirdness
+            return cname_to_lltype('FILE')
+        struct = DelayedStruct(obj.name, None, lltype.ForwardReference())
+        # Cache it early, to avoid infinite recursion
+        self.structs[obj] = struct
+        if obj.fldtypes is not None:
+            struct.fields = zip(
+                obj.fldnames,
+                [self.convert_field(field) for field in obj.fldtypes])
+        return struct
+
+    def convert_field(self, obj):
+        tp = self.convert_type(obj)
+        if isinstance(tp, DelayedStruct):
+            tp = tp.TYPE
+        return tp
+
+    def realize_struct(self, struct):
+        type_name = struct.get_type_name()
+        configname = type_name.replace(' ', '__')
+        setattr(self._Config, configname,
+            rffi_platform.Struct(type_name, struct.fields))
+        self._TYPES[configname] = struct.TYPE
+        return struct.TYPE
+
+    def build_eci(self):
+        all_sources = []
+        for cts in self.includes:
+            all_sources.extend(cts.sources)
+        all_sources.extend(self.sources)
+        all_headers = self.headers
+        for x in self.includes:
+            for hdr in x.headers:
+                if hdr not in all_headers:
+                    all_headers.append(hdr)
+        return ExternalCompilationInfo(
+            post_include_bits=all_sources, includes=all_headers)
+
+    def configure_types(self):
+        for name, (obj, quals) in self.ctx._declarations.iteritems():
+            if obj in self.ctx._included_declarations:
+                continue
+            if name in self._handled:
+                continue
+            self._handled.add(name)
+            if name.startswith('typedef '):
+                name = name[8:]
+                self.add_typedef(name, obj, quals)
+            elif name.startswith('macro '):
+                name = name[6:]
+                self.add_macro(name, obj)
+        self._Config._compilation_info_ = self.build_eci()
+        for name, TYPE in rffi_platform.configure(self._Config).iteritems():
+            # hack: prevent the source from being pasted into common_header.h
+            del TYPE._hints['eci']
+            if name in self._TYPES:
+                self._TYPES[name].become(TYPE)
+                del self._TYPES[name]
+
+    def convert_type(self, obj, quals=0):
+        if isinstance(obj, model.PrimitiveType):
+            return cname_to_lltype(obj.name)
+        elif isinstance(obj, model.StructType):
+            if obj in self.structs:
+                return self.structs[obj]
+            return self.new_struct(obj)
+        elif isinstance(obj, model.PointerType):
+            TO = self.convert_type(obj.totype)
+            if TO is lltype.Void:
+                return rffi.VOIDP
+            elif isinstance(TO, DelayedStruct):
+                TO = TO.TYPE
+            if isinstance(TO, lltype.ContainerType):
+                return lltype.Ptr(TO)
+            else:
+                if obj.quals & model.Q_CONST:
+                    return lltype.Ptr(lltype.Array(
+                        TO, hints={'nolength': True, 'render_as_const': True}))
+                else:
+                    return rffi.CArrayPtr(TO)
+        elif isinstance(obj, model.FunctionPtrType):
+            if obj.ellipsis:
+                raise NotImplementedError
+            args = [self.convert_type(arg) for arg in obj.args]
+            res = self.convert_type(obj.result)
+            return lltype.Ptr(lltype.FuncType(args, res))
+        elif isinstance(obj, model.VoidType):
+            return lltype.Void
+        elif isinstance(obj, model.ArrayType):
+            return rffi.CFixedArray(self.convert_type(obj.item), obj.length)
+        else:
+            raise NotImplementedError
+
+    def gettype(self, cdecl):
+        obj = self.ctx.parse_type(cdecl)
+        result = self.convert_type(obj)
+        if isinstance(result, DelayedStruct):
+            result = result.TYPE
+        return result
+
+    def cast(self, cdecl, value):
+        return rffi.cast(self.gettype(cdecl), value)
+
+    def parse_func(self, cdecl):
+        cdecl = cdecl.strip()
+        if cdecl[-1] != ';':
+            cdecl += ';'
+        ast, _, _ = self.ctx._parse(cdecl)
+        decl = ast.ext[-1]
+        tp, quals = self.ctx._get_type_and_quals(decl.type, name=decl.name)
+        FUNCP = self.convert_type(tp.as_function_pointer())
+        return decl.name, FUNCP.TO
+
+    def _freeze_(self):
+        if self._frozen:
+            return True
+
+        @register_flow_sc(self.cast)
+        def sc_cast(ctx, v_decl, v_arg):
+            if not isinstance(v_decl, Constant):
+                raise FlowingError(
+                    "The first argument of cts.cast() must be a constant.")
+            TP = self.gettype(v_decl.value)
+            return ctx.appcall(rffi.cast, const(TP), v_arg)
+
+        @register_flow_sc(self.gettype)
+        def sc_gettype(ctx, v_decl):
+            if not isinstance(v_decl, Constant):
+                raise FlowingError(
+                    "The argument of cts.gettype() must be a constant.")
+            return const(self.gettype(v_decl.value))
+
+        self._frozen = True
+        return True
+
+
+def parse_source(source, includes=None, headers=None, configure_now=True):
+    cts = CTypeSpace(headers=headers, includes=includes)
+    cts.parse_source(source)
+    return cts
diff --git a/pypy/module/cpyext/include/descrobject.h b/pypy/module/cpyext/include/descrobject.h
--- a/pypy/module/cpyext/include/descrobject.h
+++ b/pypy/module/cpyext/include/descrobject.h
@@ -1,16 +1,5 @@
 #ifndef Py_DESCROBJECT_H
 #define Py_DESCROBJECT_H
-typedef PyObject *(*getter)(PyObject *, void *);
-typedef int (*setter)(PyObject *, PyObject *, void *);
-
-typedef struct PyGetSetDef {
-	char *name;
-	getter get;
-	setter set;
-	char *doc;
-	void *closure;
-} PyGetSetDef;
-
 
 #define PyDescr_COMMON \
     PyObject_HEAD \
diff --git a/pypy/module/cpyext/include/methodobject.h b/pypy/module/cpyext/include/methodobject.h
--- a/pypy/module/cpyext/include/methodobject.h
+++ b/pypy/module/cpyext/include/methodobject.h
@@ -7,20 +7,6 @@
 extern "C" {
 #endif
 
-typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
-typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
-                                             PyObject *);
-typedef PyObject *(*PyNoArgsFunction)(PyObject *);
-
-struct PyMethodDef {
-    const char  *ml_name;   /* The name of the built-in function/method */
-    PyCFunction  ml_meth;   /* The C function that implements it */
-    int          ml_flags;  /* Combination of METH_xxx flags, which mostly
-                               describe the args expected by the C func */
-    const char  *ml_doc;    /* The __doc__ attribute, or NULL */
-};
-typedef struct PyMethodDef PyMethodDef;
-
 typedef struct
 {
     PyObject_HEAD
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -7,6 +7,9 @@
 extern "C" {
 #endif
 
+
+#include <cpyext_object.h>
+
 #define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
 
 
@@ -16,29 +19,12 @@
 */
 #define staticforward static
 
-#define PyObject_HEAD  \
-    Py_ssize_t ob_refcnt;        \
-    Py_ssize_t ob_pypy_link;     \
-    struct _typeobject *ob_type;
-
-#define PyObject_VAR_HEAD		\
-	PyObject_HEAD			\
-	Py_ssize_t ob_size; /* Number of items in variable part */
-
 #define PyObject_HEAD_INIT(type)	\
 	1, 0, type,
 
 #define PyVarObject_HEAD_INIT(type, size)	\
 	PyObject_HEAD_INIT(type) size,
 
-typedef struct _object {
-    PyObject_HEAD
-} PyObject;
-
-typedef struct {
-	PyObject_VAR_HEAD
-} PyVarObject;
-
 #ifdef PYPY_DEBUG_REFCOUNT
 /* Slow version, but useful for debugging */
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)(ob)))
@@ -94,84 +80,7 @@
 #define Py_GT 4
 #define Py_GE 5
 
-struct _typeobject;
-typedef void (*freefunc)(void *);
-typedef void (*destructor)(PyObject *);
-typedef int (*printfunc)(PyObject *, FILE *, int);
-typedef PyObject *(*getattrfunc)(PyObject *, char *);
-typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
-typedef int (*setattrfunc)(PyObject *, char *, PyObject *);
-typedef int (*setattrofunc)(PyObject *, PyObject *, PyObject *);
-typedef int (*cmpfunc)(PyObject *, PyObject *);
-typedef PyObject *(*reprfunc)(PyObject *);
-typedef long (*hashfunc)(PyObject *);
-typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
-typedef PyObject *(*getiterfunc) (PyObject *);
-typedef PyObject *(*iternextfunc) (PyObject *);
-typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);
-typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *);
-typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
-typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
-typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t);
-
-typedef PyObject * (*unaryfunc)(PyObject *);
-typedef PyObject * (*binaryfunc)(PyObject *, PyObject *);
-typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
-typedef int (*inquiry)(PyObject *);
-typedef Py_ssize_t (*lenfunc)(PyObject *);
-typedef int (*coercion)(PyObject **, PyObject **);
-typedef PyObject *(*intargfunc)(PyObject *, int) Py_DEPRECATED(2.5);
-typedef PyObject *(*intintargfunc)(PyObject *, int, int) Py_DEPRECATED(2.5);
-typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
-typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
-typedef int(*intobjargproc)(PyObject *, int, PyObject *);
-typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *);
-typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
-typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
-typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
-
-
-/* int-based buffer interface */
-typedef int (*getreadbufferproc)(PyObject *, int, void **);
-typedef int (*getwritebufferproc)(PyObject *, int, void **);
-typedef int (*getsegcountproc)(PyObject *, int *);
-typedef int (*getcharbufferproc)(PyObject *, int, char **);
-/* ssize_t-based buffer interface */
-typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
-typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
-typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
-typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
-
 /* Py3k buffer interface, adapted for PyPy */
-#define Py_MAX_NDIMS 32
-#define Py_MAX_FMT 128
-typedef struct bufferinfo {
-    void *buf;
-    PyObject *obj;        /* owned reference */
-    Py_ssize_t len;
-
-    /* This is Py_ssize_t so it can be
-       pointed to by strides in simple case.*/
-    Py_ssize_t itemsize;
-    int readonly;
-    int ndim;
-    char *format;
-    Py_ssize_t *shape;
-    Py_ssize_t *strides;
-    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
-    unsigned char _format[Py_MAX_FMT];
-    Py_ssize_t _strides[Py_MAX_NDIMS];
-    Py_ssize_t _shape[Py_MAX_NDIMS];
-    /* static store for shape and strides of
-       mono-dimensional buffers. */
-    /* Py_ssize_t smalltable[2]; */
-    void *internal; /* always NULL for app-level objects */
-} Py_buffer;
-
-
-typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
-typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
-
     /* Flags for getting buffers */
 #define PyBUF_SIMPLE 0
 #define PyBUF_WRITABLE 0x0001
@@ -203,186 +112,7 @@
 #define PyBUF_SHADOW 0x400
 /* end Py3k buffer interface */
 
-typedef int (*objobjproc)(PyObject *, PyObject *);
-typedef int (*visitproc)(PyObject *, void *);
-typedef int (*traverseproc)(PyObject *, visitproc, void *);
-
-typedef struct {
-	/* For numbers without flag bit Py_TPFLAGS_CHECKTYPES set, all
-	   arguments are guaranteed to be of the object's type (modulo
-	   coercion hacks -- i.e. if the type's coercion function
-	   returns other types, then these are allowed as well).  Numbers that
-	   have the Py_TPFLAGS_CHECKTYPES flag bit set should check *both*
-	   arguments for proper type and implement the necessary conversions
-	   in the slot functions themselves. */
-
-	binaryfunc nb_add;
-	binaryfunc nb_subtract;
-	binaryfunc nb_multiply;
-	binaryfunc nb_divide;
-	binaryfunc nb_remainder;
-	binaryfunc nb_divmod;
-	ternaryfunc nb_power;
-	unaryfunc nb_negative;
-	unaryfunc nb_positive;
-	unaryfunc nb_absolute;
-	inquiry nb_nonzero;
-	unaryfunc nb_invert;
-	binaryfunc nb_lshift;
-	binaryfunc nb_rshift;
-	binaryfunc nb_and;
-	binaryfunc nb_xor;
-	binaryfunc nb_or;
-	coercion nb_coerce;
-	unaryfunc nb_int;
-	unaryfunc nb_long;
-	unaryfunc nb_float;
-	unaryfunc nb_oct;
-	unaryfunc nb_hex;
-	/* Added in release 2.0 */
-	binaryfunc nb_inplace_add;
-	binaryfunc nb_inplace_subtract;
-	binaryfunc nb_inplace_multiply;
-	binaryfunc nb_inplace_divide;
-	binaryfunc nb_inplace_remainder;
-	ternaryfunc nb_inplace_power;
-	binaryfunc nb_inplace_lshift;
-	binaryfunc nb_inplace_rshift;
-	binaryfunc nb_inplace_and;
-	binaryfunc nb_inplace_xor;
-	binaryfunc nb_inplace_or;
-
-	/* Added in release 2.2 */
-	/* The following require the Py_TPFLAGS_HAVE_CLASS flag */
-	binaryfunc nb_floor_divide;
-	binaryfunc nb_true_divide;
-	binaryfunc nb_inplace_floor_divide;
-	binaryfunc nb_inplace_true_divide;
-
-	/* Added in release 2.5 */
-	unaryfunc nb_index;
-} PyNumberMethods;
-
-typedef struct {
-	lenfunc sq_length;
-	binaryfunc sq_concat;
-	ssizeargfunc sq_repeat;
-	ssizeargfunc sq_item;
-	ssizessizeargfunc sq_slice;
-	ssizeobjargproc sq_ass_item;
-	ssizessizeobjargproc sq_ass_slice;
-	objobjproc sq_contains;
-	/* Added in release 2.0 */
-	binaryfunc sq_inplace_concat;
-	ssizeargfunc sq_inplace_repeat;
-} PySequenceMethods;
-
-typedef struct {
-	lenfunc mp_length;
-	binaryfunc mp_subscript;
-	objobjargproc mp_ass_subscript;
-} PyMappingMethods;
-
-typedef struct {
-	readbufferproc bf_getreadbuffer;
-	writebufferproc bf_getwritebuffer;
-	segcountproc bf_getsegcount;
-	charbufferproc bf_getcharbuffer;
-	getbufferproc bf_getbuffer;
-	releasebufferproc bf_releasebuffer;
-} PyBufferProcs;
-
-
-
-typedef struct _typeobject {
-	PyObject_VAR_HEAD
-	const char *tp_name; /* For printing, in format "<module>.<name>" */
-	Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
-
-	/* Methods to implement standard operations */
-
-	destructor tp_dealloc;
-	printfunc tp_print;
-	getattrfunc tp_getattr;
-	setattrfunc tp_setattr;
-	cmpfunc tp_compare;
-	reprfunc tp_repr;
-
-	/* Method suites for standard classes */
-
-	PyNumberMethods *tp_as_number;
-	PySequenceMethods *tp_as_sequence;
-	PyMappingMethods *tp_as_mapping;
-
-	/* More standard operations (here for binary compatibility) */
-
-	hashfunc tp_hash;
-	ternaryfunc tp_call;
-	reprfunc tp_str;
-	getattrofunc tp_getattro;
-	setattrofunc tp_setattro;
-
-	/* Functions to access object as input/output buffer */
-	PyBufferProcs *tp_as_buffer;
-
-	/* Flags to define presence of optional/expanded features */
-	long tp_flags;
-
-	const char *tp_doc; /* Documentation string */
-
-	/* Assigned meaning in release 2.0 */
-	/* call function for all accessible objects */
-	traverseproc tp_traverse;
-
-	/* delete references to contained objects */
-	inquiry tp_clear;
-
-	/* Assigned meaning in release 2.1 */
-	/* rich comparisons */
-	richcmpfunc tp_richcompare;
-
-	/* weak reference enabler */
-	Py_ssize_t tp_weaklistoffset;
-
-	/* Added in release 2.2 */
-	/* Iterators */
-	getiterfunc tp_iter;
-	iternextfunc tp_iternext;
-
-	/* Attribute descriptor and subclassing stuff */
-	struct PyMethodDef *tp_methods;
-	struct PyMemberDef *tp_members;
-	struct PyGetSetDef *tp_getset;
-	struct _typeobject *tp_base;
-	PyObject *tp_dict;
-	descrgetfunc tp_descr_get;
-	descrsetfunc tp_descr_set;
-	Py_ssize_t tp_dictoffset;
-	initproc tp_init;
-	allocfunc tp_alloc;
-	newfunc tp_new;
-	freefunc tp_free; /* Low-level free-memory routine */
-	inquiry tp_is_gc; /* For PyObject_IS_GC */
-	PyObject *tp_bases;
-	PyObject *tp_mro; /* method resolution order */
-	PyObject *tp_cache;
-	PyObject *tp_subclasses;
-	PyObject *tp_weaklist;
-	destructor tp_del;
-
-	/* Type attribute cache version tag. Added in version 2.6 */
-	unsigned int tp_version_tag;
-
-} PyTypeObject;
-
-typedef struct {
-    PyTypeObject ht_type;
-    PyNumberMethods as_number;
-    PyMappingMethods as_mapping;
-    PySequenceMethods as_sequence;
-    PyBufferProcs as_buffer;
-    PyObject *ht_name, *ht_slots;
-} PyHeapTypeObject;
+#include <cpyext_typeobject.h>
 
 #define PyObject_Bytes PyObject_Str
 
diff --git a/pypy/module/cpyext/include/structmember.h b/pypy/module/cpyext/include/structmember.h
--- a/pypy/module/cpyext/include/structmember.h
+++ b/pypy/module/cpyext/include/structmember.h
@@ -19,22 +19,6 @@
 #define offsetof(type, member) ( (int) & ((type*)0) -> member )
 #endif
 
-/* An array of memberlist structures defines the name, type and offset
-   of selected members of a C structure.  These can be read by
-   PyMember_Get() and set by PyMember_Set() (except if their READONLY flag
-   is set).  The array must be terminated with an entry whose name
-   pointer is NULL. */
-
-
-
-typedef struct PyMemberDef {
-    /* Current version, use this */
-    char *name;
-    int type;
-    Py_ssize_t offset;
-    int flags;
-    char *doc;
-} PyMemberDef;
 
 /* Types */
 #define T_SHORT         0
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,23 +11,13 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr, slot_function)
+    PyTypeObjectPtr, slot_function, cts, api_decl)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
-PyCFunction_typedef = rffi.COpaquePtr(typedef='PyCFunction')
-PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject))
-PyCFunctionKwArgs = lltype.Ptr(lltype.FuncType([PyObject, PyObject, PyObject],
-                                               PyObject))
-
-PyMethodDef = cpython_struct(
-    'PyMethodDef',
-    [('ml_name', rffi.CONST_CCHARP),
-     ('ml_meth', PyCFunction_typedef),
-     ('ml_flags', rffi.INT_real),
-     ('ml_doc', rffi.CONST_CCHARP),
-     ])
-
+PyMethodDef = cts.gettype('PyMethodDef')
+PyCFunction = cts.gettype('PyCFunction')
+PyCFunctionKwArgs = cts.gettype('PyCFunctionWithKeywords')
 PyCFunctionObjectStruct = cpython_struct(
     'PyCFunctionObject',
     PyObjectFields + (
@@ -77,7 +67,7 @@
             raise oefmt(space.w_TypeError,
                         "%s() takes no keyword arguments", self.name)
 
-        func = rffi.cast(PyCFunction, self.ml.c_ml_meth)
+        func = self.ml.c_ml_meth
         length = space.int_w(space.len(w_args))
         if flags & METH_KEYWORDS:
             func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth)
@@ -292,7 +282,7 @@
 def PyCFunction_NewEx(space, ml, w_self, w_name):
     return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name))
 
- at cpython_api([PyObject], PyCFunction_typedef)
+ at api_decl("PyCFunction PyCFunction_GetFunction(PyObject *)", cts)
 def PyCFunction_GetFunction(space, w_obj):
     try:
         cfunction = space.interp_w(W_PyCFunctionObject, w_obj)
@@ -347,4 +337,3 @@
     if name == "__methods__":
         return space.newlist(method_list_w)
     raise OperationError(space.w_AttributeError, space.wrap(name))
-
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -50,7 +50,7 @@
     cache.  CPython includes some extra checking here to make sure the module
     being initialized lines up with what's expected, but we don't.
     """
-    from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
+    from pypy.module.cpyext.api import PyTypeObjectPtr
     modname = rffi.charp2str(name)
     state = space.fromcache(State)
     f_name, f_path = state.package_context
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -0,0 +1,307 @@
+
+typedef ssize_t Py_ssize_t;
+
+#define PyObject_HEAD  \
+    Py_ssize_t ob_refcnt;        \
+    Py_ssize_t ob_pypy_link;     \
+    struct _typeobject *ob_type;
+
+#define PyObject_VAR_HEAD		\
+	PyObject_HEAD			\
+	Py_ssize_t ob_size; /* Number of items in variable part */
+
+typedef struct _object {
+    PyObject_HEAD
+} PyObject;
+
+typedef struct {
+	PyObject_VAR_HEAD
+} PyVarObject;
+
+struct _typeobject;
+typedef void (*freefunc)(void *);
+typedef void (*destructor)(PyObject *);
+typedef int (*printfunc)(PyObject *, FILE *, int);
+typedef PyObject *(*getattrfunc)(PyObject *, char *);
+typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
+typedef int (*setattrfunc)(PyObject *, char *, PyObject *);
+typedef int (*setattrofunc)(PyObject *, PyObject *, PyObject *);
+typedef int (*cmpfunc)(PyObject *, PyObject *);
+typedef PyObject *(*reprfunc)(PyObject *);
+typedef long (*hashfunc)(PyObject *);
+typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
+typedef PyObject *(*getiterfunc) (PyObject *);
+typedef PyObject *(*iternextfunc) (PyObject *);
+typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);
+typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *);
+typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
+typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
+typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t);
+
+typedef PyObject * (*unaryfunc)(PyObject *);
+typedef PyObject * (*binaryfunc)(PyObject *, PyObject *);
+typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
+typedef int (*inquiry)(PyObject *);
+typedef Py_ssize_t (*lenfunc)(PyObject *);
+typedef int (*coercion)(PyObject **, PyObject **);
+typedef PyObject *(*intargfunc)(PyObject *, int);
+typedef PyObject *(*intintargfunc)(PyObject *, int, int);
+typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
+typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
+typedef int(*intobjargproc)(PyObject *, int, PyObject *);
+typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *);
+typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
+typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
+typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
+
+
+/* int-based buffer interface */
+typedef int (*getreadbufferproc)(PyObject *, int, void **);
+typedef int (*getwritebufferproc)(PyObject *, int, void **);
+typedef int (*getsegcountproc)(PyObject *, int *);
+typedef int (*getcharbufferproc)(PyObject *, int, char **);
+/* ssize_t-based buffer interface */
+typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
+typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
+typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
+typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
+
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 128
+typedef struct bufferinfo {
+    void *buf;
+    PyObject *obj;        /* owned reference */
+    Py_ssize_t len;
+
+    /* This is Py_ssize_t so it can be
+       pointed to by strides in simple case.*/
+    Py_ssize_t itemsize;
+    int readonly;
+    int ndim;
+    char *format;
+    Py_ssize_t *shape;
+    Py_ssize_t *strides;
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format[Py_MAX_FMT];
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
+    /* static store for shape and strides of
+       mono-dimensional buffers. */
+    /* Py_ssize_t smalltable[2]; */
+    void *internal; /* always NULL for app-level objects */
+} Py_buffer;
+
+
+typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
+typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
+/* end Py3k buffer interface */
+
+typedef int (*objobjproc)(PyObject *, PyObject *);
+typedef int (*visitproc)(PyObject *, void *);
+typedef int (*traverseproc)(PyObject *, visitproc, void *);
+
+typedef struct {
+	/* For numbers without flag bit Py_TPFLAGS_CHECKTYPES set, all
+	   arguments are guaranteed to be of the object's type (modulo
+	   coercion hacks -- i.e. if the type's coercion function
+	   returns other types, then these are allowed as well).  Numbers that
+	   have the Py_TPFLAGS_CHECKTYPES flag bit set should check *both*
+	   arguments for proper type and implement the necessary conversions
+	   in the slot functions themselves. */
+
+	binaryfunc nb_add;
+	binaryfunc nb_subtract;
+	binaryfunc nb_multiply;
+	binaryfunc nb_divide;
+	binaryfunc nb_remainder;
+	binaryfunc nb_divmod;
+	ternaryfunc nb_power;
+	unaryfunc nb_negative;
+	unaryfunc nb_positive;
+	unaryfunc nb_absolute;
+	inquiry nb_nonzero;
+	unaryfunc nb_invert;
+	binaryfunc nb_lshift;
+	binaryfunc nb_rshift;
+	binaryfunc nb_and;
+	binaryfunc nb_xor;
+	binaryfunc nb_or;
+	coercion nb_coerce;
+	unaryfunc nb_int;
+	unaryfunc nb_long;
+	unaryfunc nb_float;
+	unaryfunc nb_oct;
+	unaryfunc nb_hex;
+	/* Added in release 2.0 */
+	binaryfunc nb_inplace_add;
+	binaryfunc nb_inplace_subtract;
+	binaryfunc nb_inplace_multiply;
+	binaryfunc nb_inplace_divide;
+	binaryfunc nb_inplace_remainder;
+	ternaryfunc nb_inplace_power;
+	binaryfunc nb_inplace_lshift;
+	binaryfunc nb_inplace_rshift;
+	binaryfunc nb_inplace_and;
+	binaryfunc nb_inplace_xor;
+	binaryfunc nb_inplace_or;
+
+	/* Added in release 2.2 */
+	/* The following require the Py_TPFLAGS_HAVE_CLASS flag */
+	binaryfunc nb_floor_divide;
+	binaryfunc nb_true_divide;
+	binaryfunc nb_inplace_floor_divide;
+	binaryfunc nb_inplace_true_divide;
+
+	/* Added in release 2.5 */
+	unaryfunc nb_index;
+} PyNumberMethods;
+
+typedef struct {
+	lenfunc sq_length;
+	binaryfunc sq_concat;
+	ssizeargfunc sq_repeat;
+	ssizeargfunc sq_item;
+	ssizessizeargfunc sq_slice;
+	ssizeobjargproc sq_ass_item;
+	ssizessizeobjargproc sq_ass_slice;
+	objobjproc sq_contains;
+	/* Added in release 2.0 */
+	binaryfunc sq_inplace_concat;
+	ssizeargfunc sq_inplace_repeat;
+} PySequenceMethods;
+
+typedef struct {
+	lenfunc mp_length;
+	binaryfunc mp_subscript;
+	objobjargproc mp_ass_subscript;
+} PyMappingMethods;
+
+typedef struct {
+	readbufferproc bf_getreadbuffer;
+	writebufferproc bf_getwritebuffer;
+	segcountproc bf_getsegcount;
+	charbufferproc bf_getcharbuffer;
+	getbufferproc bf_getbuffer;
+	releasebufferproc bf_releasebuffer;
+} PyBufferProcs;
+
+/* from descrobject.h */
+typedef PyObject *(*getter)(PyObject *, void *);
+typedef int (*setter)(PyObject *, PyObject *, void *);
+
+typedef struct PyGetSetDef {
+	char *name;
+	getter get;
+	setter set;
+	char *doc;
+	void *closure;
+} PyGetSetDef;
+
+/* from methodobject.h */
+typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
+typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
+                                             PyObject *);
+typedef PyObject *(*PyNoArgsFunction)(PyObject *);
+
+struct PyMethodDef {
+    const char  *ml_name;   /* The name of the built-in function/method */
+    PyCFunction  ml_meth;   /* The C function that implements it */
+    int          ml_flags;  /* Combination of METH_xxx flags, which mostly
+                               describe the args expected by the C func */
+    const char  *ml_doc;    /* The __doc__ attribute, or NULL */
+};
+typedef struct PyMethodDef PyMethodDef;
+
+/* from structmember.h */
+typedef struct PyMemberDef {
+    /* Current version, use this */
+    char *name;
+    int type;
+    Py_ssize_t offset;
+    int flags;
+    char *doc;
+} PyMemberDef;
+
+
+typedef struct _typeobject {
+	PyObject_VAR_HEAD
+	const char *tp_name; /* For printing, in format "<module>.<name>" */
+	Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
+
+	/* Methods to implement standard operations */
+
+	destructor tp_dealloc;
+	printfunc tp_print;
+	getattrfunc tp_getattr;
+	setattrfunc tp_setattr;
+	cmpfunc tp_compare;
+	reprfunc tp_repr;
+
+	/* Method suites for standard classes */
+
+	PyNumberMethods *tp_as_number;
+	PySequenceMethods *tp_as_sequence;
+	PyMappingMethods *tp_as_mapping;
+
+	/* More standard operations (here for binary compatibility) */
+
+	hashfunc tp_hash;
+	ternaryfunc tp_call;
+	reprfunc tp_str;
+	getattrofunc tp_getattro;
+	setattrofunc tp_setattro;
+
+	/* Functions to access object as input/output buffer */
+	PyBufferProcs *tp_as_buffer;
+
+	/* Flags to define presence of optional/expanded features */
+	long tp_flags;
+
+	const char *tp_doc; /* Documentation string */
+
+	/* Assigned meaning in release 2.0 */
+	/* call function for all accessible objects */
+	traverseproc tp_traverse;
+
+	/* delete references to contained objects */
+	inquiry tp_clear;
+
+	/* Assigned meaning in release 2.1 */
+	/* rich comparisons */
+	richcmpfunc tp_richcompare;
+
+	/* weak reference enabler */
+	Py_ssize_t tp_weaklistoffset;
+
+	/* Added in release 2.2 */
+	/* Iterators */
+	getiterfunc tp_iter;
+	iternextfunc tp_iternext;
+
+	/* Attribute descriptor and subclassing stuff */
+	struct PyMethodDef *tp_methods;
+	struct PyMemberDef *tp_members;
+	struct PyGetSetDef *tp_getset;
+	struct _typeobject *tp_base;
+	PyObject *tp_dict;
+	descrgetfunc tp_descr_get;
+	descrsetfunc tp_descr_set;
+	Py_ssize_t tp_dictoffset;
+	initproc tp_init;
+	allocfunc tp_alloc;
+	newfunc tp_new;
+	freefunc tp_free; /* Low-level free-memory routine */
+	inquiry tp_is_gc; /* For PyObject_IS_GC */
+	PyObject *tp_bases;
+	PyObject *tp_mro; /* method resolution order */
+	PyObject *tp_cache;
+	PyObject *tp_subclasses;
+	PyObject *tp_weaklist;
+	destructor tp_del;
+
+	/* Type attribute cache version tag. Added in version 2.6 */
+	unsigned int tp_version_tag;
+
+} PyTypeObject;
+
diff --git a/pypy/module/cpyext/parse/cpyext_typeobject.h b/pypy/module/cpyext/parse/cpyext_typeobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_typeobject.h
@@ -0,0 +1,8 @@
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,9 +7,9 @@
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
     slot_function, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    pypy_decl, Py_buffer, Py_bufferP)
+    pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr)
 from pypy.module.cpyext.typeobjectdefs import (
-    unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
+    unaryfunc, ternaryfunc, binaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -5,7 +5,7 @@
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext.api import (
     slot_function, cpython_api, copy_header_files, INTERPLEVEL_API,
-    Py_ssize_t, Py_ssize_tP, PyObject)
+    Py_ssize_t, Py_ssize_tP, PyObject, cts)
 from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest
 from pypy.interpreter.error import OperationError
 from rpython.rlib import rawrefcount
@@ -99,10 +99,10 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert PyPy_TypedefTest1.api_func.get_c_restype(db) == 'Py_ssize_t'
-        assert PyPy_TypedefTest1.api_func.get_c_args(db) == 'Py_ssize_t arg0'
-        assert PyPy_TypedefTest2.api_func.get_c_restype(db) == 'Py_ssize_t *'
-        assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Py_ssize_t *arg0'
+        assert PyPy_TypedefTest1.api_func.get_c_restype(db) == 'Signed'
+        assert PyPy_TypedefTest1.api_func.get_c_args(db) == 'Signed arg0'
+        assert PyPy_TypedefTest2.api_func.get_c_restype(db) == 'Signed *'
+        assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Signed *arg0'
 
         PyPy_TypedefTest1(space, 0)
         ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
@@ -113,7 +113,7 @@
 @pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
-    copy_header_files(tmpdir, True)
+    copy_header_files(cts, tmpdir, True)
     def check(name):
         f = tmpdir.join(name)
         assert f.check(file=True)
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -12,6 +12,7 @@
 from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref
 from pypy.module.cpyext.object import PyObject_AsCharBuffer
+from pypy.module.cpyext.api import PyTypeObjectPtr
 
 
 class AppTestBytesObject(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -0,0 +1,218 @@
+from rpython.flowspace.model import const
+from rpython.flowspace.objspace import build_flow
+from rpython.translator.simplify import simplify_graph
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.module.cpyext.cparser import parse_source, CTypeSpace
+
+def test_configure():
+    decl = """
+    typedef ssize_t Py_ssize_t;
+
+    typedef struct {
+        Py_ssize_t ob_refcnt;
+        Py_ssize_t ob_pypy_link;
+        double ob_fval;
+    } TestFloatObject;
+    """
+    cts = parse_source(decl)
+    TestFloatObject = cts.definitions['TestFloatObject']
+    assert isinstance(TestFloatObject, lltype.Struct)
+    assert TestFloatObject.c_ob_refcnt == rffi.SSIZE_T
+    assert TestFloatObject.c_ob_pypy_link == rffi.SSIZE_T
+    assert TestFloatObject.c_ob_fval == rffi.DOUBLE
+
+def test_simple():
+    decl = "typedef ssize_t Py_ssize_t;"
+    cts = parse_source(decl)
+    assert cts.definitions == {'Py_ssize_t': rffi.SSIZE_T}
+
+def test_macro():
+    decl = """
+    typedef ssize_t Py_ssize_t;
+
+    #define PyObject_HEAD  \
+        Py_ssize_t ob_refcnt;        \
+        Py_ssize_t ob_pypy_link;     \
+
+    typedef struct {
+        PyObject_HEAD
+        double ob_fval;
+    } PyFloatObject;

From pypy.commits at gmail.com  Tue Jan 17 13:00:13 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 10:00:13 -0800 (PST)
Subject: [pypy-commit] pypy rffi-parser: Close branch rffi-parser.
Message-ID: <587e5bad.952f1c0a.d5983.6151@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rffi-parser
Changeset: r89643:3b3611054de9
Date: 2017-01-17 17:59 +0000
http://bitbucket.org/pypy/pypy/changeset/3b3611054de9/

Log:	Close branch rffi-parser.


From pypy.commits at gmail.com  Tue Jan 17 13:44:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 10:44:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <587e6623.05a81c0a.fba8d.4eeb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89644:573fc5d78d57
Date: 2017-01-17 18:44 +0000
http://bitbucket.org/pypy/pypy/changeset/573fc5d78d57/

Log:	hg merge default

diff too long, truncating to 2000 out of 2639 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -128,3 +128,9 @@
 Also fix for c-extension type that calls ``tp_hash`` during initialization
 (str, unicode types), and fix instantiating c-extension types from built-in
 classes by enforcing an order of instaniation.
+
+.. branch: rffi-parser-2
+
+rffi structures in cpyext can now be created by parsing simple C headers.
+Additionally, the cts object that holds the parsed information can act like
+cffi's ffi objects, with the methods cts.cast() and cts.gettype().
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -251,12 +251,11 @@
 
 class GetSetProperty(W_Root):
     _immutable_fields_ = ["fget", "fset", "fdel"]
-    name = '<generic property>'
     w_objclass = None
 
     @specialize.arg(7)
     def __init__(self, fget, fset=None, fdel=None, doc=None,
-                 cls=None, use_closure=False, tag=None):
+                 cls=None, use_closure=False, tag=None, name=None):
         objclass_getter, cls = make_objclass_getter(tag, fget, cls)
         fget = make_descr_typecheck_wrapper((tag, 0), fget,
                                             cls=cls, use_closure=use_closure)
@@ -264,9 +263,11 @@
                                             cls=cls, use_closure=use_closure)
         fdel = make_descr_typecheck_wrapper((tag, 2), fdel,
                                             cls=cls, use_closure=use_closure)
-        self._init(fget, fset, fdel, doc, cls, objclass_getter, use_closure)
+        self._init(fget, fset, fdel, doc, cls, objclass_getter, use_closure,
+                   name)
 
-    def _init(self, fget, fset, fdel, doc, cls, objclass_getter, use_closure):
+    def _init(self, fget, fset, fdel, doc, cls, objclass_getter, use_closure,
+              name):
         self.fget = fget
         self.fset = fset
         self.fdel = fdel
@@ -275,13 +276,13 @@
         self.qualname = None
         self.objclass_getter = objclass_getter
         self.use_closure = use_closure
+        self.name = name if name is not None else '<generic property>'
 
     def copy_for_type(self, w_objclass):
         if self.objclass_getter is None:
             new = instantiate(GetSetProperty)
             new._init(self.fget, self.fset, self.fdel, self.doc, self.reqcls,
-                      None, self.use_closure)
-            new.name = self.name
+                      None, self.use_closure, self.name)
             new.w_objclass = w_objclass
             return new
         else:
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,37 +40,35 @@
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
+from pypy.module.cpyext.cparser import CTypeSpace
 
 DEBUG_WRAPPER = True
 
-# update these for other platforms
-Py_ssize_t = lltype.Typedef(rffi.SSIZE_T, 'Py_ssize_t')
-Py_ssize_tP = rffi.CArrayPtr(Py_ssize_t)
-size_t = rffi.ULONG
-ADDR = lltype.Signed
-
 pypydir = py.path.local(pypydir)
 include_dir = pypydir / 'module' / 'cpyext' / 'include'
+parse_dir = pypydir / 'module' / 'cpyext' / 'parse'
 source_dir = pypydir / 'module' / 'cpyext' / 'src'
 translator_c_dir = py.path.local(cdir)
 include_dirs = [
     include_dir,
+    parse_dir,
     translator_c_dir,
     udir,
     ]
 
-class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
+configure_eci = ExternalCompilationInfo(
         include_dirs=include_dirs,
         includes=['Python.h', 'stdarg.h', 'structmember.h'],
-        compile_extra=['-DPy_BUILD_CORE'],
-        )
+        compile_extra=['-DPy_BUILD_CORE'])
+
+class CConfig:
+    _compilation_info_ = configure_eci
 
 class CConfig2:
-    _compilation_info_ = CConfig._compilation_info_
+    _compilation_info_ = configure_eci
 
 class CConfig_constants:
-    _compilation_info_ = CConfig._compilation_info_
+    _compilation_info_ = configure_eci
 
 CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
                                        hints={'nolength': True}),
@@ -117,6 +115,9 @@
     return is_valid_fd(c_fileno(fp))
 
 pypy_decl = 'pypy_decl.h'
+udir.join(pypy_decl).write("/* Will be filled later */\n")
+udir.join('pypy_structmember_decl.h').write("/* Will be filled later */\n")
+udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
@@ -129,9 +130,6 @@
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
-udir.join(pypy_decl).write("/* Will be filled later */\n")
-udir.join('pypy_structmember_decl.h').write("/* Will be filled later */\n")
-udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 globals().update(rffi_platform.configure(CConfig_constants))
 
 def _copy_header_files(headers, dstdir):
@@ -145,12 +143,14 @@
         target.chmod(0444) # make the file read-only, to make sure that nobody
                            # edits it by mistake
 
-def copy_header_files(dstdir, copy_numpy_headers):
+def copy_header_files(cts, dstdir, copy_numpy_headers):
     # XXX: 20 lines of code to recursively copy a directory, really??
     assert dstdir.check(dir=True)
     headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl')
     for name in ["pypy_macros.h"] + FUNCTIONS_BY_HEADER.keys():
         headers.append(udir.join(name))
+    for path in cts.parsed_headers:
+        headers.append(path)
     _copy_header_files(headers, dstdir)
 
     if copy_numpy_headers:
@@ -251,13 +251,15 @@
 
 class ApiFunction(object):
     def __init__(self, argtypes, restype, callable, error=CANNOT_FAIL,
-                 c_name=None, gil=None, result_borrowed=False, result_is_ll=False):
+                 c_name=None, cdecl=None, gil=None,
+                 result_borrowed=False, result_is_ll=False):
         self.argtypes = argtypes
         self.restype = restype
         self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
         self.callable = callable
         self.error_value = error
         self.c_name = c_name
+        self.cdecl = cdecl
 
         # extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
@@ -272,8 +274,6 @@
         self.gil = gil
         self.result_borrowed = result_borrowed
         self.result_is_ll = result_is_ll
-        if result_is_ll:    # means 'returns a low-level PyObject pointer'
-            assert is_PyObject(restype)
         #
         def get_llhelper(space):
             return llhelper(self.functype, self.get_wrapper(space))
@@ -381,6 +381,8 @@
         return unwrapper
 
     def get_c_restype(self, c_writer):
+        if self.cdecl:
+            return self.cdecl.split(self.c_name)[0].strip()
         return c_writer.gettype(self.restype).replace('@', '').strip()
 
     def get_c_args(self, c_writer):
@@ -480,6 +482,20 @@
         return unwrapper
     return decorate
 
+def api_decl(cdecl, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+    def decorate(func):
+        func._always_inline_ = 'try'
+        name, FUNC = cts.parse_func(cdecl)
+        api_function = ApiFunction(
+            FUNC.ARGS, FUNC.RESULT, func,
+            error=_compute_error(error, FUNC.RESULT), cdecl=cdecl)
+        FUNCTIONS_BY_HEADER[header][name] = api_function
+        unwrapper = api_function.get_unwrapper()
+        unwrapper.func = func
+        unwrapper.api_func = api_function
+        return unwrapper
+    return decorate
+
 def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
     def decorate(func):
         func._always_inline_ = 'try'
@@ -658,41 +674,31 @@
                              % (cpyname, ))
 build_exported_objects()
 
+cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h'])
+cts.parse_header(parse_dir / 'cpyext_object.h')
+
+Py_ssize_t = cts.gettype('Py_ssize_t')
+Py_ssize_tP = cts.gettype('Py_ssize_t *')
+size_t = rffi.ULONG
+ADDR = lltype.Signed
+
 # Note: as a special case, "PyObject" is the pointer type in RPython,
 # corresponding to "PyObject *" in C.  We do that only for PyObject.
 # For example, "PyTypeObject" is the struct type even in RPython.
-PyTypeObject = lltype.ForwardReference()
-PyTypeObjectPtr = lltype.Ptr(PyTypeObject)
-PyObjectStruct = lltype.ForwardReference()
-PyObject = lltype.Ptr(PyObjectStruct)
+PyTypeObject = cts.gettype('PyTypeObject')
+PyTypeObjectPtr = cts.gettype('PyTypeObject *')
+PyObjectStruct = cts.gettype('PyObject')
+PyObject = cts.gettype('PyObject *')
 PyObjectFields = (("ob_refcnt", lltype.Signed),
                   ("ob_pypy_link", lltype.Signed),
                   ("ob_type", PyTypeObjectPtr))
 PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), )
-cpython_struct('PyObject', PyObjectFields, PyObjectStruct)
-PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields)
-PyVarObject = lltype.Ptr(PyVarObjectStruct)
+PyVarObjectStruct = cts.gettype('PyVarObject')
+PyVarObject = cts.gettype('PyVarObject *')
 
-Py_buffer = cpython_struct(
-    "Py_buffer", (
-        ('buf', rffi.VOIDP),
-        ('obj', PyObject),
-        ('len', Py_ssize_t),
-        ('itemsize', Py_ssize_t),
+Py_buffer = cts.gettype('Py_buffer')
+Py_bufferP = cts.gettype('Py_buffer *')
 
-        ('readonly', rffi.INT_real),
-        ('ndim', rffi.INT_real),
-        ('format', rffi.CCHARP),
-        ('shape', Py_ssize_tP),
-        ('strides', Py_ssize_tP),
-        ('suboffsets', Py_ssize_tP),
-        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
-        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
-        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
-        #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
-        ('internal', rffi.VOIDP),
-))
-Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
@@ -1415,7 +1421,7 @@
         include_lines.append('RPY_EXPORTED %s %s;\n' % (typ, name))
 
     lines.append('};\n')
-    eci2 = CConfig._compilation_info_.merge(ExternalCompilationInfo(
+    eci2 = configure_eci.merge(ExternalCompilationInfo(
         separate_module_sources = [''.join(lines)],
         post_include_bits = [''.join(include_lines)],
         ))
@@ -1433,7 +1439,7 @@
 
     setup_init_functions(eci, prefix)
     trunk_include = pypydir.dirpath() / 'include'
-    copy_header_files(trunk_include, use_micronumpy)
+    copy_header_files(cts, trunk_include, use_micronumpy)
 
 
 def _load_from_cffi(space, name, path, initptr):
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/cparser.py
@@ -0,0 +1,879 @@
+from collections import OrderedDict
+from cffi import api, model
+from cffi.commontypes import COMMON_TYPES, resolve_common_type
+try:
+    from cffi import _pycparser as pycparser
+except ImportError:
+    import pycparser
+import weakref, re
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib.rfile import FILEP
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.tool import rfficache, rffi_platform
+from rpython.flowspace.model import Constant, const
+from rpython.flowspace.specialcase import register_flow_sc
+from rpython.flowspace.flowcontext import FlowingError
+
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
+_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
+_r_words = re.compile(r"\w+|\S")
+_parser_cache = None
+_r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE)
+_r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b")
+_r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b")
+_r_cdecl = re.compile(r"\b__cdecl\b")
+_r_star_const_space = re.compile(       # matches "* const "
+    r"[*]\s*((const|volatile|restrict)\b\s*)+")
+
+def _get_parser():
+    global _parser_cache
+    if _parser_cache is None:
+        _parser_cache = pycparser.CParser()
+    return _parser_cache
+
+def _preprocess(csource, macros):
+    # Remove comments.  NOTE: this only work because the cdef() section
+    # should not contain any string literal!
+    csource = _r_comment.sub(' ', csource)
+    # Remove the "#define FOO x" lines
+    for match in _r_define.finditer(csource):
+        macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
+        macros[macroname] = macrovalue
+    csource = _r_define.sub('', csource)
+    #
+    # BIG HACK: replace WINAPI or __stdcall with "volatile const".
+    # It doesn't make sense for the return type of a function to be
+    # "volatile volatile const", so we abuse it to detect __stdcall...
+    # Hack number 2 is that "int(volatile *fptr)();" is not valid C
+    # syntax, so we place the "volatile" before the opening parenthesis.
+    csource = _r_stdcall2.sub(' volatile volatile const(', csource)
+    csource = _r_stdcall1.sub(' volatile volatile const ', csource)
+    csource = _r_cdecl.sub(' ', csource)
+
+    for name, value in reversed(macros.items()):
+        csource = re.sub(r'\b%s\b' % name, value, csource)
+
+    return csource, macros
+
+def _common_type_names(csource):
+    # Look in the source for what looks like usages of types from the
+    # list of common types.  A "usage" is approximated here as the
+    # appearance of the word, minus a "definition" of the type, which
+    # is the last word in a "typedef" statement.  Approximative only
+    # but should be fine for all the common types.
+    look_for_words = set(COMMON_TYPES)
+    look_for_words.add(';')
+    look_for_words.add(',')
+    look_for_words.add('(')
+    look_for_words.add(')')
+    look_for_words.add('typedef')
+    words_used = set()
+    is_typedef = False
+    paren = 0
+    previous_word = ''
+    for word in _r_words.findall(csource):
+        if word in look_for_words:
+            if word == ';':
+                if is_typedef:
+                    words_used.discard(previous_word)
+                    look_for_words.discard(previous_word)
+                    is_typedef = False
+            elif word == 'typedef':
+                is_typedef = True
+                paren = 0
+            elif word == '(':
+                paren += 1
+            elif word == ')':
+                paren -= 1
+            elif word == ',':
+                if is_typedef and paren == 0:
+                    words_used.discard(previous_word)
+                    look_for_words.discard(previous_word)
+            else:   # word in COMMON_TYPES
+                words_used.add(word)
+        previous_word = word
+    return words_used
+
+
+class Parser(object):
+
+    def __init__(self):
+        self._declarations = {}
+        self._included_declarations = set()
+        self._anonymous_counter = 0
+        self._structnode2type = weakref.WeakKeyDictionary()
+        self._options = {}
+        self._int_constants = {}
+        self._recomplete = []
+        self._macros = OrderedDict()
+
+    def _parse(self, csource):
+        # modifies self._macros in-place
+        csource, macros = _preprocess(csource, self._macros)
+        # XXX: for more efficiency we would need to poke into the
+        # internals of CParser...  the following registers the
+        # typedefs, because their presence or absence influences the
+        # parsing itself (but what they are typedef'ed to plays no role)
+        ctn = _common_type_names(csource)
+        typenames = []
+        for name in sorted(self._declarations):
+            if name.startswith('typedef '):
+                name = name[8:]
+                typenames.append(name)
+                ctn.discard(name)
+        typenames += sorted(ctn)
+        #
+        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines.append('typedef int __dotdotdot__;')
+        csourcelines.append(csource)
+        csource = '\n'.join(csourcelines)
+        try:
+            ast = _get_parser().parse(csource)
+        except pycparser.c_parser.ParseError as e:
+            self.convert_pycparser_error(e, csource)
+        # csource will be used to find buggy source text
+        return ast, macros, csource
+
+    def _convert_pycparser_error(self, e, csource):
+        # xxx look for ":NUM:" at the start of str(e) and try to interpret
+        # it as a line number
+        line = None
+        msg = str(e)
+        if msg.startswith(':') and ':' in msg[1:]:
+            linenum = msg[1:msg.find(':',1)]
+            if linenum.isdigit():
+                linenum = int(linenum, 10)
+                csourcelines = csource.splitlines()
+                if 1 <= linenum <= len(csourcelines):
+                    line = csourcelines[linenum-1]
+        return line
+
+    def convert_pycparser_error(self, e, csource):
+        line = self._convert_pycparser_error(e, csource)
+
+        msg = str(e)
+        if line:
+            msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
+        else:
+            msg = 'parse error\n%s' % (msg,)
+        raise api.CDefError(msg)
+
+    def parse(self, csource, override=False, packed=False, dllexport=False):
+        prev_options = self._options
+        try:
+            self._options = {'override': override,
+                             'packed': packed,
+                             'dllexport': dllexport}
+            self._internal_parse(csource)
+        finally:
+            self._options = prev_options
+
+    def _internal_parse(self, csource):
+        ast, macros, csource = self._parse(csource)
+        # add the macros
+        self._process_macros(macros)
+        # find the first "__dotdotdot__" and use that as a separator
+        # between the repeated typedefs and the real csource
+        iterator = iter(ast.ext)
+        for decl in iterator:
+            if decl.name == '__dotdotdot__':
+                break
+        #
+        try:
+            for decl in iterator:
+                if isinstance(decl, pycparser.c_ast.Decl):
+                    self._parse_decl(decl)
+                elif isinstance(decl, pycparser.c_ast.Typedef):
+                    if not decl.name:
+                        raise api.CDefError("typedef does not declare any name",
+                                            decl)
+                    quals = 0
+                    realtype, quals = self._get_type_and_quals(
+                        decl.type, name=decl.name, partial_length_ok=True)
+                    self._declare('typedef ' + decl.name, realtype, quals=quals)
+                elif decl.__class__.__name__ == 'Pragma':
+                    pass    # skip pragma, only in pycparser 2.15
+                else:
+                    raise api.CDefError("unrecognized construct", decl)
+        except api.FFIError as e:
+            msg = self._convert_pycparser_error(e, csource)
+            if msg:
+                e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
+            raise
+
+    def _add_constants(self, key, val):
+        if key in self._int_constants:
+            if self._int_constants[key] == val:
+                return     # ignore identical double declarations
+            raise api.FFIError(
+                "multiple declarations of constant: %s" % (key,))
+        self._int_constants[key] = val
+
+    def _add_integer_constant(self, name, int_str):
+        int_str = int_str.lower().rstrip("ul")
+        neg = int_str.startswith('-')
+        if neg:
+            int_str = int_str[1:]
+        # "010" is not valid oct in py3
+        if (int_str.startswith("0") and int_str != '0'
+                and not int_str.startswith("0x")):
+            int_str = "0o" + int_str[1:]
+        pyvalue = int(int_str, 0)
+        if neg:
+            pyvalue = -pyvalue
+        self._add_constants(name, pyvalue)
+        self._declare('macro ' + name, pyvalue)
+
+    def _process_macros(self, macros):
+        for key, value in macros.items():
+            value = value.strip()
+            if _r_int_literal.match(value):
+                self._add_integer_constant(key, value)
+            else:
+                self._declare('macro ' + key, value)
+
+    def _declare_function(self, tp, quals, decl):
+        tp = self._get_type_pointer(tp, quals)
+        if self._options.get('dllexport'):
+            tag = 'dllexport_python '
+        else:
+            tag = 'function '
+        self._declare(tag + decl.name, tp)
+
+    def _parse_decl(self, decl):
+        node = decl.type
+        if isinstance(node, pycparser.c_ast.FuncDecl):
+            tp, quals = self._get_type_and_quals(node, name=decl.name)
+            assert isinstance(tp, model.RawFunctionType)
+            self._declare_function(tp, quals, decl)
+        else:
+            if isinstance(node, pycparser.c_ast.Struct):
+                self._get_struct_union_enum_type('struct', node)
+            elif isinstance(node, pycparser.c_ast.Union):
+                self._get_struct_union_enum_type('union', node)
+            elif isinstance(node, pycparser.c_ast.Enum):
+                self._get_struct_union_enum_type('enum', node)
+            elif not decl.name:
+                raise api.CDefError("construct does not declare any variable",
+                                    decl)
+            #
+            if decl.name:
+                tp, quals = self._get_type_and_quals(node,
+                                                     partial_length_ok=True)
+                if tp.is_raw_function:
+                    self._declare_function(tp, quals, decl)
+                elif (tp.is_integer_type() and
+                        hasattr(decl, 'init') and
+                        hasattr(decl.init, 'value') and
+                        _r_int_literal.match(decl.init.value)):
+                    self._add_integer_constant(decl.name, decl.init.value)
+                elif (tp.is_integer_type() and
+                        isinstance(decl.init, pycparser.c_ast.UnaryOp) and
+                        decl.init.op == '-' and
+                        hasattr(decl.init.expr, 'value') and
+                        _r_int_literal.match(decl.init.expr.value)):
+                    self._add_integer_constant(decl.name,
+                                               '-' + decl.init.expr.value)
+                else:
+                    if (quals & model.Q_CONST) and not tp.is_array_type:
+                        self._declare('constant ' + decl.name, tp, quals=quals)
+                    else:
+                        self._declare('variable ' + decl.name, tp, quals=quals)
+
+    def parse_type(self, cdecl):
+        return self.parse_type_and_quals(cdecl)[0]
+
+    def parse_type_and_quals(self, cdecl):
+        ast, _, _ = self._parse('void __dummy(\n%s\n);' % cdecl)
+        exprnode = ast.ext[-1].type.args.params[0]
+        if isinstance(exprnode, pycparser.c_ast.ID):
+            raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
+        return self._get_type_and_quals(exprnode.type)
+
+    def _declare(self, name, obj, included=False, quals=0):
+        if name in self._declarations:
+            prevobj, prevquals = self._declarations[name]
+            if prevobj is obj and prevquals == quals:
+                return
+        self._declarations[name] = (obj, quals)
+        if included:
+            self._included_declarations.add(obj)
+
+    def _extract_quals(self, type):
+        quals = 0
+        if isinstance(type, (pycparser.c_ast.TypeDecl,
+                             pycparser.c_ast.PtrDecl)):
+            if 'const' in type.quals:
+                quals |= model.Q_CONST
+            if 'volatile' in type.quals:
+                quals |= model.Q_VOLATILE
+            if 'restrict' in type.quals:
+                quals |= model.Q_RESTRICT
+        return quals
+
+    def _get_type_pointer(self, type, quals, declname=None):
+        if isinstance(type, model.RawFunctionType):
+            return type.as_function_pointer()
+        if (isinstance(type, model.StructOrUnionOrEnum) and
+                type.name.startswith('$') and type.name[1:].isdigit() and
+                type.forcename is None and declname is not None):
+            return model.NamedPointerType(type, declname, quals)
+        return model.PointerType(type, quals)
+
+    def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False):
+        # first, dereference typedefs, if we have it already parsed, we're good
+        if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
+            isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
+            len(typenode.type.names) == 1 and
+            ('typedef ' + typenode.type.names[0]) in self._declarations):
+            tp, quals = self._declarations['typedef ' + typenode.type.names[0]]
+            quals |= self._extract_quals(typenode)
+            return tp, quals
+        #
+        if isinstance(typenode, pycparser.c_ast.ArrayDecl):
+            # array type
+            if typenode.dim is None:
+                length = None
+            else:
+                length = self._parse_constant(
+                    typenode.dim, partial_length_ok=partial_length_ok)
+            tp, quals = self._get_type_and_quals(typenode.type,
+                                partial_length_ok=partial_length_ok)
+            return model.ArrayType(tp, length), quals
+        #
+        if isinstance(typenode, pycparser.c_ast.PtrDecl):
+            # pointer type
+            itemtype, itemquals = self._get_type_and_quals(typenode.type)
+            tp = self._get_type_pointer(itemtype, itemquals, declname=name)
+            quals = self._extract_quals(typenode)
+            return tp, quals
+        #
+        if isinstance(typenode, pycparser.c_ast.TypeDecl):
+            quals = self._extract_quals(typenode)
+            type = typenode.type
+            if isinstance(type, pycparser.c_ast.IdentifierType):
+                # assume a primitive type.  get it from .names, but reduce
+                # synonyms to a single chosen combination
+                names = list(type.names)
+                if names != ['signed', 'char']:    # keep this unmodified
+                    prefixes = {}
+                    while names:
+                        name = names[0]
+                        if name in ('short', 'long', 'signed', 'unsigned'):
+                            prefixes[name] = prefixes.get(name, 0) + 1
+                            del names[0]
+                        else:
+                            break
+                    # ignore the 'signed' prefix below, and reorder the others
+                    newnames = []
+                    for prefix in ('unsigned', 'short', 'long'):
+                        for i in range(prefixes.get(prefix, 0)):
+                            newnames.append(prefix)
+                    if not names:
+                        names = ['int']    # implicitly
+                    if names == ['int']:   # but kill it if 'short' or 'long'
+                        if 'short' in prefixes or 'long' in prefixes:
+                            names = []
+                    names = newnames + names
+                ident = ' '.join(names)
+                if ident == 'void':
+                    return model.void_type, quals
+                tp0, quals0 = resolve_common_type(self, ident)
+                return tp0, (quals | quals0)
+            #
+            if isinstance(type, pycparser.c_ast.Struct):
+                # 'struct foobar'
+                tp = self._get_struct_union_enum_type('struct', type, name)
+                return tp, quals
+            #
+            if isinstance(type, pycparser.c_ast.Union):
+                # 'union foobar'
+                tp = self._get_struct_union_enum_type('union', type, name)
+                return tp, quals
+            #
+            if isinstance(type, pycparser.c_ast.Enum):
+                # 'enum foobar'
+                tp = self._get_struct_union_enum_type('enum', type, name)
+                return tp, quals
+        #
+        if isinstance(typenode, pycparser.c_ast.FuncDecl):
+            # a function type
+            return self._parse_function_type(typenode, name), 0
+        #
+        # nested anonymous structs or unions end up here
+        if isinstance(typenode, pycparser.c_ast.Struct):
+            return self._get_struct_union_enum_type('struct', typenode, name,
+                                                    nested=True), 0
+        if isinstance(typenode, pycparser.c_ast.Union):
+            return self._get_struct_union_enum_type('union', typenode, name,
+                                                    nested=True), 0
+        #
+        raise api.FFIError(":%d: bad or unsupported type declaration" %
+                typenode.coord.line)
+
+    def _parse_function_type(self, typenode, funcname=None):
+        params = list(getattr(typenode.args, 'params', []))
+        for i, arg in enumerate(params):
+            if not hasattr(arg, 'type'):
+                raise api.CDefError("%s arg %d: unknown type '%s'"
+                    " (if you meant to use the old C syntax of giving"
+                    " untyped arguments, it is not supported)"
+                    % (funcname or 'in expression', i + 1,
+                       getattr(arg, 'name', '?')))
+        ellipsis = (
+            len(params) > 0 and
+            isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and
+            isinstance(params[-1].type.type,
+                       pycparser.c_ast.IdentifierType) and
+            params[-1].type.type.names == ['__dotdotdot__'])
+        if ellipsis:
+            params.pop()
+            if not params:
+                raise api.CDefError(
+                    "%s: a function with only '(...)' as argument"
+                    " is not correct C" % (funcname or 'in expression'))
+        args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type))
+                for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
+        result, quals = self._get_type_and_quals(typenode.type)
+        # the 'quals' on the result type are ignored.  HACK: we absure them
+        # to detect __stdcall functions: we textually replace "__stdcall"
+        # with "volatile volatile const" above.
+        abi = None
+        if hasattr(typenode.type, 'quals'): # else, probable syntax error anyway
+            if typenode.type.quals[-3:] == ['volatile', 'volatile', 'const']:
+                abi = '__stdcall'
+        return model.RawFunctionType(tuple(args), result, ellipsis, abi)
+
+    def _as_func_arg(self, type, quals):
+        if isinstance(type, model.ArrayType):
+            return model.PointerType(type.item, quals)
+        elif isinstance(type, model.RawFunctionType):
+            return type.as_function_pointer()
+        else:
+            return type
+
+    def _get_struct_union_enum_type(self, kind, type, name=None, nested=False):
+        # First, a level of caching on the exact 'type' node of the AST.
+        # This is obscure, but needed because pycparser "unrolls" declarations
+        # such as "typedef struct { } foo_t, *foo_p" and we end up with
+        # an AST that is not a tree, but a DAG, with the "type" node of the
+        # two branches foo_t and foo_p of the trees being the same node.
+        # It's a bit silly but detecting "DAG-ness" in the AST tree seems
+        # to be the only way to distinguish this case from two independent
+        # structs.  See test_struct_with_two_usages.
+        try:
+            return self._structnode2type[type]
+        except KeyError:
+            pass
+        #
+        # Note that this must handle parsing "struct foo" any number of
+        # times and always return the same StructType object.  Additionally,
+        # one of these times (not necessarily the first), the fields of
+        # the struct can be specified with "struct foo { ...fields... }".
+        # If no name is given, then we have to create a new anonymous struct
+        # with no caching; in this case, the fields are either specified
+        # right now or never.
+        #
+        force_name = name
+        name = type.name
+        #
+        # get the type or create it if needed
+        if name is None:
+            # 'force_name' is used to guess a more readable name for
+            # anonymous structs, for the common case "typedef struct { } foo".
+            if force_name is not None:
+                explicit_name = '$%s' % force_name
+            else:
+                self._anonymous_counter += 1
+                explicit_name = '$%d' % self._anonymous_counter
+            tp = None
+        else:
+            explicit_name = name
+            key = '%s %s' % (kind, name)
+            tp, _ = self._declarations.get(key, (None, None))
+        #
+        if tp is None:
+            if kind == 'struct':
+                tp = model.StructType(explicit_name, None, None, None)
+            elif kind == 'union':
+                tp = model.UnionType(explicit_name, None, None, None)
+            elif kind == 'enum':
+                tp = self._build_enum_type(explicit_name, type.values)
+            else:
+                raise AssertionError("kind = %r" % (kind,))
+            if name is not None:
+                self._declare(key, tp)
+        else:
+            if kind == 'enum' and type.values is not None:
+                raise NotImplementedError(
+                    "enum %s: the '{}' declaration should appear on the first "
+                    "time the enum is mentioned, not later" % explicit_name)
+        if not tp.forcename:
+            tp.force_the_name(force_name)
+        if tp.forcename and '$' in tp.name:
+            self._declare('anonymous %s' % tp.forcename, tp)
+        #
+        self._structnode2type[type] = tp
+        #
+        # enums: done here
+        if kind == 'enum':
+            return tp
+        #
+        # is there a 'type.decls'?  If yes, then this is the place in the
+        # C sources that declare the fields.  If no, then just return the
+        # existing type, possibly still incomplete.
+        if type.decls is None:
+            return tp
+        #
+        if tp.fldnames is not None:
+            raise api.CDefError("duplicate declaration of struct %s" % name)
+        fldnames = []
+        fldtypes = []
+        fldbitsize = []
+        fldquals = []
+        for decl in type.decls:
+            if decl.bitsize is None:
+                bitsize = -1
+            else:
+                bitsize = self._parse_constant(decl.bitsize)
+            self._partial_length = False
+            type, fqual = self._get_type_and_quals(decl.type,
+                                                   partial_length_ok=True)
+            if self._partial_length:
+                self._make_partial(tp, nested)
+            if isinstance(type, model.StructType) and type.partial:
+                self._make_partial(tp, nested)
+            fldnames.append(decl.name or '')
+            fldtypes.append(type)
+            fldbitsize.append(bitsize)
+            fldquals.append(fqual)
+        tp.fldnames = tuple(fldnames)
+        tp.fldtypes = tuple(fldtypes)
+        tp.fldbitsize = tuple(fldbitsize)
+        tp.fldquals = tuple(fldquals)
+        if fldbitsize != [-1] * len(fldbitsize):
+            if isinstance(tp, model.StructType) and tp.partial:
+                raise NotImplementedError("%s: using both bitfields and '...;'"
+                                          % (tp,))
+        tp.packed = self._options.get('packed')
+        if tp.completed:    # must be re-completed: it is not opaque any more
+            tp.completed = 0
+            self._recomplete.append(tp)
+        return tp
+
+    def _make_partial(self, tp, nested):
+        if not isinstance(tp, model.StructOrUnion):
+            raise api.CDefError("%s cannot be partial" % (tp,))
+        if not tp.has_c_name() and not nested:
+            raise NotImplementedError("%s is partial but has no C name" %(tp,))
+        tp.partial = True
+
+    def _parse_constant(self, exprnode, partial_length_ok=False):
+        # for now, limited to expressions that are an immediate number
+        # or positive/negative number
+        if isinstance(exprnode, pycparser.c_ast.Constant):
+            s = exprnode.value
+            if s.startswith('0'):
+                if s.startswith('0x') or s.startswith('0X'):
+                    return int(s, 16)
+                return int(s, 8)
+            elif '1' <= s[0] <= '9':
+                return int(s, 10)
+            elif s[0] == "'" and s[-1] == "'" and (
+                    len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
+                return ord(s[-2])
+            else:
+                raise api.CDefError("invalid constant %r" % (s,))
+        #
+        if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
+                exprnode.op == '+'):
+            return self._parse_constant(exprnode.expr)
+        #
+        if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
+                exprnode.op == '-'):
+            return -self._parse_constant(exprnode.expr)
+        # load previously defined int constant
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
+                exprnode.name in self._int_constants):
+            return self._int_constants[exprnode.name]
+        #
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
+                    exprnode.name == '__dotdotdotarray__'):
+            if partial_length_ok:
+                self._partial_length = True
+                return '...'
+            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
+                               "the actual array length in this context"
+                               % exprnode.coord.line)
+        #
+        raise api.FFIError(":%d: unsupported expression: expected a "
+                           "simple numeric constant" % exprnode.coord.line)
+
+    def _build_enum_type(self, explicit_name, decls):
+        if decls is not None:
+            partial = False
+            enumerators = []
+            enumvalues = []
+            nextenumvalue = 0
+            for enum in decls.enumerators:
+                if enum.value is not None:
+                    nextenumvalue = self._parse_constant(enum.value)
+                enumerators.append(enum.name)
+                enumvalues.append(nextenumvalue)
+                self._add_constants(enum.name, nextenumvalue)
+                nextenumvalue += 1
+            enumerators = tuple(enumerators)
+            enumvalues = tuple(enumvalues)
+            tp = model.EnumType(explicit_name, enumerators, enumvalues)
+            tp.partial = partial
+        else:   # opaque enum
+            tp = model.EnumType(explicit_name, (), ())
+        return tp
+
+    def include(self, other):
+        for name, (tp, quals) in other._declarations.items():
+            if name.startswith('anonymous $enum_$'):
+                continue   # fix for test_anonymous_enum_include
+            kind = name.split(' ', 1)[0]
+            if kind in ('struct', 'union', 'enum', 'anonymous', 'typedef', 'macro'):
+                self._declare(name, tp, included=True, quals=quals)
+        for k, v in other._int_constants.items():
+            self._add_constants(k, v)
+        for k, v in other._macros.items():
+            self._macros[k] = v
+
+CNAME_TO_LLTYPE = {
+    'char': rffi.CHAR,
+    'double': rffi.DOUBLE, 'long double': rffi.LONGDOUBLE,
+    'float': rffi.FLOAT, 'FILE': FILEP.TO}
+
+def add_inttypes():
+    for name in rffi.TYPES:
+        if name.startswith('unsigned'):
+            rname = 'u' + name[9:]
+        else:
+            rname = name
+        rname = rname.replace(' ', '').upper()
+        CNAME_TO_LLTYPE[name] = rfficache.platform.types[rname]
+
+add_inttypes()
+CNAME_TO_LLTYPE['int'] = rffi.INT_real
+
+def cname_to_lltype(name):
+    return CNAME_TO_LLTYPE[name]
+
+class DelayedStruct(object):
+    def __init__(self, name, fields, TYPE):
+        self.struct_name = name
+        self.type_name = None
+        self.fields = fields
+        self.TYPE = TYPE
+
+    def get_type_name(self):
+        if self.type_name is not None:
+            return self.type_name
+        elif not self.struct_name.startswith('$'):
+            return 'struct %s' % self.struct_name
+        else:
+            raise ValueError('Anonymous struct')
+
+    def __repr__(self):
+        return "<struct {struct_name}>".format(**vars(self))
+
+
+class CTypeSpace(object):
+    def __init__(self, parser=None, definitions=None, macros=None,
+                 headers=None, includes=None):
+        self.definitions = definitions if definitions is not None else {}
+        self.macros = macros if macros is not None else {}
+        self.structs = {}
+        self.ctx = parser if parser else Parser()
+        self.headers = headers if headers is not None else ['sys/types.h']
+        self.parsed_headers = []
+        self.sources = []
+        self._Config = type('Config', (object,), {})
+        self._TYPES = {}
+        self.includes = []
+        self.struct_typedefs = {}
+        self._handled = set()
+        self._frozen = False
+        if includes is not None:
+            for header in includes:
+                self.include(header)
+
+    def include(self, other):
+        self.ctx.include(other.ctx)
+        self.structs.update(other.structs)
+        self.includes.append(other)
+
+    def parse_source(self, source):
+        self.sources.append(source)
+        self.ctx.parse(source)
+        self.configure_types()
+
+    def parse_header(self, header_path):
+        self.headers.append(str(header_path))
+        self.parsed_headers.append(header_path)
+        self.ctx.parse(header_path.read())
+        self.configure_types()
+
+    def add_typedef(self, name, obj, quals):
+        assert name not in self.definitions
+        tp = self.convert_type(obj, quals)
+        if isinstance(tp, DelayedStruct):
+            if tp.type_name is None:
+                tp.type_name = name
+            tp = self.realize_struct(tp)
+        self.definitions[name] = tp
+
+    def add_macro(self, name, value):
+        assert name not in self.macros
+        self.macros[name] = value
+
+    def new_struct(self, obj):
+        if obj.name == '_IO_FILE':  # cffi weirdness
+            return cname_to_lltype('FILE')
+        struct = DelayedStruct(obj.name, None, lltype.ForwardReference())
+        # Cache it early, to avoid infinite recursion
+        self.structs[obj] = struct
+        if obj.fldtypes is not None:
+            struct.fields = zip(
+                obj.fldnames,
+                [self.convert_field(field) for field in obj.fldtypes])
+        return struct
+
+    def convert_field(self, obj):
+        tp = self.convert_type(obj)
+        if isinstance(tp, DelayedStruct):
+            tp = tp.TYPE
+        return tp
+
+    def realize_struct(self, struct):
+        type_name = struct.get_type_name()
+        configname = type_name.replace(' ', '__')
+        setattr(self._Config, configname,
+            rffi_platform.Struct(type_name, struct.fields))
+        self._TYPES[configname] = struct.TYPE
+        return struct.TYPE
+
+    def build_eci(self):
+        all_sources = []
+        for cts in self.includes:
+            all_sources.extend(cts.sources)
+        all_sources.extend(self.sources)
+        all_headers = self.headers
+        for x in self.includes:
+            for hdr in x.headers:
+                if hdr not in all_headers:
+                    all_headers.append(hdr)
+        return ExternalCompilationInfo(
+            post_include_bits=all_sources, includes=all_headers)
+
+    def configure_types(self):
+        for name, (obj, quals) in self.ctx._declarations.iteritems():
+            if obj in self.ctx._included_declarations:
+                continue
+            if name in self._handled:
+                continue
+            self._handled.add(name)
+            if name.startswith('typedef '):
+                name = name[8:]
+                self.add_typedef(name, obj, quals)
+            elif name.startswith('macro '):
+                name = name[6:]
+                self.add_macro(name, obj)
+        self._Config._compilation_info_ = self.build_eci()
+        for name, TYPE in rffi_platform.configure(self._Config).iteritems():
+            # hack: prevent the source from being pasted into common_header.h
+            del TYPE._hints['eci']
+            if name in self._TYPES:
+                self._TYPES[name].become(TYPE)
+                del self._TYPES[name]
+
+    def convert_type(self, obj, quals=0):
+        if isinstance(obj, model.PrimitiveType):
+            return cname_to_lltype(obj.name)
+        elif isinstance(obj, model.StructType):
+            if obj in self.structs:
+                return self.structs[obj]
+            return self.new_struct(obj)
+        elif isinstance(obj, model.PointerType):
+            TO = self.convert_type(obj.totype)
+            if TO is lltype.Void:
+                return rffi.VOIDP
+            elif isinstance(TO, DelayedStruct):
+                TO = TO.TYPE
+            if isinstance(TO, lltype.ContainerType):
+                return lltype.Ptr(TO)
+            else:
+                if obj.quals & model.Q_CONST:
+                    return lltype.Ptr(lltype.Array(
+                        TO, hints={'nolength': True, 'render_as_const': True}))
+                else:
+                    return rffi.CArrayPtr(TO)
+        elif isinstance(obj, model.FunctionPtrType):
+            if obj.ellipsis:
+                raise NotImplementedError
+            args = [self.convert_type(arg) for arg in obj.args]
+            res = self.convert_type(obj.result)
+            return lltype.Ptr(lltype.FuncType(args, res))
+        elif isinstance(obj, model.VoidType):
+            return lltype.Void
+        elif isinstance(obj, model.ArrayType):
+            return rffi.CFixedArray(self.convert_type(obj.item), obj.length)
+        else:
+            raise NotImplementedError
+
+    def gettype(self, cdecl):
+        obj = self.ctx.parse_type(cdecl)
+        result = self.convert_type(obj)
+        if isinstance(result, DelayedStruct):
+            result = result.TYPE
+        return result
+
+    def cast(self, cdecl, value):
+        return rffi.cast(self.gettype(cdecl), value)
+
+    def parse_func(self, cdecl):
+        cdecl = cdecl.strip()
+        if cdecl[-1] != ';':
+            cdecl += ';'
+        ast, _, _ = self.ctx._parse(cdecl)
+        decl = ast.ext[-1]
+        tp, quals = self.ctx._get_type_and_quals(decl.type, name=decl.name)
+        FUNCP = self.convert_type(tp.as_function_pointer())
+        return decl.name, FUNCP.TO
+
+    def _freeze_(self):
+        if self._frozen:
+            return True
+
+        @register_flow_sc(self.cast)
+        def sc_cast(ctx, v_decl, v_arg):
+            if not isinstance(v_decl, Constant):
+                raise FlowingError(
+                    "The first argument of cts.cast() must be a constant.")
+            TP = self.gettype(v_decl.value)
+            return ctx.appcall(rffi.cast, const(TP), v_arg)
+
+        @register_flow_sc(self.gettype)
+        def sc_gettype(ctx, v_decl):
+            if not isinstance(v_decl, Constant):
+                raise FlowingError(
+                    "The argument of cts.gettype() must be a constant.")
+            return const(self.gettype(v_decl.value))
+
+        self._frozen = True
+        return True
+
+
+def parse_source(source, includes=None, headers=None, configure_now=True):
+    cts = CTypeSpace(headers=headers, includes=includes)
+    cts.parse_source(source)
+    return cts
diff --git a/pypy/module/cpyext/include/descrobject.h b/pypy/module/cpyext/include/descrobject.h
--- a/pypy/module/cpyext/include/descrobject.h
+++ b/pypy/module/cpyext/include/descrobject.h
@@ -1,16 +1,5 @@
 #ifndef Py_DESCROBJECT_H
 #define Py_DESCROBJECT_H
-typedef PyObject *(*getter)(PyObject *, void *);
-typedef int (*setter)(PyObject *, PyObject *, void *);
-
-typedef struct PyGetSetDef {
-	char *name;
-	getter get;
-	setter set;
-	char *doc;
-	void *closure;
-} PyGetSetDef;
-
 
 #define PyDescr_COMMON \
     PyObject_HEAD \
diff --git a/pypy/module/cpyext/include/methodobject.h b/pypy/module/cpyext/include/methodobject.h
--- a/pypy/module/cpyext/include/methodobject.h
+++ b/pypy/module/cpyext/include/methodobject.h
@@ -7,20 +7,6 @@
 extern "C" {
 #endif
 
-typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
-typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
-                                             PyObject *);
-typedef PyObject *(*PyNoArgsFunction)(PyObject *);
-
-struct PyMethodDef {
-    const char  *ml_name;   /* The name of the built-in function/method */
-    PyCFunction  ml_meth;   /* The C function that implements it */
-    int          ml_flags;  /* Combination of METH_xxx flags, which mostly
-                               describe the args expected by the C func */
-    const char  *ml_doc;    /* The __doc__ attribute, or NULL */
-};
-typedef struct PyMethodDef PyMethodDef;
-
 typedef struct
 {
     PyObject_HEAD
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -9,6 +9,7 @@
 
 #define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
 
+#include <cpyext_object.h>
 
 /*
 CPython has this for backwards compatibility with really old extensions, and now
@@ -16,29 +17,12 @@
 */
 #define staticforward static
 
-#define PyObject_HEAD  \
-    Py_ssize_t ob_refcnt;        \
-    Py_ssize_t ob_pypy_link;     \
-    struct _typeobject *ob_type;
-
-#define PyObject_VAR_HEAD		\
-	PyObject_HEAD			\
-	Py_ssize_t ob_size; /* Number of items in variable part */
-
 #define PyObject_HEAD_INIT(type)	\
 	1, 0, type,
 
 #define PyVarObject_HEAD_INIT(type, size)	\
 	PyObject_HEAD_INIT(type) size,
 
-typedef struct _object {
-    PyObject_HEAD
-} PyObject;
-
-typedef struct {
-	PyObject_VAR_HEAD
-} PyVarObject;
-
 #ifdef PYPY_DEBUG_REFCOUNT
 /* Slow version, but useful for debugging */
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)(ob)))
@@ -94,65 +78,8 @@
 #define Py_GT 4
 #define Py_GE 5
 
-struct _typeobject;
-typedef void (*freefunc)(void *);
-typedef void (*destructor)(PyObject *);
-typedef int (*printfunc)(PyObject *, FILE *, int);
-typedef PyObject *(*getattrfunc)(PyObject *, char *);
-typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
-typedef int (*setattrfunc)(PyObject *, char *, PyObject *);
-typedef int (*setattrofunc)(PyObject *, PyObject *, PyObject *);
-typedef int (*cmpfunc)(PyObject *, PyObject *);
-typedef PyObject *(*reprfunc)(PyObject *);
-typedef long (*hashfunc)(PyObject *);
-typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
-typedef PyObject *(*getiterfunc) (PyObject *);
-typedef PyObject *(*iternextfunc) (PyObject *);
-typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);
-typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *);
-typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
-typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
-typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t);
-
-typedef PyObject * (*unaryfunc)(PyObject *);
-typedef PyObject * (*binaryfunc)(PyObject *, PyObject *);
-typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
-typedef int (*inquiry)(PyObject *);
-typedef Py_ssize_t (*lenfunc)(PyObject *);
-typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
-typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
-typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
-typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
-typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
-
 
 /* Py3k buffer interface, adapted for PyPy */
-#define Py_MAX_NDIMS 32
-#define Py_MAX_FMT 128
-typedef struct bufferinfo {
-    void *buf;
-    PyObject *obj;        /* owned reference */
-    Py_ssize_t len;
-    Py_ssize_t itemsize;  /* This is Py_ssize_t so it can be
-                             pointed to by strides in simple case.*/
-    int readonly;
-    int ndim;
-    char *format;
-    Py_ssize_t *shape;
-    Py_ssize_t *strides;
-    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
-    unsigned char _format[Py_MAX_FMT];
-    Py_ssize_t _strides[Py_MAX_NDIMS];
-    Py_ssize_t _shape[Py_MAX_NDIMS];
-    /* static store for shape and strides of
-       mono-dimensional buffers. */
-    /* Py_ssize_t smalltable[2]; */
-    void *internal; /* always NULL for app-level objects */
-} Py_buffer;
-
-typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
-typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
-
     /* Flags for getting buffers */
 #define PyBUF_SIMPLE 0
 #define PyBUF_WRITABLE 0x0001
@@ -184,180 +111,7 @@
 #define PyBUF_SHADOW 0x400
 /* end Py3k buffer interface */
 
-typedef int (*objobjproc)(PyObject *, PyObject *);
-typedef int (*visitproc)(PyObject *, void *);
-typedef int (*traverseproc)(PyObject *, visitproc, void *);
-
-
-typedef struct {
-    /* Number implementations must check *both*
-       arguments for proper type and implement the necessary conversions
-       in the slot functions themselves. */
-
-    binaryfunc nb_add;
-    binaryfunc nb_subtract;
-    binaryfunc nb_multiply;
-    binaryfunc nb_remainder;
-    binaryfunc nb_divmod;
-    ternaryfunc nb_power;
-    unaryfunc nb_negative;
-    unaryfunc nb_positive;
-    unaryfunc nb_absolute;
-    inquiry nb_bool;
-    unaryfunc nb_invert;
-    binaryfunc nb_lshift;
-    binaryfunc nb_rshift;
-    binaryfunc nb_and;
-    binaryfunc nb_xor;
-    binaryfunc nb_or;
-    unaryfunc nb_int;
-    void *nb_reserved;  /* the slot formerly known as nb_long */
-    unaryfunc nb_float;
-
-    binaryfunc nb_inplace_add;
-    binaryfunc nb_inplace_subtract;
-    binaryfunc nb_inplace_multiply;
-    binaryfunc nb_inplace_remainder;
-    ternaryfunc nb_inplace_power;
-    binaryfunc nb_inplace_lshift;
-    binaryfunc nb_inplace_rshift;
-    binaryfunc nb_inplace_and;
-    binaryfunc nb_inplace_xor;
-    binaryfunc nb_inplace_or;
-
-    binaryfunc nb_floor_divide;
-    binaryfunc nb_true_divide;
-    binaryfunc nb_inplace_floor_divide;
-    binaryfunc nb_inplace_true_divide;
-
-    unaryfunc nb_index;
-
-    binaryfunc nb_matrix_multiply;
-    binaryfunc nb_inplace_matrix_multiply;
-} PyNumberMethods;
-
-typedef struct {
-    lenfunc sq_length;
-    binaryfunc sq_concat;
-    ssizeargfunc sq_repeat;
-    ssizeargfunc sq_item;
-    void *was_sq_slice;
-    ssizeobjargproc sq_ass_item;
-    void *was_sq_ass_slice;
-    objobjproc sq_contains;
-
-    binaryfunc sq_inplace_concat;
-    ssizeargfunc sq_inplace_repeat;
-} PySequenceMethods;
-
-typedef struct {
-    lenfunc mp_length;
-    binaryfunc mp_subscript;
-    objobjargproc mp_ass_subscript;
-} PyMappingMethods;
-
-typedef struct {
-    unaryfunc am_await;
-    unaryfunc am_aiter;
-    unaryfunc am_anext;
-} PyAsyncMethods;
-
-typedef struct {
-     getbufferproc bf_getbuffer;
-     releasebufferproc bf_releasebuffer;
-} PyBufferProcs;
-
-
-typedef struct _typeobject {
-    PyObject_VAR_HEAD
-    const char *tp_name; /* For printing, in format "<module>.<name>" */
-    Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
-
-    /* Methods to implement standard operations */
-
-    destructor tp_dealloc;
-    printfunc tp_print;
-    getattrfunc tp_getattr;
-    setattrfunc tp_setattr;
-    PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2)
-                                    or tp_reserved (Python 3) */
-    reprfunc tp_repr;
-
-    /* Method suites for standard classes */
-
-    PyNumberMethods *tp_as_number;
-    PySequenceMethods *tp_as_sequence;
-    PyMappingMethods *tp_as_mapping;
-
-    /* More standard operations (here for binary compatibility) */
-
-    hashfunc tp_hash;
-    ternaryfunc tp_call;
-    reprfunc tp_str;
-    getattrofunc tp_getattro;
-    setattrofunc tp_setattro;
-
-    /* Functions to access object as input/output buffer */
-    PyBufferProcs *tp_as_buffer;
-
-    /* Flags to define presence of optional/expanded features */
-    long tp_flags;
-
-    const char *tp_doc; /* Documentation string */
-
-    /* Assigned meaning in release 2.0 */
-    /* call function for all accessible objects */
-    traverseproc tp_traverse;
-
-    /* delete references to contained objects */
-    inquiry tp_clear;
-
-    /* Assigned meaning in release 2.1 */
-    /* rich comparisons */
-    richcmpfunc tp_richcompare;
-
-    /* weak reference enabler */
-    Py_ssize_t tp_weaklistoffset;
-
-    /* Iterators */
-    getiterfunc tp_iter;
-    iternextfunc tp_iternext;
-
-    /* Attribute descriptor and subclassing stuff */
-    struct PyMethodDef *tp_methods;
-    struct PyMemberDef *tp_members;
-    struct PyGetSetDef *tp_getset;
-    struct _typeobject *tp_base;
-    PyObject *tp_dict;
-    descrgetfunc tp_descr_get;
-    descrsetfunc tp_descr_set;
-    Py_ssize_t tp_dictoffset;
-    initproc tp_init;
-    allocfunc tp_alloc;
-    newfunc tp_new;
-    freefunc tp_free; /* Low-level free-memory routine */
-    inquiry tp_is_gc; /* For PyObject_IS_GC */
-    PyObject *tp_bases;
-    PyObject *tp_mro; /* method resolution order */
-    PyObject *tp_cache;
-    PyObject *tp_subclasses;
-    PyObject *tp_weaklist;
-    destructor tp_del;
-
-    /* Type attribute cache version tag. Added in version 2.6 */
-    unsigned int tp_version_tag;
-
-    destructor tp_finalize;
-} PyTypeObject;
-
-typedef struct {
-    PyTypeObject ht_type;
-    PyNumberMethods as_number;
-    PyMappingMethods as_mapping;
-    PySequenceMethods as_sequence;
-    PyBufferProcs as_buffer;
-    PyObject *ht_name, *ht_slots;
-} PyHeapTypeObject;
+#include <cpyext_typeobject.h>
 
 #define PyObject_Bytes PyObject_Str
 
diff --git a/pypy/module/cpyext/include/structmember.h b/pypy/module/cpyext/include/structmember.h
--- a/pypy/module/cpyext/include/structmember.h
+++ b/pypy/module/cpyext/include/structmember.h
@@ -19,22 +19,6 @@
 #define offsetof(type, member) ( (int) & ((type*)0) -> member )
 #endif
 
-/* An array of memberlist structures defines the name, type and offset
-   of selected members of a C structure.  These can be read by
-   PyMember_Get() and set by PyMember_Set() (except if their READONLY flag
-   is set).  The array must be terminated with an entry whose name
-   pointer is NULL. */
-
-
-
-typedef struct PyMemberDef {
-    /* Current version, use this */
-    char *name;
-    int type;
-    Py_ssize_t offset;
-    int flags;
-    char *doc;
-} PyMemberDef;
 
 /* Types */
 #define T_SHORT         0
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,23 +11,13 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr, slot_function)
+    PyTypeObjectPtr, slot_function, cts, api_decl)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
-PyCFunction_typedef = rffi.COpaquePtr(typedef='PyCFunction')
-PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject))
-PyCFunctionKwArgs = lltype.Ptr(lltype.FuncType([PyObject, PyObject, PyObject],
-                                               PyObject))
-
-PyMethodDef = cpython_struct(
-    'PyMethodDef',
-    [('ml_name', rffi.CONST_CCHARP),
-     ('ml_meth', PyCFunction_typedef),
-     ('ml_flags', rffi.INT_real),
-     ('ml_doc', rffi.CONST_CCHARP),
-     ])
-
+PyMethodDef = cts.gettype('PyMethodDef')
+PyCFunction = cts.gettype('PyCFunction')
+PyCFunctionKwArgs = cts.gettype('PyCFunctionWithKeywords')
 PyCFunctionObjectStruct = cpython_struct(
     'PyCFunctionObject',
     PyObjectFields + (
@@ -77,7 +67,7 @@
             raise oefmt(space.w_TypeError,
                         "%s() takes no keyword arguments", self.name)
 
-        func = rffi.cast(PyCFunction, self.ml.c_ml_meth)
+        func = self.ml.c_ml_meth
         length = space.int_w(space.len(w_args))
         if flags & METH_KEYWORDS:
             func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth)
@@ -282,7 +272,7 @@
 def PyCFunction_NewEx(space, ml, w_self, w_name):
     return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name))
 
- at cpython_api([PyObject], PyCFunction_typedef)
+ at api_decl("PyCFunction PyCFunction_GetFunction(PyObject *)", cts)
 def PyCFunction_GetFunction(space, w_obj):
     try:
         cfunction = space.interp_w(W_PyCFunctionObject, w_obj)
@@ -337,4 +327,3 @@
     if name == "__methods__":
         return space.newlist(method_list_w)
     raise OperationError(space.w_AttributeError, space.wrap(name))
-
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -0,0 +1,282 @@
+
+typedef ssize_t Py_ssize_t;
+
+#define PyObject_HEAD  \
+    Py_ssize_t ob_refcnt;        \
+    Py_ssize_t ob_pypy_link;     \
+    struct _typeobject *ob_type;
+
+#define PyObject_VAR_HEAD		\
+	PyObject_HEAD			\
+	Py_ssize_t ob_size; /* Number of items in variable part */
+
+typedef struct _object {
+    PyObject_HEAD
+} PyObject;
+
+typedef struct {
+	PyObject_VAR_HEAD
+} PyVarObject;
+
+struct _typeobject;
+typedef void (*freefunc)(void *);
+typedef void (*destructor)(PyObject *);
+typedef int (*printfunc)(PyObject *, FILE *, int);
+typedef PyObject *(*getattrfunc)(PyObject *, char *);
+typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
+typedef int (*setattrfunc)(PyObject *, char *, PyObject *);
+typedef int (*setattrofunc)(PyObject *, PyObject *, PyObject *);
+typedef int (*cmpfunc)(PyObject *, PyObject *);
+typedef PyObject *(*reprfunc)(PyObject *);
+typedef long (*hashfunc)(PyObject *);
+typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
+typedef PyObject *(*getiterfunc) (PyObject *);
+typedef PyObject *(*iternextfunc) (PyObject *);
+typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);
+typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *);
+typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
+typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
+typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t);
+
+typedef PyObject * (*unaryfunc)(PyObject *);
+typedef PyObject * (*binaryfunc)(PyObject *, PyObject *);
+typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
+typedef int (*inquiry)(PyObject *);
+typedef Py_ssize_t (*lenfunc)(PyObject *);
+typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
+typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
+typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
+typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
+typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
+
+
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 128
+typedef struct bufferinfo {
+    void *buf;
+    PyObject *obj;        /* owned reference */
+    Py_ssize_t len;
+    Py_ssize_t itemsize;  /* This is Py_ssize_t so it can be
+                             pointed to by strides in simple case.*/
+    int readonly;
+    int ndim;
+    char *format;
+    Py_ssize_t *shape;
+    Py_ssize_t *strides;
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format[Py_MAX_FMT];
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
+    /* static store for shape and strides of
+       mono-dimensional buffers. */
+    /* Py_ssize_t smalltable[2]; */
+    void *internal; /* always NULL for app-level objects */
+} Py_buffer;
+
+typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
+typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
+/* end Py3k buffer interface */
+
+typedef int (*objobjproc)(PyObject *, PyObject *);
+typedef int (*visitproc)(PyObject *, void *);
+typedef int (*traverseproc)(PyObject *, visitproc, void *);
+
+
+typedef struct {
+    /* Number implementations must check *both*
+       arguments for proper type and implement the necessary conversions
+       in the slot functions themselves. */
+
+    binaryfunc nb_add;
+    binaryfunc nb_subtract;
+    binaryfunc nb_multiply;
+    binaryfunc nb_remainder;
+    binaryfunc nb_divmod;
+    ternaryfunc nb_power;
+    unaryfunc nb_negative;
+    unaryfunc nb_positive;
+    unaryfunc nb_absolute;
+    inquiry nb_bool;
+    unaryfunc nb_invert;
+    binaryfunc nb_lshift;
+    binaryfunc nb_rshift;
+    binaryfunc nb_and;
+    binaryfunc nb_xor;
+    binaryfunc nb_or;
+    unaryfunc nb_int;
+    void *nb_reserved;  /* the slot formerly known as nb_long */
+    unaryfunc nb_float;
+
+    binaryfunc nb_inplace_add;
+    binaryfunc nb_inplace_subtract;
+    binaryfunc nb_inplace_multiply;
+    binaryfunc nb_inplace_remainder;
+    ternaryfunc nb_inplace_power;
+    binaryfunc nb_inplace_lshift;
+    binaryfunc nb_inplace_rshift;
+    binaryfunc nb_inplace_and;
+    binaryfunc nb_inplace_xor;
+    binaryfunc nb_inplace_or;
+
+    binaryfunc nb_floor_divide;
+    binaryfunc nb_true_divide;
+    binaryfunc nb_inplace_floor_divide;
+    binaryfunc nb_inplace_true_divide;
+
+    unaryfunc nb_index;
+
+    binaryfunc nb_matrix_multiply;
+    binaryfunc nb_inplace_matrix_multiply;
+} PyNumberMethods;
+
+typedef struct {
+    lenfunc sq_length;
+    binaryfunc sq_concat;
+    ssizeargfunc sq_repeat;
+    ssizeargfunc sq_item;
+    void *was_sq_slice;
+    ssizeobjargproc sq_ass_item;
+    void *was_sq_ass_slice;
+    objobjproc sq_contains;
+
+    binaryfunc sq_inplace_concat;
+    ssizeargfunc sq_inplace_repeat;
+} PySequenceMethods;
+
+typedef struct {
+    lenfunc mp_length;
+    binaryfunc mp_subscript;
+    objobjargproc mp_ass_subscript;
+} PyMappingMethods;
+
+typedef struct {
+    unaryfunc am_await;
+    unaryfunc am_aiter;
+    unaryfunc am_anext;
+} PyAsyncMethods;
+
+typedef struct {
+     getbufferproc bf_getbuffer;
+     releasebufferproc bf_releasebuffer;
+} PyBufferProcs;
+
+/* from descrobject.h */
+typedef PyObject *(*getter)(PyObject *, void *);
+typedef int (*setter)(PyObject *, PyObject *, void *);
+
+typedef struct PyGetSetDef {
+	char *name;
+	getter get;
+	setter set;
+	char *doc;
+	void *closure;
+} PyGetSetDef;
+
+/* from methodobject.h */
+typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
+typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
+                                             PyObject *);
+typedef PyObject *(*PyNoArgsFunction)(PyObject *);
+
+struct PyMethodDef {
+    const char  *ml_name;   /* The name of the built-in function/method */
+    PyCFunction  ml_meth;   /* The C function that implements it */
+    int          ml_flags;  /* Combination of METH_xxx flags, which mostly
+                               describe the args expected by the C func */
+    const char  *ml_doc;    /* The __doc__ attribute, or NULL */
+};
+typedef struct PyMethodDef PyMethodDef;
+
+/* from structmember.h */
+typedef struct PyMemberDef {
+    /* Current version, use this */
+    char *name;
+    int type;
+    Py_ssize_t offset;
+    int flags;
+    char *doc;
+} PyMemberDef;
+
+
+typedef struct _typeobject {
+    PyObject_VAR_HEAD
+    const char *tp_name; /* For printing, in format "<module>.<name>" */
+    Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
+
+    /* Methods to implement standard operations */
+
+    destructor tp_dealloc;
+    printfunc tp_print;
+    getattrfunc tp_getattr;
+    setattrfunc tp_setattr;
+    PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2)
+                                    or tp_reserved (Python 3) */
+    reprfunc tp_repr;
+
+    /* Method suites for standard classes */
+
+    PyNumberMethods *tp_as_number;
+    PySequenceMethods *tp_as_sequence;
+    PyMappingMethods *tp_as_mapping;
+
+    /* More standard operations (here for binary compatibility) */
+
+    hashfunc tp_hash;
+    ternaryfunc tp_call;
+    reprfunc tp_str;
+    getattrofunc tp_getattro;
+    setattrofunc tp_setattro;
+
+    /* Functions to access object as input/output buffer */
+    PyBufferProcs *tp_as_buffer;
+
+    /* Flags to define presence of optional/expanded features */
+    long tp_flags;
+
+    const char *tp_doc; /* Documentation string */
+
+    /* Assigned meaning in release 2.0 */
+    /* call function for all accessible objects */
+    traverseproc tp_traverse;
+
+    /* delete references to contained objects */
+    inquiry tp_clear;
+
+    /* Assigned meaning in release 2.1 */
+    /* rich comparisons */
+    richcmpfunc tp_richcompare;
+
+    /* weak reference enabler */
+    Py_ssize_t tp_weaklistoffset;
+
+    /* Iterators */
+    getiterfunc tp_iter;
+    iternextfunc tp_iternext;
+
+    /* Attribute descriptor and subclassing stuff */
+    struct PyMethodDef *tp_methods;
+    struct PyMemberDef *tp_members;
+    struct PyGetSetDef *tp_getset;
+    struct _typeobject *tp_base;
+    PyObject *tp_dict;
+    descrgetfunc tp_descr_get;
+    descrsetfunc tp_descr_set;
+    Py_ssize_t tp_dictoffset;
+    initproc tp_init;
+    allocfunc tp_alloc;
+    newfunc tp_new;
+    freefunc tp_free; /* Low-level free-memory routine */
+    inquiry tp_is_gc; /* For PyObject_IS_GC */
+    PyObject *tp_bases;
+    PyObject *tp_mro; /* method resolution order */
+    PyObject *tp_cache;
+    PyObject *tp_subclasses;
+    PyObject *tp_weaklist;
+    destructor tp_del;
+
+    /* Type attribute cache version tag. Added in version 2.6 */
+    unsigned int tp_version_tag;
+
+    destructor tp_finalize;
+} PyTypeObject;
diff --git a/pypy/module/cpyext/parse/cpyext_typeobject.h b/pypy/module/cpyext/parse/cpyext_typeobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_typeobject.h
@@ -0,0 +1,9 @@
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
+
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,9 +7,9 @@
 from rpython.rlib import rgc # Force registration of gc.collect
 from pypy.module.cpyext.api import (
     slot_function, generic_cpy_call, PyObject, Py_ssize_t,
-    pypy_decl, Py_buffer, Py_bufferP)
+    pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr)
 from pypy.module.cpyext.typeobjectdefs import (
-    unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
+    unaryfunc, ternaryfunc, binaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -5,7 +5,7 @@
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext.api import (
     slot_function, cpython_api, copy_header_files, INTERPLEVEL_API,
-    Py_ssize_t, Py_ssize_tP, PyObject)
+    Py_ssize_t, Py_ssize_tP, PyObject, cts)
 from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest
 from pypy.interpreter.error import OperationError
 from rpython.rlib import rawrefcount
@@ -99,10 +99,10 @@
     def test_typedef(self, space):
         from rpython.translator.c.database import LowLevelDatabase
         db = LowLevelDatabase()
-        assert PyPy_TypedefTest1.api_func.get_c_restype(db) == 'Py_ssize_t'
-        assert PyPy_TypedefTest1.api_func.get_c_args(db) == 'Py_ssize_t arg0'
-        assert PyPy_TypedefTest2.api_func.get_c_restype(db) == 'Py_ssize_t *'
-        assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Py_ssize_t *arg0'
+        assert PyPy_TypedefTest1.api_func.get_c_restype(db) == 'Signed'
+        assert PyPy_TypedefTest1.api_func.get_c_args(db) == 'Signed arg0'
+        assert PyPy_TypedefTest2.api_func.get_c_restype(db) == 'Signed *'
+        assert PyPy_TypedefTest2.api_func.get_c_args(db) == 'Signed *arg0'
 
         PyPy_TypedefTest1(space, 0)
         ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
@@ -113,7 +113,7 @@
 @pytest.mark.skipif(os.environ.get('USER')=='root',
                     reason='root can write to all files')
 def test_copy_header_files(tmpdir):
-    copy_header_files(tmpdir, True)
+    copy_header_files(cts, tmpdir, True)
     def check(name):
         f = tmpdir.join(name)
         assert f.check(file=True)
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -7,8 +7,7 @@
 from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject
 from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call, Py_buffer
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref, as_pyobj
-from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
-
+from pypy.module.cpyext.api import PyTypeObjectPtr
 
 
 class AppTestBytesObject(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -0,0 +1,218 @@
+from rpython.flowspace.model import const
+from rpython.flowspace.objspace import build_flow
+from rpython.translator.simplify import simplify_graph
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.module.cpyext.cparser import parse_source, CTypeSpace
+
+def test_configure():
+    decl = """
+    typedef ssize_t Py_ssize_t;
+
+    typedef struct {
+        Py_ssize_t ob_refcnt;
+        Py_ssize_t ob_pypy_link;
+        double ob_fval;
+    } TestFloatObject;
+    """
+    cts = parse_source(decl)
+    TestFloatObject = cts.definitions['TestFloatObject']
+    assert isinstance(TestFloatObject, lltype.Struct)
+    assert TestFloatObject.c_ob_refcnt == rffi.SSIZE_T
+    assert TestFloatObject.c_ob_pypy_link == rffi.SSIZE_T
+    assert TestFloatObject.c_ob_fval == rffi.DOUBLE
+
+def test_simple():
+    decl = "typedef ssize_t Py_ssize_t;"
+    cts = parse_source(decl)
+    assert cts.definitions == {'Py_ssize_t': rffi.SSIZE_T}
+
+def test_macro():
+    decl = """
+    typedef ssize_t Py_ssize_t;
+
+    #define PyObject_HEAD  \
+        Py_ssize_t ob_refcnt;        \
+        Py_ssize_t ob_pypy_link;     \
+
+    typedef struct {
+        PyObject_HEAD
+        double ob_fval;
+    } PyFloatObject;
+    """
+    cts = parse_source(decl)
+    assert 'PyFloatObject' in cts.definitions
+    assert 'PyObject_HEAD' in cts.macros
+
+def test_include():
+    cdef1 = """
+    typedef ssize_t Py_ssize_t;
+
+    #define PyObject_HEAD  \
+        Py_ssize_t ob_refcnt;        \
+        Py_ssize_t ob_pypy_link;     \
+
+    typedef struct {

From pypy.commits at gmail.com  Tue Jan 17 14:23:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 11:23:53 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix translation: add required casts
 from const char* to char*
Message-ID: <587e6f49.8a9a1c0a.46f5d.fc72@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89645:0cce543d57b5
Date: 2017-01-17 19:23 +0000
http://bitbucket.org/pypy/pypy/changeset/0cce543d57b5/

Log:	Fix translation: add required casts from const char* to char*

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,6 +1,6 @@
 from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER, cts)
 from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
@@ -10,10 +10,8 @@
     flags = pyobj.c_ob_type.c_tp_flags
     if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
         return 1
-    name = rffi.charp2str(pyobj.c_ob_type.c_tp_name)
+    name = rffi.charp2str(cts.cast('char*', pyobj.c_ob_type.c_tp_name))
     if  name in ('str', 'bytes'):
         # XXX remove once wrapper of __buffer__ -> bf_getbuffer works
         return 1
-    return 0  
-
-    
+    return 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -360,7 +360,8 @@
                 wrapper_func_kwds, doc, func_voidp, offset=offset)
         dict_w[method_name] = space.wrap(w_obj)
     if pto.c_tp_doc:
-        dict_w['__doc__'] = space.newbytes(rffi.charp2str(pto.c_tp_doc))
+        dict_w['__doc__'] = space.newbytes(
+            rffi.charp2str(cts.cast('char*', pto.c_tp_doc)))
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
 
@@ -487,7 +488,7 @@
         convert_getset_defs(space, dict_w, pto.c_tp_getset, self)
         convert_member_defs(space, dict_w, pto.c_tp_members, self)
 
-        name = rffi.charp2str(pto.c_tp_name)
+        name = rffi.charp2str(cts.cast('char*', pto.c_tp_name))
         flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         if flag_heaptype:
             minsize = rffi.sizeof(PyHeapTypeObject.TO)
@@ -506,7 +507,8 @@
               not (pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_slice)):
             self.flag_map_or_seq = 'M'
         if pto.c_tp_doc:
-            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
+            self.w_doc = space.newbytes(
+                rffi.charp2str(cts.cast('char*', pto.c_tp_doc)))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -800,7 +802,7 @@
     try:
         w_obj = _type_realize(space, py_obj)
     finally:
-        name = rffi.charp2str(pto.c_tp_name)
+        name = rffi.charp2str(cts.cast('char*', pto.c_tp_name))
         pto.c_tp_flags &= ~Py_TPFLAGS_READYING
     pto.c_tp_flags |= Py_TPFLAGS_READY
     return w_obj
@@ -907,7 +909,7 @@
     base = pto.c_tp_base
     base_pyo = rffi.cast(PyObject, pto.c_tp_base)
     if base and not base.c_tp_flags & Py_TPFLAGS_READY:
-        name = rffi.charp2str(base.c_tp_name)
+        name = rffi.charp2str(cts.cast('char*', base.c_tp_name))
         type_realize(space, base_pyo)
     if base and not pto.c_ob_type: # will be filled later
         pto.c_ob_type = base.c_ob_type

From pypy.commits at gmail.com  Tue Jan 17 14:32:34 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 11:32:34 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <587e7152.c3e31c0a.ebbbe.5813@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89646:2a79477f1501
Date: 2017-01-17 19:31 +0000
http://bitbucket.org/pypy/pypy/changeset/2a79477f1501/

Log:	hg merge default

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -359,7 +359,8 @@
                 wrapper_func_kwds, doc, func_voidp, offset=offset)
         dict_w[method_name] = space.wrap(w_obj)
     if pto.c_tp_doc:
-        dict_w['__doc__'] = space.newbytes(rffi.charp2str(pto.c_tp_doc))
+        dict_w['__doc__'] = space.newbytes(
+            rffi.charp2str(cts.cast('char*', pto.c_tp_doc)))
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
 
@@ -485,7 +486,7 @@
         convert_getset_defs(space, dict_w, pto.c_tp_getset, self)
         convert_member_defs(space, dict_w, pto.c_tp_members, self)
 
-        name = rffi.charp2str(pto.c_tp_name)
+        name = rffi.charp2str(cts.cast('char*', pto.c_tp_name))
         flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         if flag_heaptype:
             minsize = rffi.sizeof(PyHeapTypeObject.TO)
@@ -503,7 +504,8 @@
         elif pto.c_tp_as_mapping and pto.c_tp_as_mapping.c_mp_subscript:
             self.flag_map_or_seq = 'M'
         if pto.c_tp_doc:
-            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
+            self.w_doc = space.newbytes(
+                rffi.charp2str(cts.cast('char*', pto.c_tp_doc)))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -715,7 +717,7 @@
     try:
         w_obj = _type_realize(space, py_obj)
     finally:
-        name = rffi.charp2str(pto.c_tp_name)
+        name = rffi.charp2str(cts.cast('char*', pto.c_tp_name))
         pto.c_tp_flags &= ~Py_TPFLAGS_READYING
     pto.c_tp_flags |= Py_TPFLAGS_READY
     return w_obj
@@ -813,7 +815,7 @@
     base = pto.c_tp_base
     base_pyo = rffi.cast(PyObject, pto.c_tp_base)
     if base and not base.c_tp_flags & Py_TPFLAGS_READY:
-        name = rffi.charp2str(base.c_tp_name)
+        name = rffi.charp2str(cts.cast('char*', base.c_tp_name))
         type_realize(space, base_pyo)
     if base and not pto.c_ob_type: # will be filled later
         pto.c_ob_type = base.c_ob_type

From pypy.commits at gmail.com  Tue Jan 17 16:18:42 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 13:18:42 -0800 (PST)
Subject: [pypy-commit] pypy default: Use cparser in unicodeobject.py;
 add support for wchar_t
Message-ID: <587e8a32.54161c0a.a7cf8.aa3d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89647:c515efeff6d1
Date: 2017-01-17 21:18 +0000
http://bitbucket.org/pypy/pypy/changeset/c515efeff6d1/

Log:	Use cparser in unicodeobject.py; add support for wchar_t

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -671,7 +671,7 @@
                              % (cpyname, ))
 build_exported_objects()
 
-cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h'])
+cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h', 'stddef.h'])
 cts.parse_header(parse_dir / 'cpyext_object.h')
 
 Py_ssize_t = cts.gettype('Py_ssize_t')
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -664,6 +664,7 @@
 
 add_inttypes()
 CNAME_TO_LLTYPE['int'] = rffi.INT_real
+CNAME_TO_LLTYPE['wchar_t'] = lltype.UniChar
 
 def cname_to_lltype(name):
     return CNAME_TO_LLTYPE[name]
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -5,29 +5,7 @@
 extern "C" {
 #endif
 
-
-typedef unsigned int Py_UCS4;
-#ifdef HAVE_USABLE_WCHAR_T
-#define PY_UNICODE_TYPE wchar_t
-#elif Py_UNICODE_SIZE == 4
-#define PY_UNICODE_TYPE Py_UCS4
-#else
-#define PY_UNICODE_TYPE unsigned short
-#endif
-typedef PY_UNICODE_TYPE Py_UNICODE;
-
-#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
-
-typedef struct {
-    PyObject_HEAD
-    Py_UNICODE *str;
-    Py_ssize_t length;
-    long hash;                  /* Hash value; -1 if not set */
-    PyObject *defenc;           /* (Default) Encoded version as Python
-                                   string, or NULL; this is used for
-                                   implementing the buffer protocol */
-} PyUnicodeObject;
-
+#include <cpyext_unicodeobject.h>
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/parse/cpyext_unicodeobject.h b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
@@ -0,0 +1,16 @@
+typedef unsigned int Py_UCS4;
+/* On PyPy, Py_UNICODE is always wchar_t */
+#define PY_UNICODE_TYPE wchar_t
+typedef PY_UNICODE_TYPE Py_UNICODE;
+
+#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
+
+typedef struct {
+    PyObject_HEAD
+    Py_UNICODE *str;
+    Py_ssize_t length;
+    long hash;                  /* Hash value; -1 if not set */
+    PyObject *defenc;           /* (Default) Encoded version as Python
+                                   string, or NULL; this is used for
+                                   implementing the buffer protocol */
+} PyUnicodeObject;
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -190,6 +190,17 @@
     assert FUNC.RESULT == cts.gettype('func_t')
     assert FUNC.ARGS == (cts.gettype('TestFloatObject *'),)
 
+def test_wchar_t():
+    cdef = """
+    typedef struct { wchar_t* x; } test;
+    """
+    cts = parse_source(cdef, headers=['stddef.h'])
+    obj = lltype.malloc(cts.gettype('test'), flavor='raw')
+    obj.c_x = cts.cast('wchar_t*', 0)
+    obj.c_x =  lltype.nullptr(rffi.CWCHARP.TO)
+    lltype.free(obj, flavor='raw')
+
+
 def test_translate_cast():
     cdef = "typedef ssize_t Py_ssize_t;"
     cts = parse_source(cdef)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -3,12 +3,12 @@
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
-    bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
-    CONST_WSTRING, slot_function)
+    bootstrap_function, CONST_STRING,
+    CONST_WSTRING, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
-    make_typedescr, get_typedescr, as_pyobj)
+    make_typedescr, get_typedescr)
 from pypy.module.cpyext.bytesobject import PyString_Check
 from pypy.module.sys.interp_encoding import setdefaultencoding
 from pypy.module._codecs.interp_codecs import CodecState
@@ -19,12 +19,9 @@
 
 ## See comment in bytesobject.py.
 
-PyUnicodeObjectStruct = lltype.ForwardReference()
-PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct)
-PyUnicodeObjectFields = (PyObjectFields +
-    (("str", rffi.CWCHARP), ("length", Py_ssize_t),
-     ("hash", rffi.LONG), ("defenc", PyObject)))
-cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct)
+cts.parse_header(parse_dir / 'cpyext_unicodeobject.h')
+PyUnicodeObject = cts.gettype('PyUnicodeObject*')
+Py_UNICODE = cts.gettype('Py_UNICODE')
 
 @bootstrap_function
 def init_unicodeobject(space):
@@ -41,7 +38,6 @@
 
 PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode")
 
-Py_UNICODE = lltype.UniChar
 
 def new_empty_unicode(space, length):
     """
@@ -202,7 +198,7 @@
 def PyUnicode_GET_DATA_SIZE(space, w_obj):
     """Return the size of the object's internal buffer in bytes.  o has to be a
     PyUnicodeObject (not checked)."""
-    return rffi.sizeof(lltype.UniChar) * PyUnicode_GET_SIZE(space, w_obj)
+    return rffi.sizeof(Py_UNICODE) * PyUnicode_GET_SIZE(space, w_obj)
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PyUnicode_GET_SIZE(space, w_obj):

From pypy.commits at gmail.com  Tue Jan 17 19:33:29 2017
From: pypy.commits at gmail.com (wlav)
Date: Tue, 17 Jan 2017 16:33:29 -0800 (PST)
Subject: [pypy-commit] pypy default: from Ryan,
 Tobias: C++ language fixes for clang
Message-ID: <587eb7d9.85e11c0a.98277.1df4@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r89648:4baf1af96226
Date: 2017-01-17 16:23 -0800
http://bitbucket.org/pypy/pypy/changeset/4baf1af96226/

Log:	from Ryan, Tobias: C++ language fixes for clang

diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -522,7 +522,7 @@
    if ( FastCall( method, args, self, (void*)cppresult ) ) {
 	  cstr = cppstring_to_cstring( *cppresult );
       *length = cppresult->size();
-      cppresult->std::string::~string();
+      cppresult->std::string::~basic_string();
    } else
       *length = 0;
    free( (void*)cppresult ); 
@@ -712,7 +712,7 @@
          msg << "failed offset calculation between " << cb->GetName() << " and " << cd->GetName();
          // TODO: propagate this warning to caller w/o use of Python C-API
          // PyErr_Warn( PyExc_RuntimeWarning, const_cast<char*>( msg.str().c_str() ) );
-         std::cerr << "Warning: " << msg << '\n';
+         std::cerr << "Warning: " << msg.str() << '\n';
       }
 
    // return -1 to signal caller NOT to apply offset

From pypy.commits at gmail.com  Tue Jan 17 21:20:19 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 18:20:19 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix/hack around issues with clang and
 MSVC
Message-ID: <587ed0e3.4f831c0a.985af.41d9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89649:4cbdd68326aa
Date: 2017-01-18 01:36 +0000
http://bitbucket.org/pypy/pypy/changeset/4cbdd68326aa/

Log:	Fix/hack around issues with clang and MSVC

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -1,3 +1,4 @@
+import sys
 from collections import OrderedDict
 from cffi import api, model
 from cffi.commontypes import COMMON_TYPES, resolve_common_type
@@ -665,6 +666,8 @@
 add_inttypes()
 CNAME_TO_LLTYPE['int'] = rffi.INT_real
 CNAME_TO_LLTYPE['wchar_t'] = lltype.UniChar
+if 'ssize_t' not in CNAME_TO_LLTYPE:  # on Windows
+    CNAME_TO_LLTYPE['ssize_t'] = CNAME_TO_LLTYPE['long']
 
 def cname_to_lltype(name):
     return CNAME_TO_LLTYPE[name]
@@ -773,8 +776,13 @@
             for hdr in x.headers:
                 if hdr not in all_headers:
                     all_headers.append(hdr)
+        if sys.platform == 'win32':
+            compile_extra = ['-Dssize_t=long']
+        else:
+            compile_extra = []
         return ExternalCompilationInfo(
-            post_include_bits=all_sources, includes=all_headers)
+            post_include_bits=all_sources, includes=all_headers,
+            compile_extra=compile_extra)
 
     def configure_types(self):
         for name, (obj, quals) in self.ctx._declarations.iteritems():
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -57,13 +57,6 @@
 #endif
 #include <stdlib.h>
 
-#ifndef _WIN32
-typedef intptr_t Py_ssize_t;
-#else
-typedef long Py_ssize_t;
-#endif
-#define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
-#define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
 #define Py_SAFE_DOWNCAST(VALUE, WIDE, NARROW) (NARROW)(VALUE)
 
 #define Py_USING_UNICODE
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -7,12 +7,20 @@
 extern "C" {
 #endif
 
+/* Hack: MSVC doesn't support ssize_t */
+#ifdef _WIN32
+#define ssize_t long
+#endif
+#include <cpyext_object.h>
+#ifdef _WIN32
+#undef ssize_t
+#endif
 
-#include <cpyext_object.h>
+#define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
+#define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
 
 #define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
 
-
 /*
 CPython has this for backwards compatibility with really old extensions, and now
 we have it for compatibility with CPython.

From pypy.commits at gmail.com  Tue Jan 17 21:20:21 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 18:20:21 -0800 (PST)
Subject: [pypy-commit] pypy default: Undo 68058ce4557d and add a comment
Message-ID: <587ed0e5.e4361c0a.3f39d.4059@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89650:47cc21180694
Date: 2017-01-18 02:01 +0000
http://bitbucket.org/pypy/pypy/changeset/47cc21180694/

Log:	Undo 68058ce4557d and add a comment

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -665,6 +665,11 @@
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
+        if self.config.objspace.usemodules.cpyext:
+            # Special-case this to have state.install_dll() called early, which
+            # is required to initialise sys on Windows.
+            from pypy.module.cpyext.state import State
+            self.fromcache(State).build_api()
         self.getbuiltinmodule('sys')
         self.getbuiltinmodule('imp')
         self.getbuiltinmodule('__builtin__')
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -12,11 +12,6 @@
 
     atexit_funcs = []
 
-    def setup_after_space_initialization(self):
-        state = self.space.fromcache(State)
-        state.setup_rawrefcount()
-        state.build_api()
-
     def startup(self, space):
         space.fromcache(State).startup(space)
         method = pypy.module.cpyext.typeobject.get_new_method_def(space)
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -83,6 +83,7 @@
         This function is called when at object space creation,
         and drives the compilation of the cpyext library
         """
+        self.setup_rawrefcount()
         from pypy.module.cpyext import api
         if not self.space.config.translating:
             self.api_lib = str(api.build_bridge(self.space))

From pypy.commits at gmail.com  Tue Jan 17 21:20:23 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 18:20:23 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <587ed0e7.54b31c0a.b4442.3ee6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89651:b34a338909b6
Date: 2017-01-18 02:19 +0000
http://bitbucket.org/pypy/pypy/changeset/b34a338909b6/

Log:	merge heads

diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -522,7 +522,7 @@
    if ( FastCall( method, args, self, (void*)cppresult ) ) {
 	  cstr = cppstring_to_cstring( *cppresult );
       *length = cppresult->size();
-      cppresult->std::string::~string();
+      cppresult->std::string::~basic_string();
    } else
       *length = 0;
    free( (void*)cppresult ); 
@@ -712,7 +712,7 @@
          msg << "failed offset calculation between " << cb->GetName() << " and " << cd->GetName();
          // TODO: propagate this warning to caller w/o use of Python C-API
          // PyErr_Warn( PyExc_RuntimeWarning, const_cast<char*>( msg.str().c_str() ) );
-         std::cerr << "Warning: " << msg << '\n';
+         std::cerr << "Warning: " << msg.str() << '\n';
       }
 
    // return -1 to signal caller NOT to apply offset

From pypy.commits at gmail.com  Tue Jan 17 21:33:13 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 17 Jan 2017 18:33:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <587ed3e9.d3811c0a.7bfda.2949@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89652:bf8056bf0d0b
Date: 2017-01-18 02:32 +0000
http://bitbucket.org/pypy/pypy/changeset/bf8056bf0d0b/

Log:	hg merge default

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -662,6 +662,11 @@
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
+        if self.config.objspace.usemodules.cpyext:
+            # Special-case this to have state.install_dll() called early, which
+            # is required to initialise sys on Windows.
+            from pypy.module.cpyext.state import State
+            self.fromcache(State).build_api()
         self.getbuiltinmodule('sys')
         self.getbuiltinmodule('_imp')
         self.getbuiltinmodule('_frozen_importlib')
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -522,7 +522,7 @@
    if ( FastCall( method, args, self, (void*)cppresult ) ) {
 	  cstr = cppstring_to_cstring( *cppresult );
       *length = cppresult->size();
-      cppresult->std::string::~string();
+      cppresult->std::string::~basic_string();
    } else
       *length = 0;
    free( (void*)cppresult ); 
@@ -712,7 +712,7 @@
          msg << "failed offset calculation between " << cb->GetName() << " and " << cd->GetName();
          // TODO: propagate this warning to caller w/o use of Python C-API
          // PyErr_Warn( PyExc_RuntimeWarning, const_cast<char*>( msg.str().c_str() ) );
-         std::cerr << "Warning: " << msg << '\n';
+         std::cerr << "Warning: " << msg.str() << '\n';
       }
 
    // return -1 to signal caller NOT to apply offset
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -12,11 +12,6 @@
 
     atexit_funcs = []
 
-    def setup_after_space_initialization(self):
-        state = self.space.fromcache(State)
-        state.setup_rawrefcount()
-        state.build_api()
-
     def startup(self, space):
         space.fromcache(State).startup(space)
         method = pypy.module.cpyext.typeobject.get_new_method_def(space)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -674,7 +674,7 @@
                              % (cpyname, ))
 build_exported_objects()
 
-cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h'])
+cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h', 'stddef.h'])
 cts.parse_header(parse_dir / 'cpyext_object.h')
 
 Py_ssize_t = cts.gettype('Py_ssize_t')
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -1,3 +1,4 @@
+import sys
 from collections import OrderedDict
 from cffi import api, model
 from cffi.commontypes import COMMON_TYPES, resolve_common_type
@@ -664,6 +665,9 @@
 
 add_inttypes()
 CNAME_TO_LLTYPE['int'] = rffi.INT_real
+CNAME_TO_LLTYPE['wchar_t'] = lltype.UniChar
+if 'ssize_t' not in CNAME_TO_LLTYPE:  # on Windows
+    CNAME_TO_LLTYPE['ssize_t'] = CNAME_TO_LLTYPE['long']
 
 def cname_to_lltype(name):
     return CNAME_TO_LLTYPE[name]
@@ -772,8 +776,13 @@
             for hdr in x.headers:
                 if hdr not in all_headers:
                     all_headers.append(hdr)
+        if sys.platform == 'win32':
+            compile_extra = ['-Dssize_t=long']
+        else:
+            compile_extra = []
         return ExternalCompilationInfo(
-            post_include_bits=all_sources, includes=all_headers)
+            post_include_bits=all_sources, includes=all_headers,
+            compile_extra=compile_extra)
 
     def configure_types(self):
         for name, (obj, quals) in self.ctx._declarations.iteritems():
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -57,13 +57,6 @@
 #endif
 #include <stdlib.h>
 
-#ifndef _WIN32
-typedef intptr_t Py_ssize_t;
-#else
-typedef long Py_ssize_t;
-#endif
-#define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
-#define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
 #define Py_SAFE_DOWNCAST(VALUE, WIDE, NARROW) (NARROW)(VALUE)
 
 #define Py_USING_UNICODE
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -7,10 +7,20 @@
 extern "C" {
 #endif
 
+/* Hack: MSVC doesn't support ssize_t */
+#ifdef _WIN32
+#define ssize_t long
+#endif
+#include <cpyext_object.h>
+#ifdef _WIN32
+#undef ssize_t
+#endif
+
+#define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
+#define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
+
 #define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
 
-#include <cpyext_object.h>
-
 /*
 CPython has this for backwards compatibility with really old extensions, and now
 we have it for compatibility with CPython.
@@ -78,7 +88,6 @@
 #define Py_GT 4
 #define Py_GE 5
 
-
 /* Py3k buffer interface, adapted for PyPy */
     /* Flags for getting buffers */
 #define PyBUF_SIMPLE 0
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -5,26 +5,7 @@
 extern "C" {
 #endif
 
-
-typedef unsigned int Py_UCS4;
-#ifdef HAVE_USABLE_WCHAR_T
-#define PY_UNICODE_TYPE wchar_t
-#elif Py_UNICODE_SIZE == 4
-#define PY_UNICODE_TYPE Py_UCS4
-#else
-#define PY_UNICODE_TYPE unsigned short
-#endif
-typedef PY_UNICODE_TYPE Py_UNICODE;
-
-#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
-
-typedef struct {
-    PyObject_HEAD
-    Py_UNICODE *buffer;
-    Py_ssize_t length;
-    char *utf8buffer;
-} PyUnicodeObject;
-
+#include <cpyext_unicodeobject.h>
 
 PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char *format, va_list vargs);
 PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char *format, ...);
diff --git a/pypy/module/cpyext/parse/cpyext_unicodeobject.h b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
@@ -0,0 +1,13 @@
+typedef unsigned int Py_UCS4;
+/* On PyPy, Py_UNICODE is always wchar_t */
+#define PY_UNICODE_TYPE wchar_t
+typedef PY_UNICODE_TYPE Py_UNICODE;
+
+#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
+
+typedef struct {
+    PyObject_HEAD
+    Py_UNICODE *buffer;
+    Py_ssize_t length;
+    char *utf8buffer;
+} PyUnicodeObject;
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -83,6 +83,7 @@
         This function is called when at object space creation,
         and drives the compilation of the cpyext library
         """
+        self.setup_rawrefcount()
         from pypy.module.cpyext import api
         if not self.space.config.translating:
             self.api_lib = str(api.build_bridge(self.space))
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -190,6 +190,17 @@
     assert FUNC.RESULT == cts.gettype('func_t')
     assert FUNC.ARGS == (cts.gettype('TestFloatObject *'),)
 
+def test_wchar_t():
+    cdef = """
+    typedef struct { wchar_t* x; } test;
+    """
+    cts = parse_source(cdef, headers=['stddef.h'])
+    obj = lltype.malloc(cts.gettype('test'), flavor='raw')
+    obj.c_x = cts.cast('wchar_t*', 0)
+    obj.c_x =  lltype.nullptr(rffi.CWCHARP.TO)
+    lltype.free(obj, flavor='raw')
+
+
 def test_translate_cast():
     cdef = "typedef ssize_t Py_ssize_t;"
     cts = parse_source(cdef)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -3,8 +3,8 @@
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
-    bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
-    CONST_WSTRING, Py_CLEANUP_SUPPORTED, slot_function)
+    bootstrap_function, CONST_STRING,
+    CONST_WSTRING, Py_CLEANUP_SUPPORTED, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
@@ -18,12 +18,9 @@
 
 ## See comment in bytesobject.py.
 
-PyUnicodeObjectStruct = lltype.ForwardReference()
-PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct)
-PyUnicodeObjectFields = (PyObjectFields +
-    (("buffer", rffi.CWCHARP), ("length", Py_ssize_t),
-     ("utf8buffer", rffi.CCHARP)))
-cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct)
+cts.parse_header(parse_dir / 'cpyext_unicodeobject.h')
+PyUnicodeObject = cts.gettype('PyUnicodeObject*')
+Py_UNICODE = cts.gettype('Py_UNICODE')
 
 @bootstrap_function
 def init_unicodeobject(space):
@@ -40,7 +37,6 @@
 
 PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode")
 
-Py_UNICODE = lltype.UniChar
 
 def new_empty_unicode(space, length):
     """
@@ -195,7 +191,7 @@
 def PyUnicode_GET_DATA_SIZE(space, w_obj):
     """Return the size of the object's internal buffer in bytes.  o has to be a
     PyUnicodeObject (not checked)."""
-    return rffi.sizeof(lltype.UniChar) * PyUnicode_GET_SIZE(space, w_obj)
+    return rffi.sizeof(Py_UNICODE) * PyUnicode_GET_SIZE(space, w_obj)
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PyUnicode_GET_SIZE(space, w_obj):

From pypy.commits at gmail.com  Wed Jan 18 04:38:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 01:38:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix tests
Message-ID: <587f37b3.4f831c0a.985af.1f84@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89653:608eb2f6470b
Date: 2017-01-18 10:38 +0100
http://bitbucket.org/pypy/pypy/changeset/608eb2f6470b/

Log:	fix tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -83,6 +83,7 @@
         assert entry_bridge.match_by_id('call', """
             p30 = getfield_gc_r(ConstPtr(ptr29), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             p38 = call_r(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=<Callr . i EF=1 OS=5>)
+            p99 = getfield_gc_r(p38, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_sys_exc_operror .*>)
             p39 = getfield_gc_r(p38, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
             i40 = force_token()
             p41 = getfield_gc_r(p38, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
@@ -445,6 +446,7 @@
             p21 = getfield_gc_r(ConstPtr(ptr20), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             guard_not_invalidated(descr=...)
             p29 = call_r(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=<Callr . i EF=1 OS=5>)
+            p99 = getfield_gc_r(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_sys_exc_operror .*>)
             p30 = getfield_gc_r(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
             p31 = force_token()
             p32 = getfield_gc_r(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)

From pypy.commits at gmail.com  Wed Jan 18 06:44:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 03:44:18 -0800 (PST)
Subject: [pypy-commit] pypy default: sys.settrace(f) when we see a 'call'
 can return a different
Message-ID: <587f5512.51471c0a.eeb12.4226@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89654:f47609af5171
Date: 2017-01-18 12:43 +0100
http://bitbucket.org/pypy/pypy/changeset/f47609af5171/

Log:	sys.settrace(f) when we see a 'call' can return a different function
	g() or None. But such a function g() cannot return None: if it does
	then g() will continue to be called. See
	http://bugs.python.org/issue11992

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -336,7 +336,9 @@
                 try:
                     w_result = space.call_function(w_callback, space.wrap(frame), space.wrap(event), w_arg)
                     if space.is_w(w_result, space.w_None):
-                        d.w_f_trace = None
+                        # bug-to-bug compatibility with CPython
+                        # http://bugs.python.org/issue11992
+                        pass   #d.w_f_trace = None
                     else:
                         d.w_f_trace = w_result
                 except:
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -67,8 +67,12 @@
 
     def test_f_lineno_set(self):
         def tracer(f, *args):
+            def y(f, *args):
+                return y
             def x(f, *args):
                 f.f_lineno += 1
+                return y  # "return None" should have the same effect, but see
+                          # test_local_trace_function_returning_None_ignored
             return x
 
         open    # force fetching of this name now
@@ -602,3 +606,26 @@
         # on Python 3 we get an extra 'exception' when 'for' catches
         # StopIteration
         assert seen == ['call', 'line', 'call', 'return', 'return']
+
+    def test_local_trace_function_returning_None_ignored(self):
+        # behave the same as CPython does, and in contradiction with
+        # the documentation.
+        def tracer(f, event, arg):
+            assert event == 'call'
+            return local_tracer
+
+        seen = []
+        def local_tracer(f, event, arg):
+            seen.append(event)
+            return None     # but 'local_tracer' will be called again
+
+        def function():
+            a = 1
+            a = 2
+            a = 3
+
+        import sys
+        sys.settrace(tracer)
+        function()
+        sys.settrace(None)
+        assert seen == ["line", "line", "line", "return"]

From pypy.commits at gmail.com  Wed Jan 18 06:48:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 03:48:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <587f55fc.52301c0a.a3c4a.6493@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89655:4d7690bc23c2
Date: 2017-01-18 12:47 +0100
http://bitbucket.org/pypy/pypy/changeset/4d7690bc23c2/

Log:	hg merge default

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -325,7 +325,9 @@
                 try:
                     w_result = space.call_function(w_callback, space.wrap(frame), space.wrap(event), w_arg)
                     if space.is_w(w_result, space.w_None):
-                        d.w_f_trace = None
+                        # bug-to-bug compatibility with CPython
+                        # http://bugs.python.org/issue11992
+                        pass   #d.w_f_trace = None
                     else:
                         d.w_f_trace = w_result
                 except:
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -67,8 +67,12 @@
 
     def test_f_lineno_set(self):
         def tracer(f, *args):
+            def y(f, *args):
+                return y
             def x(f, *args):
                 f.f_lineno += 1
+                return y  # "return None" should have the same effect, but see
+                          # test_local_trace_function_returning_None_ignored
             return x
 
         # obscure: call open beforehand, py3k's open invokes some app
@@ -773,6 +777,29 @@
         assert frames[-4].f_code.co_name == 'f'
         """
 
+    def test_local_trace_function_returning_None_ignored(self):
+        # behave the same as CPython does, and in contradiction with
+        # the documentation.
+        def tracer(f, event, arg):
+            assert event == 'call'
+            return local_tracer
+
+        seen = []
+        def local_tracer(f, event, arg):
+            seen.append(event)
+            return None     # but 'local_tracer' will be called again
+
+        def function():
+            a = 1
+            a = 2
+            a = 3
+
+        import sys
+        sys.settrace(tracer)
+        function()
+        sys.settrace(None)
+        assert seen == ["line", "line", "line", "return"]
+
     def test_clear_locals(self):
         def make_frames():
             def outer():

From pypy.commits at gmail.com  Wed Jan 18 06:55:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 03:55:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix tests
Message-ID: <587f57c2.42061c0a.2db68.51bd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89656:4991383b53ef
Date: 2017-01-18 12:55 +0100
http://bitbucket.org/pypy/pypy/changeset/4991383b53ef/

Log:	fix tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -10,7 +10,7 @@
                 pass
             a = A()
             a.x = 1
-            for s in sys.modules.keys() * 1000:
+            for s in list(sys.modules.keys()) * 1000:
                 d.get(s)  # force pending setfields etc.
                 inc = a.x # ID: look
                 d[s] = d.get(s, 0) + inc
@@ -64,11 +64,14 @@
             i8 = int_lt(i5, i7)
             guard_true(i8, descr=...)
             guard_not_invalidated(descr=...)
-            p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
+            p109 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
+            guard_no_exception(descr=...)
+            guard_nonnull(p109, descr=...)
+            p10 = call_r(ConstClass(ll_str2unicode__rpy_stringPtr), p109, descr=<Callr . r EF=4 OS=2>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i99 = strhash(p10)
-            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=2>)
+            i99 = unicodehash(p10)
+            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_unicodePtr), p10, descr=<Calli . r EF=2>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
@@ -87,7 +90,7 @@
             call_n(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<Callv 0 rrrii EF=5>)
             setfield_gc(p20, i5, descr=<FieldS .*W_IntObject.inst_intval .* pure>)
             guard_no_exception(descr=...)
-            i98 = strhash(p10)
+            i98 = unicodehash(p10)
             i23 = call_i(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=<Calli . rrii EF=5 OS=4>)
             guard_no_exception(descr=...)
             i27 = int_lt(i23, 0)
@@ -115,7 +118,7 @@
             while i < n:
                 z = list(())
                 z.append(1)
-                i += z[-1] / len(z)
+                i += z[-1] // len(z)
             return i
 
         log = self.run(main, [1000])
@@ -247,7 +250,7 @@
                 n -= 1
 
         log = self.run(main, [1000])
-        assert log.result == main(1000)
+        assert log.result == None
         loop, = log.loops_by_filename(self.filepath)
         ops = loop.ops_by_id('getitem', include_guard_not_invalidated=False)
         assert log.opnames(ops) == []

From pypy.commits at gmail.com  Wed Jan 18 07:11:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 04:11:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Restore this test to its CPython version.
 Then skip a test in the
Message-ID: <587f5b66.85e11c0a.98277.6d5a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89657:946775085781
Date: 2017-01-18 13:10 +0100
http://bitbucket.org/pypy/pypy/changeset/946775085781/

Log:	Restore this test to its CPython version. Then skip a test in the
	middle (manually for now)

diff --git a/lib-python/3/test/test_peepholer.py b/lib-python/3/test/test_peepholer.py
--- a/lib-python/3/test/test_peepholer.py
+++ b/lib-python/3/test/test_peepholer.py
@@ -79,11 +79,11 @@
         # On CPython, "a,b,c=1,2,3" turns into "a,b,c=<constant (1,2,3)>"
         # but on PyPy, it turns into "a=1;b=2;c=3".
         for line, elem in (
-            ('a = 1,2,3', '((1, 2, 3))'),
-            ('("a","b","c")', "(('a', 'b', 'c'))"),
-            ('a,b,c = 1,2,3', '((1, 2, 3))'),
-            ('(None, 1, None)', '((None, 1, None))'),
-            ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'),
+            ('a = 1,2,3', (1, 2, 3)),
+            ('("a","b","c")', ('a', 'b', 'c')),
+            #('a,b,c = 1,2,3', (1, 2, 3)),
+            ('(None, 1, None)', (None, 1, None)),
+            ('((1, 2), 3, 4)', ((1, 2), 3, 4)),
             ):
             code = compile(line,'','single')
             self.assertInBytecode(code, 'LOAD_CONST', elem)

From pypy.commits at gmail.com  Wed Jan 18 07:15:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 04:15:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip on pypy only
Message-ID: <587f5c53.c3e31c0a.ebbbe.51ef@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89658:ed5cf18954c8
Date: 2017-01-18 13:14 +0100
http://bitbucket.org/pypy/pypy/changeset/ed5cf18954c8/

Log:	skip on pypy only

diff --git a/lib-python/3/test/test_peepholer.py b/lib-python/3/test/test_peepholer.py
--- a/lib-python/3/test/test_peepholer.py
+++ b/lib-python/3/test/test_peepholer.py
@@ -4,7 +4,7 @@
 from io import StringIO
 import unittest
 from math import copysign
-from test.support import cpython_only
+from test.support import cpython_only, impl_detail
 
 from test.bytecode_helper import BytecodeTestCase
 
@@ -76,16 +76,19 @@
             self.assertNotInBytecode(code, 'UNPACK_TUPLE')
 
     def test_folding_of_tuples_of_constants(self):
-        # On CPython, "a,b,c=1,2,3" turns into "a,b,c=<constant (1,2,3)>"
-        # but on PyPy, it turns into "a=1;b=2;c=3".
         for line, elem in (
             ('a = 1,2,3', (1, 2, 3)),
             ('("a","b","c")', ('a', 'b', 'c')),
-            #('a,b,c = 1,2,3', (1, 2, 3)),
+            ('a,b,c = 1,2,3', (1, 2, 3)),
             ('(None, 1, None)', (None, 1, None)),
             ('((1, 2), 3, 4)', ((1, 2), 3, 4)),
             ):
             code = compile(line,'','single')
+            if line == 'a,b,c = 1,2,3' and impl_detail(pypy=True):
+                # On CPython, "a,b,c=1,2,3" turns into
+                # "a,b,c=<constant (1,2,3)>"
+                # but on PyPy, it turns into "a=1;b=2;c=3".
+                continue
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             self.assertNotInBytecode(code, 'BUILD_TUPLE')
 

From pypy.commits at gmail.com  Wed Jan 18 07:19:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 04:19:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix error message
Message-ID: <587f5d43.54161c0a.82bc7.773c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89659:fd8e2f1e76e5
Date: 2017-01-18 13:18 +0100
http://bitbucket.org/pypy/pypy/changeset/fd8e2f1e76e5/

Log:	fix error message

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -300,7 +300,7 @@
                 raise oefmt(space.w_RuntimeError,
                             "coroutine wrapper %R attempted "
                             "to recursively wrap %R",
-                            w_wrapper, self)
+                            w_wrapper, self.getcode())
             ec.in_coroutine_wrapper = True
             try:
                 w_gen = space.call_function(w_wrapper, w_gen)

From pypy.commits at gmail.com  Wed Jan 18 11:33:52 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 18 Jan 2017 08:33:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Kill all Py_TPFLAGS that were removed in
 Python3
Message-ID: <587f98f0.958c1c0a.37608.ff06@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89660:0c80211612f5
Date: 2017-01-18 16:32 +0000
http://bitbucket.org/pypy/pypy/changeset/0c80211612f5/

Log:	Kill all Py_TPFLAGS that were removed in Python3

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,10 +120,10 @@
 udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 
 constant_names = """
-Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
+Py_TPFLAGS_READY Py_TPFLAGS_READYING
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
+Py_TPFLAGS_HEAPTYPE
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_MAX_NDIMS
 Py_CLEANUP_SUPPORTED
 PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -150,31 +150,6 @@
 given type object has a specified feature.
 */
 
-/* PyBufferProcs contains bf_getcharbuffer */
-#define Py_TPFLAGS_HAVE_GETCHARBUFFER  (1L<<0)
-
-/* PySequenceMethods contains sq_contains */
-#define Py_TPFLAGS_HAVE_SEQUENCE_IN (1L<<1)
-
-/* This is here for backwards compatibility.  Extensions that use the old GC
- * API will still compile but the objects will not be tracked by the GC. */
-#define Py_TPFLAGS_GC 0 /* used to be (1L<<2) */
-
-/* PySequenceMethods and PyNumberMethods contain in-place operators */
-#define Py_TPFLAGS_HAVE_INPLACEOPS (1L<<3)
-
-/* tp_richcompare is defined */
-#define Py_TPFLAGS_HAVE_RICHCOMPARE (1L<<5)
-
-/* Objects which are weakly referencable if their tp_weaklistoffset is >0 */
-#define Py_TPFLAGS_HAVE_WEAKREFS (1L<<6)
-
-/* tp_iter is defined */
-#define Py_TPFLAGS_HAVE_ITER (1L<<7)
-
-/* New members introduced by Python 2.2 exist */
-#define Py_TPFLAGS_HAVE_CLASS (1L<<8)
-
 /* Set if the type object is dynamically allocated */
 #define Py_TPFLAGS_HEAPTYPE (1L<<9)
 
@@ -197,9 +172,6 @@
 #define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION 0
 #endif
 
-/* Objects support nb_index in PyNumberMethods */
-#define Py_TPFLAGS_HAVE_INDEX (1L<<17)
-
 /* Objects support type attribute cache */
 #define Py_TPFLAGS_HAVE_VERSION_TAG   (1L<<18)
 #define Py_TPFLAGS_VALID_VERSION_TAG  (1L<<19)
@@ -208,33 +180,32 @@
 #define Py_TPFLAGS_IS_ABSTRACT (1L<<20)
 
 /* These flags are used to determine if a type is a subclass. */
-#define Py_TPFLAGS_INT_SUBCLASS		(1L<<23)
-#define Py_TPFLAGS_LONG_SUBCLASS	(1L<<24)
-#define Py_TPFLAGS_LIST_SUBCLASS	(1L<<25)
-#define Py_TPFLAGS_TUPLE_SUBCLASS	(1L<<26)
-#define Py_TPFLAGS_STRING_SUBCLASS	(1L<<27)
-#define Py_TPFLAGS_UNICODE_SUBCLASS	(1L<<28)
-#define Py_TPFLAGS_DICT_SUBCLASS	(1L<<29)
-#define Py_TPFLAGS_BASE_EXC_SUBCLASS	(1L<<30)
-#define Py_TPFLAGS_TYPE_SUBCLASS	(1L<<31)
+#define Py_TPFLAGS_LONG_SUBCLASS        (1UL << 24)
+#define Py_TPFLAGS_LIST_SUBCLASS        (1UL << 25)
+#define Py_TPFLAGS_TUPLE_SUBCLASS       (1UL << 26)
+#define Py_TPFLAGS_BYTES_SUBCLASS       (1UL << 27)
+#define Py_TPFLAGS_UNICODE_SUBCLASS     (1UL << 28)
+#define Py_TPFLAGS_DICT_SUBCLASS        (1UL << 29)
+#define Py_TPFLAGS_BASE_EXC_SUBCLASS    (1UL << 30)
+#define Py_TPFLAGS_TYPE_SUBCLASS        (1UL << 31)
 
-#define Py_TPFLAGS_DEFAULT_EXTERNAL ( \
-                             Py_TPFLAGS_HAVE_GETCHARBUFFER | \
-                             Py_TPFLAGS_HAVE_SEQUENCE_IN | \
-                             Py_TPFLAGS_HAVE_INPLACEOPS | \
-                             Py_TPFLAGS_HAVE_RICHCOMPARE | \
-                             Py_TPFLAGS_HAVE_WEAKREFS | \
-                             Py_TPFLAGS_HAVE_ITER | \
-                             Py_TPFLAGS_HAVE_CLASS | \
-                             Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \
-                             Py_TPFLAGS_HAVE_INDEX | \
-                             0)
-#define Py_TPFLAGS_DEFAULT_CORE (Py_TPFLAGS_DEFAULT_EXTERNAL | \
-                                 Py_TPFLAGS_HAVE_VERSION_TAG)
+#define Py_TPFLAGS_DEFAULT  ( \
+                 Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \
+                 Py_TPFLAGS_HAVE_VERSION_TAG | \
+                0)
 
-#define Py_TPFLAGS_DEFAULT Py_TPFLAGS_DEFAULT_EXTERNAL
+/* NOTE: The following flags reuse lower bits (removed as part of the
+ * Python 3.0 transition). */
 
+/* Type structure has tp_finalize member (3.4) */
+#define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0)
+
+#ifdef Py_LIMITED_API
+#define PyType_HasFeature(t,f)  ((PyType_GetFlags(t) & (f)) != 0)
+#else
 #define PyType_HasFeature(t,f)  (((t)->tp_flags & (f)) != 0)
+#endif
+#define PyType_FastSubclass(t,f)  PyType_HasFeature(t,f)
 
 /* objimpl.h ----------------------------------------------*/
 #define PyObject_New(type, typeobj) \
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -14,9 +14,9 @@
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
     slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_buffer,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
+    build_type_checkers,
     PyObjectFields, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_INPLACEOPS, cts, parse_dir)
+    cts, parse_dir)
 from pypy.module.cpyext.cparser import parse_source
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
@@ -421,7 +421,6 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
-    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
 
 def check_descr(space, w_self, w_type):
     if not space.isinstance_w(w_self, w_type):
@@ -574,7 +573,6 @@
     else:
         c_buf.c_bf_getbuffer = llslot(space, bf_getbuffer)
     pto.c_tp_as_buffer = c_buf
-    pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
 
 @slot_function([PyObject], lltype.Void)
 def type_dealloc(space, obj):
@@ -798,10 +796,8 @@
         # XXX refactor - parts of this are done in finish_type_2 -> inherit_slots
         if not py_type.c_tp_as_number:
             py_type.c_tp_as_number = base.c_tp_as_number
-            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_sequence:
             py_type.c_tp_as_sequence = base.c_tp_as_sequence
-            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping:
             py_type.c_tp_as_mapping = base.c_tp_as_mapping
         #if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer

From pypy.commits at gmail.com  Wed Jan 18 12:33:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 18 Jan 2017 09:33:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add dummy _PyGC_FINALIZED macro
Message-ID: <587fa6ff.c80e1c0a.d0d12.2040@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89661:7448016b9307
Date: 2017-01-18 17:33 +0000
http://bitbucket.org/pypy/pypy/changeset/7448016b9307/

Log:	Add dummy _PyGC_FINALIZED macro

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -249,6 +249,9 @@
     char dummy;
 } PyGC_Head;
 
+/* dummy GC macros */
+#define _PyGC_FINALIZED(o) 1
+
 /* Utility macro to help write tp_traverse functions.
  * To use this macro, the tp_traverse function must name its arguments
  * "visit" and "arg".  This is intended to keep tp_traverse functions

From pypy.commits at gmail.com  Wed Jan 18 13:16:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 10:16:52 -0800 (PST)
Subject: [pypy-commit] pypy default: rposix.{get,set}priority()
Message-ID: <587fb114.c3e31c0a.ebbbe.1c19@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89662:b7bdc19909d9
Date: 2017-01-18 19:13 +0100
http://bitbucket.org/pypy/pypy/changeset/b7bdc19909d9/

Log:	rposix.{get,set}priority()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -234,6 +234,7 @@
         _ptyh = 'pty.h'
     includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
+                'sys/resource.h',
                 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('freebsd'):
@@ -249,6 +250,9 @@
     SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET')
     SEEK_CUR = rffi_platform.DefinedConstantInteger('SEEK_CUR')
     SEEK_END = rffi_platform.DefinedConstantInteger('SEEK_END')
+    PRIO_PROCESS = rffi_platform.DefinedConstantInteger('PRIO_PROCESS')
+    PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP')
+    PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
@@ -261,6 +265,7 @@
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
+        ID_T = rffi_platform.SimpleType('id_t', rffi.UINT)
         TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
@@ -1734,6 +1739,22 @@
     def setresgid(rgid, egid, sgid):
         handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
 
+    c_getpriority = external('getpriority', [rffi.INT, ID_T], rffi.INT,
+                             save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+    c_setpriority = external('setpriority', [rffi.INT, ID_T, rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+
+    def getpriority(which, who):
+        result = widen(c_getpriority(which, who))
+        error = get_saved_errno()
+        if error != 0:
+            raise OSError(error, 'getpriority failed')
+        return result
+
+    def setpriority(which, who, prio):
+        handle_posix_error('setpriority', c_setpriority(which, who, prio))
+
+
 #___________________________________________________________________
 
 c_chroot = external('chroot', [rffi.CCHARP], rffi.INT,
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -669,3 +669,10 @@
     finally:
         os.close(fd1)
         os.close(fd2)
+
+ at rposix_requires('getpriority')
+def test_getpriority():
+    # a "don't crash" kind of test only
+    prio = rposix.getpriority(rposix.PRIO_PROCESS, 0)
+    rposix.setpriority(rposix.PRIO_PROCESS, 0, prio)
+    py.test.raises(OSError, rposix.getpriority, rposix.PRIO_PGRP, 123456789)

From pypy.commits at gmail.com  Wed Jan 18 13:16:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 10:16:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <587fb116.145e1c0a.eb01a.1ee6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89663:920d60110d57
Date: 2017-01-18 19:14 +0100
http://bitbucket.org/pypy/pypy/changeset/920d60110d57/

Log:	hg merge default

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -234,6 +234,7 @@
         _ptyh = 'pty.h'
     includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
+                'sys/resource.h',
                 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('freebsd'):
@@ -249,6 +250,9 @@
     SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET')
     SEEK_CUR = rffi_platform.DefinedConstantInteger('SEEK_CUR')
     SEEK_END = rffi_platform.DefinedConstantInteger('SEEK_END')
+    PRIO_PROCESS = rffi_platform.DefinedConstantInteger('PRIO_PROCESS')
+    PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP')
+    PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
@@ -261,6 +265,7 @@
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
+        ID_T = rffi_platform.SimpleType('id_t', rffi.UINT)
         TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
@@ -1734,6 +1739,22 @@
     def setresgid(rgid, egid, sgid):
         handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
 
+    c_getpriority = external('getpriority', [rffi.INT, ID_T], rffi.INT,
+                             save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+    c_setpriority = external('setpriority', [rffi.INT, ID_T, rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+
+    def getpriority(which, who):
+        result = widen(c_getpriority(which, who))
+        error = get_saved_errno()
+        if error != 0:
+            raise OSError(error, 'getpriority failed')
+        return result
+
+    def setpriority(which, who, prio):
+        handle_posix_error('setpriority', c_setpriority(which, who, prio))
+
+
 #___________________________________________________________________
 
 c_chroot = external('chroot', [rffi.CCHARP], rffi.INT,
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -669,3 +669,10 @@
     finally:
         os.close(fd1)
         os.close(fd2)
+
+ at rposix_requires('getpriority')
+def test_getpriority():
+    # a "don't crash" kind of test only
+    prio = rposix.getpriority(rposix.PRIO_PROCESS, 0)
+    rposix.setpriority(rposix.PRIO_PROCESS, 0, prio)
+    py.test.raises(OSError, rposix.getpriority, rposix.PRIO_PGRP, 123456789)

From pypy.commits at gmail.com  Wed Jan 18 13:37:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jan 2017 10:37:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: posix.{get,set}priority()
Message-ID: <587fb5e6.8a9a1c0a.46f5d.cd03@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89664:f979a9155582
Date: 2017-01-18 19:36 +0100
http://bitbucket.org/pypy/pypy/changeset/f979a9155582/

Log:	posix.{get,set}priority()

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -206,6 +206,13 @@
         interpleveldefs['get_blocking'] = 'interp_posix.get_blocking'
         interpleveldefs['set_blocking'] = 'interp_posix.set_blocking'
 
+    if hasattr(rposix, 'getpriority'):
+        interpleveldefs['getpriority'] = 'interp_posix.getpriority'
+        interpleveldefs['setpriority'] = 'interp_posix.setpriority'
+        for _name in ['PRIO_PROCESS', 'PRIO_PGRP', 'PRIO_USER']:
+            assert getattr(rposix, _name) is not None, "missing %r" % (_name,)
+            interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
+
     for _name in ["O_CLOEXEC"]:
         if getattr(rposix, _name) is not None:
             interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1900,6 +1900,29 @@
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
 
+ at unwrap_spec(which=int, who=int)
+def getpriority(space, which, who):
+    """ getpriority(which, who) -> int
+
+    Get program scheduling priority.
+    """
+    try:
+        returned_priority = rposix.getpriority(which, who)
+    except OSError as e:
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.wrap(returned_priority)
+
+ at unwrap_spec(which=int, who=int, priority=int)
+def setpriority(space, which, who, priority):
+    """ setpriority(which, who, priority)
+
+    Set program scheduling priority.
+    """
+    try:
+        rposix.setpriority(which, who, priority)
+    except OSError as e:
+        raise wrap_oserror(space, e, eintr_retry=False)
+
 def declare_new_w_star(name):
     if name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG'):
         @unwrap_spec(status=c_int)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -853,6 +853,31 @@
         assert st.st_size == 10000000000
     test_largefile.need_sparse_files = True
 
+    if hasattr(rposix, 'getpriority'):
+        def test_os_set_get_priority(self):
+            posix, os = self.posix, self.os
+            childpid = os.fork()
+            if childpid == 0:
+                # in the child (avoids changing the priority of the parent
+                # process)
+                orig_priority = posix.getpriority(posix.PRIO_PROCESS,
+                                                  os.getpid())
+                orig_grp_priority = posix.getpriority(posix.PRIO_PGRP,
+                                                      os.getpgrp())
+                posix.setpriority(posix.PRIO_PROCESS, os.getpid(),
+                                  orig_priority + 1)
+                new_priority = posix.getpriority(posix.PRIO_PROCESS,
+                                                 os.getpid())
+                assert new_priority == orig_priority + 1
+                assert posix.getpriority(posix.PRIO_PGRP, os.getpgrp()) == (
+                    orig_grp_priority)
+                os._exit(0)    # ok
+            #
+            pid1, status1 = os.waitpid(childpid, 0)
+            assert pid1 == childpid
+            assert os.WIFEXITED(status1)
+            assert os.WEXITSTATUS(status1) == 0   # else, test failure
+
     def test_write_buffer(self):
         os = self.posix
         fd = os.open(self.path2 + 'test_write_buffer',

From pypy.commits at gmail.com  Wed Jan 18 13:57:01 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 18 Jan 2017 10:57:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Implement PyImport_ImportModuleLevelObject
Message-ID: <587fba7d.54161c0a.82bc7.58b7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89665:5e0d28baa9d9
Date: 2017-01-18 18:53 +0000
http://bitbucket.org/pypy/pypy/changeset/5e0d28baa9d9/

Log:	Implement PyImport_ImportModuleLevelObject

diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -1,8 +1,8 @@
-from pypy.interpreter import module
 from pypy.module.cpyext.api import (
-    generic_cpy_call, cpython_api, PyObject, CONST_STRING, CANNOT_FAIL)
+    cpython_api, PyObject, CONST_STRING, CANNOT_FAIL,
+    cts, api_decl)
 from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.module import Module
 from pypy.interpreter.pycode import PyCode
 from pypy.module.imp import importing
@@ -50,6 +50,28 @@
         space.w_RuntimeWarning)
     return PyImport_Import(space, space.wrap(rffi.charp2str(name)))
 
+
+ at api_decl(
+    '''PyObject * PyImport_ImportModuleLevelObject(
+        PyObject *name, PyObject *given_globals, PyObject *locals,
+        PyObject *given_fromlist, int level)''', cts)
+def PyImport_ImportModuleLevelObject(space, w_name, w_glob, w_loc, w_fromlist, level):
+    if w_glob is None:
+        w_glob = space.newdict()
+    else:
+        if level > 0 and not space.isinstance_w(w_glob, space.w_dict):
+            raise oefmt(space.w_TypeError, "globals must be a dict")
+    if w_fromlist is None:
+        w_fromlist = space.newlist([])
+    if w_name is None:
+        raise oefmt(space.w_ValueError, "Empty module name")
+    w_import = space.builtin.get('__import__')
+    if level < 0:
+        raise oefmt(space.w_ValueError, "level must be >= 0")
+    return space.call_function(
+        w_import, w_name, w_glob, w_loc, w_fromlist, space.newint(level))
+
+
 @cpython_api([PyObject], PyObject)
 def PyImport_ReloadModule(space, w_mod):
     w_import = space.builtin.get('__import__')
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -40,6 +40,12 @@
         stat = PyImport_ReloadModule(space, stat)
         assert space.getattr(stat, space.wrap("S_IMODE"))
 
+    def test_ImportModuleLevelObject(self, space):
+        w_mod = PyImport_ImportModuleLevelObject(
+            space, space.wrap('stat'), None, None, None, 0)
+        assert w_mod
+        assert space.getattr(w_mod, space.wrap("S_IMODE"))
+
     def test_lock(self, space):
         # "does not crash"
         _PyImport_AcquireLock(space, )

From pypy.commits at gmail.com  Wed Jan 18 14:38:57 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 18 Jan 2017 11:38:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix warning on clang
Message-ID: <587fc451.05371c0a.39090.6544@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89666:60a8a54cf3fa
Date: 2017-01-18 19:20 +0000
http://bitbucket.org/pypy/pypy/changeset/60a8a54cf3fa/

Log:	fix warning on clang

diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2091,7 +2091,7 @@
     if (len == 0) {
         return PyUnicode_FromFormat("array('%c')", (int)typecode);
     }
-    if ((typecode == 'u'))
+    if (typecode == 'u')
         v = array_tounicode(a, NULL);
     else
         v = array_tolist(a, NULL);

From pypy.commits at gmail.com  Wed Jan 18 14:38:59 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 18 Jan 2017 11:38:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add C implementation (from CPython) for
 PyImport_ImportModuleLevel and PyImport_ImportModuleEx
Message-ID: <587fc453.0e1d1c0a.46224.cdbb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89667:00eea104b29e
Date: 2017-01-18 19:38 +0000
http://bitbucket.org/pypy/pypy/changeset/00eea104b29e/

Log:	Add C implementation (from CPython) for PyImport_ImportModuleLevel
	and PyImport_ImportModuleEx

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -583,6 +583,8 @@
 
     'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
 
+    'PyImport_ImportModuleLevel',
+
     'PyOS_getsig', 'PyOS_setsig',
     'PyThread_get_thread_ident', 'PyThread_allocate_lock', 'PyThread_free_lock',
     'PyThread_acquire_lock', 'PyThread_release_lock',
@@ -1296,6 +1298,7 @@
                          source_dir / "pymem.c",
                          source_dir / "bytesobject.c",
                          source_dir / "complexobject.c",
+                         source_dir / "import.c",
                          ]
 
 def build_eci(code, use_micronumpy=False, translating=False):
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -52,7 +52,7 @@
 
 
 @api_decl(
-    '''PyObject * PyImport_ImportModuleLevelObject(
+    '''PyObject* PyImport_ImportModuleLevelObject(
         PyObject *name, PyObject *given_globals, PyObject *locals,
         PyObject *given_fromlist, int level)''', cts)
 def PyImport_ImportModuleLevelObject(space, w_name, w_glob, w_loc, w_fromlist, level):
diff --git a/pypy/module/cpyext/include/import.h b/pypy/module/cpyext/include/import.h
--- a/pypy/module/cpyext/include/import.h
+++ b/pypy/module/cpyext/include/import.h
@@ -1,1 +1,26 @@
-/* empty */
+
+/* Module definition and import interface */
+
+#ifndef Py_IMPORT_H
+#define Py_IMPORT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(PyObject *) PyImport_ImportModuleLevel(
+    const char *name,           /* UTF-8 encoded string */
+    PyObject *globals,
+    PyObject *locals,
+    PyObject *fromlist,
+    int level
+    );
+
+#define PyImport_ImportModuleEx(n, g, l, f) \
+    PyImport_ImportModuleLevel(n, g, l, f, 0)
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_IMPORT_H */
diff --git a/pypy/module/cpyext/src/import.c b/pypy/module/cpyext/src/import.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/import.c
@@ -0,0 +1,15 @@
+#include <Python.h>
+
+PyObject *
+PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals,
+                           PyObject *fromlist, int level)
+{
+    PyObject *nameobj, *mod;
+    nameobj = PyUnicode_FromString(name);
+    if (nameobj == NULL)
+        return NULL;
+    mod = PyImport_ImportModuleLevelObject(nameobj, globals, locals,
+                                           fromlist, level);
+    Py_DECREF(nameobj);
+    return mod;
+}
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -686,38 +686,6 @@
     NULL."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP, PyObject, PyObject, PyObject], PyObject)
-def PyImport_ImportModuleEx(space, name, globals, locals, fromlist):
-    """
-
-
-
-    Import a module.  This is best described by referring to the built-in Python
-    function __import__(), as the standard __import__() function calls
-    this function directly.
-
-    The return value is a new reference to the imported module or top-level
-    package, or NULL with an exception set on failure.  Like for
-    __import__(), the return value when a submodule of a package was
-    requested is normally the top-level package, unless a non-empty fromlist
-    was given.
-
-    Failing imports remove incomplete module objects, like with
-    PyImport_ImportModule()."""
-    raise NotImplementedError
-
- at cpython_api([rffi.CCHARP, PyObject, PyObject, PyObject, rffi.INT_real], PyObject)
-def PyImport_ImportModuleLevel(space, name, globals, locals, fromlist, level):
-    """Import a module.  This is best described by referring to the built-in Python
-    function __import__(), as the standard __import__() function calls
-    this function directly.
-
-    The return value is a new reference to the imported module or top-level package,
-    or NULL with an exception set on failure.  Like for __import__(),
-    the return value when a submodule of a package was requested is normally the
-    top-level package, unless a non-empty fromlist was given."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyImport_ExecCodeModuleWithPathnames(space, name, co, pathname, cpathname):
     """Like PyImport_ExecCodeModuleEx(), but the __cached__

From pypy.commits at gmail.com  Wed Jan 18 15:04:32 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 18 Jan 2017 12:04:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix translation
Message-ID: <587fca50.896f1c0a.ce77e.8049@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89668:8a00871322e3
Date: 2017-01-18 20:03 +0000
http://bitbucket.org/pypy/pypy/changeset/8a00871322e3/

Log:	fix translation

diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -56,6 +56,7 @@
         PyObject *name, PyObject *given_globals, PyObject *locals,
         PyObject *given_fromlist, int level)''', cts)
 def PyImport_ImportModuleLevelObject(space, w_name, w_glob, w_loc, w_fromlist, level):
+    level = rffi.cast(lltype.Signed, level)
     if w_glob is None:
         w_glob = space.newdict()
     else:

From pypy.commits at gmail.com  Thu Jan 19 03:56:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 19 Jan 2017 00:56:23 -0800 (PST)
Subject: [pypy-commit] pypy default: Rename this file to start with "z",
 as it is very slow to run
Message-ID: <58807f37.145e1c0a.eb01a.cb89@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89670:b1c862cb933e
Date: 2017-01-19 09:55 +0100
http://bitbucket.org/pypy/pypy/changeset/b1c862cb933e/

Log:	Rename this file to start with "z", as it is very slow to run

diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_zvector.py
rename from rpython/jit/metainterp/test/test_vector.py
rename to rpython/jit/metainterp/test/test_zvector.py

From pypy.commits at gmail.com  Thu Jan 19 03:56:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 19 Jan 2017 00:56:21 -0800 (PST)
Subject: [pypy-commit] pypy default: issue2470
Message-ID: <58807f35.c3e31c0a.ebbbe.c6da@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89669:5390a8769eb3
Date: 2017-01-19 09:50 +0100
http://bitbucket.org/pypy/pypy/changeset/5390a8769eb3/

Log:	issue2470

	Remove FrontendTagOverflow and raise directly SwitchToBlackhole
	(there are a few obscure paths where the convertion would not
	occur).

	Catch such SwitchToBlackhole more widely to avoid other obscure
	paths where it could escape the JIT.

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -726,15 +726,7 @@
             op.setref_base(value)
 
     def _record_op(self, opnum, argboxes, descr=None):
-        from rpython.jit.metainterp.opencoder import FrontendTagOverflow
-
-        try:
-            return self.trace.record_op(opnum, argboxes, descr)
-        except FrontendTagOverflow:
-            # note that with the default settings this one should not
-            # happen - however if we hit that case, we don't get
-            # anything disabled
-            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        return self.trace.record_op(opnum, argboxes, descr)
 
     @specialize.argtype(3)
     def record(self, opnum, argboxes, value, descr=None):
diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py
--- a/rpython/jit/metainterp/opencoder.py
+++ b/rpython/jit/metainterp/opencoder.py
@@ -49,8 +49,12 @@
     way up to lltype.Signed for indexes everywhere
     """
 
-class FrontendTagOverflow(Exception):
-    pass
+def frontend_tag_overflow():
+    # Minor abstraction leak: raise directly the right exception
+    # expected by the rest of the machinery
+    from rpython.jit.metainterp import history
+    from rpython.rlib.jit import Counters
+    raise history.SwitchToBlackhole(Counters.ABORT_TOO_LONG)
 
 class BaseTrace(object):
     pass
@@ -296,7 +300,7 @@
             # grow by 2X
             self._ops = self._ops + [rffi.cast(model.STORAGE_TP, 0)] * len(self._ops)
         if not model.MIN_VALUE <= v <= model.MAX_VALUE:
-            raise FrontendTagOverflow
+            raise frontend_tag_overflow()
         self._ops[self._pos] = rffi.cast(model.STORAGE_TP, v)
         self._pos += 1
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2424,7 +2424,6 @@
         self.staticdata.profiler.start_tracing()
         assert jitdriver_sd is self.jitdriver_sd
         self.staticdata.try_to_free_some_loops()
-        self.create_empty_history()
         try:
             original_boxes = self.initialize_original_boxes(jitdriver_sd, *args)
             return self._compile_and_run_once(original_boxes)
@@ -2438,10 +2437,11 @@
         num_green_args = self.jitdriver_sd.num_green_args
         original_greenkey = original_boxes[:num_green_args]
         self.resumekey = compile.ResumeFromInterpDescr(original_greenkey)
-        self.history.set_inputargs(original_boxes[num_green_args:],
-                                   self.staticdata)
         self.seen_loop_header_for_jdindex = -1
         try:
+            self.create_empty_history()
+            self.history.set_inputargs(original_boxes[num_green_args:],
+                                       self.staticdata)
             self.interpret()
         except SwitchToBlackhole as stb:
             self.run_blackhole_interp_to_cancel_tracing(stb)
@@ -2461,9 +2461,11 @@
         if self.resumekey_original_loop_token is None:
             raise compile.giveup() # should be rare
         self.staticdata.try_to_free_some_loops()
-        inputargs = self.initialize_state_from_guard_failure(key, deadframe)
         try:
+            inputargs = self.initialize_state_from_guard_failure(key, deadframe)
             return self._handle_guard_failure(resumedescr, key, inputargs, deadframe)
+        except SwitchToBlackhole as stb:
+            self.run_blackhole_interp_to_cancel_tracing(stb)
         finally:
             self.resumekey_original_loop_token = None
             self.staticdata.profiler.end_tracing()
@@ -2475,13 +2477,10 @@
         self.seen_loop_header_for_jdindex = -1
         if isinstance(key, compile.ResumeAtPositionDescr):
             self.seen_loop_header_for_jdindex = self.jitdriver_sd.index
-        try:
-            self.prepare_resume_from_failure(deadframe, inputargs, resumedescr)
-            if self.resumekey_original_loop_token is None:   # very rare case
-                raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
-            self.interpret()
-        except SwitchToBlackhole as stb:
-            self.run_blackhole_interp_to_cancel_tracing(stb)
+        self.prepare_resume_from_failure(deadframe, inputargs, resumedescr)
+        if self.resumekey_original_loop_token is None:   # very rare case
+            raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
+        self.interpret()
         assert False, "should always raise"
 
     def run_blackhole_interp_to_cancel_tracing(self, stb):
diff --git a/rpython/jit/metainterp/test/test_opencoder.py b/rpython/jit/metainterp/test/test_opencoder.py
--- a/rpython/jit/metainterp/test/test_opencoder.py
+++ b/rpython/jit/metainterp/test/test_opencoder.py
@@ -1,6 +1,5 @@
 import py
 from rpython.jit.metainterp.opencoder import Trace, untag, TAGINT, TAGBOX
-from rpython.jit.metainterp.opencoder import FrontendTagOverflow
 from rpython.jit.metainterp.resoperation import rop, AbstractResOp
 from rpython.jit.metainterp.history import ConstInt, IntFrontendOp
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
@@ -8,6 +7,7 @@
 from rpython.jit.metainterp.test.strategies import lists_of_operations
 from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest
 from rpython.jit.metainterp.history import TreeLoop, AbstractDescr
+from rpython.jit.metainterp.history import SwitchToBlackhole
 from hypothesis import given, strategies
 
 class JitCode(object):
@@ -209,5 +209,5 @@
     def test_tag_overflow(self):
         t = Trace([], metainterp_sd)
         i0 = FakeOp(100000)
-        py.test.raises(FrontendTagOverflow, t.record_op, rop.FINISH, [i0])
+        py.test.raises(SwitchToBlackhole, t.record_op, rop.FINISH, [i0])
         assert t.unpack() == ([], [])

From pypy.commits at gmail.com  Thu Jan 19 08:39:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 19 Jan 2017 05:39:31 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe: in-progress
Message-ID: <5880c193.8a281c0a.ef2f3.7537@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe
Changeset: r89671:8ecb513bacb4
Date: 2017-01-19 14:38 +0100
http://bitbucket.org/pypy/pypy/changeset/8ecb513bacb4/

Log:	in-progress

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -72,6 +72,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT_SHIFT
 from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import rthread
 from rpython.memory.gc.minimarkpage import out_of_memory
 
 #
@@ -190,10 +191,9 @@
 FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
 NURSARRAY = lltype.Array(llmemory.Address)
 
-GCTL = lltype.Struct('GCThreadLocal',
-                     ('nursery_free', llmemory.Address),
-                     ('nursery_top', llmemory.Address),
-                     hints={'thread_local': True})
+NURSERY_FREE = rthread.ThreadLocalField(llmemory.Address, 'nursery_free')
+NURSERY_TOP  = rthread.ThreadLocalField(llmemory.Address, 'nursery_top')
+NEXT_NUBLOCK = rthread.ThreadLocalField(llmemory.Address, 'next_nublock')
 
 # ____________________________________________________________
 
@@ -275,7 +275,7 @@
 
         # Objects whose total size is at least 'large_object' bytes are
         # allocated out of the nursery immediately, as old objects.
-        "large_object": 13000,
+        "large_object": 26000,
 
         # Thread-local Block size: the nursery is divided into blocks of
         # at most this size each, and allocations go on in a
@@ -285,12 +285,10 @@
         # allocated nursery size is 2 times "tl_block_size".
         # "cache_line_min" is used to round the actual thread-local
         # blocks to a cache line, to avoid pointless cache conflicts.
-        "tl_block_size": 32768,
+        "tl_block_size": 65536,
         "cache_line_min": 256,
         }
 
-    tl = lltype.malloc(GCTL, flavor='raw', immortal=True)
-
     def __init__(self, config,
                  read_from_env=False,
                  nursery_size=32*WORD,
@@ -335,8 +333,8 @@
         self.cache_line_min = cache_line_min
         #
         self.nursery      = llmemory.NULL
-        self.tl.nursery_free = llmemory.NULL
-        self.tl.nursery_top  = llmemory.NULL
+        self.nublocks     = llmemory.NULL # <= linked list of threadlocal_base
+                                          # for all active nursery consumers
         self.debug_tiny_nursery = -1
         self.debug_rotating_nurseries = lltype.nullptr(NURSARRAY)
         self.extra_threshold = 0
@@ -539,6 +537,7 @@
                   "cache_line_min is not a power a two")
         self.tl_block_size = ((self.tl_block_size + self.cache_line_min - 1)
                               & ~(self.cache_line_min - 1))
+        self.gil_gc_color = llop.new_gil_color(lltype.Signed)
 
 
     def _nursery_memory_size(self):
@@ -636,6 +635,15 @@
                         "size", self.nursery_size)
             debug_stop("gc-debug")
 
+    get_nursery_free = NURSERY_FREE.getraw
+    set_nursery_free = NURSERY_FREE.setraw
+
+    get_nursery_top = NURSERY_TOP.getraw
+    set_nursery_top = NURSERY_TOP.setraw
+
+    get_next_nublock = NEXT_NUBLOCK.getraw
+    set_next_nublock = NEXT_NUBLOCK.setraw
+
 
     def malloc_fixedsize(self, typeid, size,
                                needs_finalizer=False,
@@ -673,10 +681,11 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.tl.nursery_free
-            ll_assert(result != llmemory.NULL, "uninitialized nursery")
-            self.tl.nursery_free = new_free = result + totalsize
-            if new_free > self.tl.nursery_top:
+            result = self.get_nursery_free()
+            #ll_assert(result != llmemory.NULL, "uninitialized nursery")
+            new_free = result + totalsize
+            self.set_nursery_free(new_free)
+            if new_free > self.get_nursery_top():
                 result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
@@ -733,10 +742,11 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.tl.nursery_free
-            ll_assert(result != llmemory.NULL, "uninitialized nursery")
-            self.tl.nursery_free = new_free = result + totalsize
-            if new_free > self.tl.nursery_top:
+            result = self.get_nursery_free()
+            #ll_assert(result != llmemory.NULL, "uninitialized nursery")
+            new_free = result + totalsize
+            self.set_nursery_free(new_free)
+            if new_free > self.get_nursery_top():
                 result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
@@ -825,13 +835,14 @@
         major collection, and finally reserve totalsize bytes.
         """
         minor_collection_count = 0
-        must_downgrade_gil = False
+        old_color = 0
+        self._gc_lock()
+
         while True:
-            self.tl.nursery_free = llmemory.NULL      # debug: don't use me
+            self.set_nursery_free(llmemory.NULL)      # debug: don't use me
             # note: no "raise MemoryError" between here and the next time
             # we initialize nursery_free!
 
-            self._gc_lock()
             if self.nursery_barriers.non_empty():
                 # Pinned object in front of nursery_top. Try reserving totalsize
                 # by jumping into the next, yet unused, area inside the
@@ -860,16 +871,17 @@
                 # four addresses which match the four "B".
                 #
                 # update used nursery space to allocate objects
-                self.tl.nursery_free = self.nursery_barriers.popleft()
-                self.tl.nursery_top = self.nursery_barriers.popleft()
-                self._gc_unlock()
-                prev_gil = False
+                self.set_nursery_free(self.nursery_barriers.popleft())
+                self.set_nursery_top(self.nursery_barriers.popleft())
+                if self.get_nursery_top() == llmemory.NULL:
+                    # needs to add this threadlocal to the nublocks list
+                    self.set_next_nublock(self.nublocks)
+                    self.nublocks = rthread.get_threadlocal_base()
             else:
-                self._gc_unlock()
-                if not llop.gil_is_exclusive(lltype.Bool):
-                    ll_assert(not must_downgrade_gil,
-                              "collect_and_reverse: bad gil state")
-                    must_downgrade_gil = llop.gil_wait(lltype.Bool)
+                if llop.get_gil_share_count(lltype.Signed) > 1:
+                    assert old_color == 0
+                    old_color = llop.get_gil_color(lltype.Signed)
+                    llop.set_gil_color(lltype.Void, self.gil_gc_color)
                     continue      # waited, maybe the situation changed
                 minor_collection_count += 1
                 if minor_collection_count == 1:
@@ -897,14 +909,14 @@
             # Tried to do something about nursery_free overflowing
             # nursery_top before this point. Try to reserve totalsize now.
             # If this succeeds break out of loop.
-            result = self.tl.nursery_free
-            if self.tl.nursery_free + totalsize <= self.tl.nursery_top:
-                self.tl.nursery_free = result + totalsize
-                ll_assert(self.tl.nursery_free <= self.tl.nursery_top, "nursery overflow")
+            result = self.get_nursery_free()
+            if result + totalsize <= self.get_nursery_top():
+                self.set_nursery_free(result + totalsize)
                 break
             #
-        if must_downgrade_gil:
-            llop.gil_downgrade(lltype.Void)
+        self._gc_unlock()
+        if old_color != 0:
+            llop.set_gil_color(lltype.Void, old_color)
         #
         if self.debug_tiny_nursery >= 0:   # for debugging
             if self.tl.nursery_top - self.tl.nursery_free > self.debug_tiny_nursery:
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -373,6 +373,10 @@
     def _freeze_(self):
         return True
 
+ at jit.loop_invariant
+def get_threadlocal_base():
+    return llop.threadlocalref_addr(llmemory.Address)
+
 
 class ThreadLocalReference(ThreadLocalField):
     # A thread-local that points to an object.  The object stored in such

From pypy.commits at gmail.com  Thu Jan 19 09:40:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 19 Jan 2017 06:40:27 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix,
 notably for Pillow's setup.py which calls find_library_file() to
Message-ID: <5880cfdb.952f1c0a.d5983.bc8d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89672:66ef3eb6a91e
Date: 2017-01-19 15:39 +0100
http://bitbucket.org/pypy/pypy/changeset/66ef3eb6a91e/

Log:	Fix, notably for Pillow's setup.py which calls find_library_file()
	to locate the system's lib files

diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -310,6 +310,10 @@
                 static = os.path.join(sysroot, dir[1:], static_f)
                 xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f)
 
+            # PyPy extension here: 'shared' usually ends in something
+            # like '.pypy-41.so'.  Try without the '.pypy-41' part too.
+            shared_no_pypy = re.sub(r'[.]pypy[^.]+([.][^.]+)$', r'\1', shared)
+
             # We're second-guessing the linker here, with not much hard
             # data to go on: GCC seems to prefer the shared library, so I'm
             # assuming that *all* Unix C compilers do.  And of course I'm
@@ -320,6 +324,8 @@
                 return xcode_stub
             elif os.path.exists(shared):
                 return shared
+            elif os.path.exists(shared_no_pypy):
+                return shared_no_pypy
             elif os.path.exists(static):
                 return static
 

From pypy.commits at gmail.com  Thu Jan 19 10:41:58 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 19 Jan 2017 07:41:58 -0800 (PST)
Subject: [pypy-commit] pypy default: fix commit a0105e0d00db,
 which was just wrong. In the upstream numpy,
 NUMPY_IMPORT_ARRAY_RETVAL is returned only in case of error,
 which never happens on pypy. As it was written,
 import_array() caused the immediate return of the *caller*
Message-ID: <5880de46.44641c0a.ad077.31c0@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r89673:cccff8a7d738
Date: 2017-01-19 15:41 +0000
http://bitbucket.org/pypy/pypy/changeset/cccff8a7d738/

Log:	fix commit a0105e0d00db, which was just wrong. In the upstream
	numpy, NUMPY_IMPORT_ARRAY_RETVAL is returned only in case of error,
	which never happens on pypy. As it was written, import_array()
	caused the immediate return of the *caller*

diff --git a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h
--- a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h
+++ b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h
@@ -11,6 +11,7 @@
 #define NUMPY_IMPORT_ARRAY_RETVAL
 #endif
 
-#define import_array() {return NUMPY_IMPORT_ARRAY_RETVAL;}
+/* on pypy import_array never fails, so it's just an empty macro */
+#define import_array()
 
 

From pypy.commits at gmail.com  Thu Jan 19 11:15:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 19 Jan 2017 08:15:35 -0800 (PST)
Subject: [pypy-commit] pypy default: document the thread-safely issue of
 _collections.py
Message-ID: <5880e627.4f831c0a.985af.d2ae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89674:0ef49555300c
Date: 2017-01-19 17:14 +0100
http://bitbucket.org/pypy/pypy/changeset/0ef49555300c/

Log:	document the thread-safely issue of _collections.py

diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -1,13 +1,18 @@
 """High performance data structures
+
+Note that PyPy also contains a built-in module '_collections' which will hide
+this one if compiled in.
+
+THIS ONE IS BOGUS in the sense that it is NOT THREAD-SAFE!  It is provided
+only as documentation nowadays.  Please don't run in production a PyPy
+without the '_collections' built-in module.  The built-in module is
+correctly thread-safe, like it is on CPython.
 """
 #
 # Copied and completed from the sandbox of CPython
 #   (nondist/sandbox/collections/pydeque.py rev 1.1, Raymond Hettinger)
 #
 
-# Note that PyPy also contains a built-in module '_collections' which will hide
-# this one if compiled in.
-
 try:
     from threading import _get_ident as _thread_ident
 except ImportError:

From pypy.commits at gmail.com  Thu Jan 19 19:32:11 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 19 Jan 2017 16:32:11 -0800 (PST)
Subject: [pypy-commit] pypy default: Typedef Py_ssize_t to long as it's
 (hopefully, maybe?) the right size on all supported platforms
Message-ID: <58815a8b.d5091c0a.30ed4.bc37@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89675:e83a89914c1b
Date: 2017-01-20 00:31 +0000
http://bitbucket.org/pypy/pypy/changeset/e83a89914c1b/

Log:	Typedef Py_ssize_t to long as it's (hopefully, maybe?) the right
	size on all supported platforms

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -7,14 +7,7 @@
 extern "C" {
 #endif
 
-/* Hack: MSVC doesn't support ssize_t */
-#ifdef _WIN32
-#define ssize_t long
-#endif
 #include <cpyext_object.h>
-#ifdef _WIN32
-#undef ssize_t
-#endif
 
 #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
 #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -1,5 +1,5 @@
 
-typedef ssize_t Py_ssize_t;
+typedef long Py_ssize_t;
 
 #define PyObject_HEAD  \
     Py_ssize_t ob_refcnt;        \
diff --git a/pypy/module/cpyext/test/test_borrow.py b/pypy/module/cpyext/test/test_borrow.py
--- a/pypy/module/cpyext/test/test_borrow.py
+++ b/pypy/module/cpyext/test/test_borrow.py
@@ -12,13 +12,13 @@
                 PyObject *t = PyTuple_New(1);
                 PyObject *f = PyFloat_FromDouble(42.0);
                 PyObject *g = NULL;
-                printf("Refcnt1: %zd\\n", f->ob_refcnt);
+                printf("Refcnt1: %ld\\n", f->ob_refcnt);
                 PyTuple_SetItem(t, 0, f); // steals reference
-                printf("Refcnt2: %zd\\n", f->ob_refcnt);
+                printf("Refcnt2: %ld\\n", f->ob_refcnt);
                 f = PyTuple_GetItem(t, 0); // borrows reference
-                printf("Refcnt3: %zd\\n", f->ob_refcnt);
+                printf("Refcnt3: %ld\\n", f->ob_refcnt);
                 g = PyTuple_GetItem(t, 0); // borrows reference again
-                printf("Refcnt4: %zd\\n", f->ob_refcnt);
+                printf("Refcnt4: %ld\\n", f->ob_refcnt);
                 printf("COMPARE: %i\\n", f == g);
                 fflush(stdout);
                 Py_DECREF(t);
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -646,7 +646,7 @@
             refcnt_after = true_obj->ob_refcnt;
             Py_DECREF(true_obj);
             Py_DECREF(true_obj);
-            fprintf(stderr, "REFCNT %zd %zd\\n", refcnt, refcnt_after);
+            fprintf(stderr, "REFCNT %ld %ld\\n", refcnt, refcnt_after);
             return PyBool_FromLong(refcnt_after == refcnt + 2);
         }
         static PyObject* foo_bar(PyObject* self, PyObject *args)
@@ -662,7 +662,7 @@
                 return NULL;
             refcnt_after = true_obj->ob_refcnt;
             Py_DECREF(tup);
-            fprintf(stderr, "REFCNT2 %zd %zd %zd\\n", refcnt, refcnt_after,
+            fprintf(stderr, "REFCNT2 %ld %ld %ld\\n", refcnt, refcnt_after,
                     true_obj->ob_refcnt);
             return PyBool_FromLong(refcnt_after == refcnt + 1 &&
                                    refcnt == true_obj->ob_refcnt);

From pypy.commits at gmail.com  Thu Jan 19 20:39:42 2017
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 19 Jan 2017 17:39:42 -0800 (PST)
Subject: [pypy-commit] pypy default: Add missing check for the extended
 short preamble builder
Message-ID: <58816a5e.8c7e1c0a.a8214.c871@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: 
Changeset: r89676:bd7b1902c4df
Date: 2017-01-19 19:59 -0500
http://bitbucket.org/pypy/pypy/changeset/bd7b1902c4df/

Log:	Add missing check for the extended short preamble builder

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -519,6 +519,8 @@
         self.jump_args.append(preamble_op.preamble_op)
 
     def use_box(self, box, preamble_op, optimizer=None):
+        if not self.build_inplace:
+            raise InvalidLoop("Forcing boxes would modify an existing short preamble")
         jump_op = self.short.pop()
         AbstractShortPreambleBuilder.use_box(self, box, preamble_op, optimizer)
         self.short.append(jump_op)

From pypy.commits at gmail.com  Thu Jan 19 20:39:44 2017
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 19 Jan 2017 17:39:44 -0800 (PST)
Subject: [pypy-commit] pypy default: Merge with default
Message-ID: <58816a60.8dae1c0a.32137.c9c7@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: 
Changeset: r89677:6a4e761daeb3
Date: 2017-01-19 20:38 -0500
http://bitbucket.org/pypy/pypy/changeset/6a4e761daeb3/

Log:	Merge with default

diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -1,13 +1,18 @@
 """High performance data structures
+
+Note that PyPy also contains a built-in module '_collections' which will hide
+this one if compiled in.
+
+THIS ONE IS BOGUS in the sense that it is NOT THREAD-SAFE!  It is provided
+only as documentation nowadays.  Please don't run in production a PyPy
+without the '_collections' built-in module.  The built-in module is
+correctly thread-safe, like it is on CPython.
 """
 #
 # Copied and completed from the sandbox of CPython
 #   (nondist/sandbox/collections/pydeque.py rev 1.1, Raymond Hettinger)
 #
 
-# Note that PyPy also contains a built-in module '_collections' which will hide
-# this one if compiled in.
-
 try:
     from threading import _get_ident as _thread_ident
 except ImportError:
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -7,14 +7,7 @@
 extern "C" {
 #endif
 
-/* Hack: MSVC doesn't support ssize_t */
-#ifdef _WIN32
-#define ssize_t long
-#endif
 #include <cpyext_object.h>
-#ifdef _WIN32
-#undef ssize_t
-#endif
 
 #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
 #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -1,5 +1,5 @@
 
-typedef ssize_t Py_ssize_t;
+typedef long Py_ssize_t;
 
 #define PyObject_HEAD  \
     Py_ssize_t ob_refcnt;        \
diff --git a/pypy/module/cpyext/test/test_borrow.py b/pypy/module/cpyext/test/test_borrow.py
--- a/pypy/module/cpyext/test/test_borrow.py
+++ b/pypy/module/cpyext/test/test_borrow.py
@@ -12,13 +12,13 @@
                 PyObject *t = PyTuple_New(1);
                 PyObject *f = PyFloat_FromDouble(42.0);
                 PyObject *g = NULL;
-                printf("Refcnt1: %zd\\n", f->ob_refcnt);
+                printf("Refcnt1: %ld\\n", f->ob_refcnt);
                 PyTuple_SetItem(t, 0, f); // steals reference
-                printf("Refcnt2: %zd\\n", f->ob_refcnt);
+                printf("Refcnt2: %ld\\n", f->ob_refcnt);
                 f = PyTuple_GetItem(t, 0); // borrows reference
-                printf("Refcnt3: %zd\\n", f->ob_refcnt);
+                printf("Refcnt3: %ld\\n", f->ob_refcnt);
                 g = PyTuple_GetItem(t, 0); // borrows reference again
-                printf("Refcnt4: %zd\\n", f->ob_refcnt);
+                printf("Refcnt4: %ld\\n", f->ob_refcnt);
                 printf("COMPARE: %i\\n", f == g);
                 fflush(stdout);
                 Py_DECREF(t);
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -646,7 +646,7 @@
             refcnt_after = true_obj->ob_refcnt;
             Py_DECREF(true_obj);
             Py_DECREF(true_obj);
-            fprintf(stderr, "REFCNT %zd %zd\\n", refcnt, refcnt_after);
+            fprintf(stderr, "REFCNT %ld %ld\\n", refcnt, refcnt_after);
             return PyBool_FromLong(refcnt_after == refcnt + 2);
         }
         static PyObject* foo_bar(PyObject* self, PyObject *args)
@@ -662,7 +662,7 @@
                 return NULL;
             refcnt_after = true_obj->ob_refcnt;
             Py_DECREF(tup);
-            fprintf(stderr, "REFCNT2 %zd %zd %zd\\n", refcnt, refcnt_after,
+            fprintf(stderr, "REFCNT2 %ld %ld %ld\\n", refcnt, refcnt_after,
                     true_obj->ob_refcnt);
             return PyBool_FromLong(refcnt_after == refcnt + 1 &&
                                    refcnt == true_obj->ob_refcnt);

From pypy.commits at gmail.com  Fri Jan 20 03:24:34 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 20 Jan 2017 00:24:34 -0800 (PST)
Subject: [pypy-commit] cffi default: Avoid cyclic imports by moving
 exceptions to a separate module
Message-ID: <5881c942.9d711c0a.3f964.80f6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r2862:b87441f6f36c
Date: 2017-01-20 03:05 +0000
http://bitbucket.org/cffi/cffi/changeset/b87441f6f36c/

Log:	Avoid cyclic imports by moving exceptions to a separate module

diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -1,8 +1,8 @@
 __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError',
            'FFIError']
 
-from .api import FFI, CDefError, FFIError
-from .ffiplatform import VerificationError, VerificationMissing
+from .api import FFI
+from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
 __version__ = "1.9.2"
 __version_info__ = (1, 9, 2)
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -1,5 +1,7 @@
 import sys, types
 from .lock import allocate_lock
+from .error import CDefError
+from . import cparser, model
 
 try:
     callable
@@ -15,17 +17,6 @@
     basestring = str
 
 
-class FFIError(Exception):
-    pass
-
-class CDefError(Exception):
-    def __str__(self):
-        try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
-        except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
-
 
 class FFI(object):
     r'''
@@ -49,7 +40,6 @@
         """Create an FFI instance.  The 'backend' argument is used to
         select a non-default backend, mostly for tests.
         """
-        from . import cparser, model
         if backend is None:
             # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with
             # _cffi_backend.so compiled.
@@ -221,7 +211,7 @@
 
     def offsetof(self, cdecl, *fields_or_indexes):
         """Return the offset of the named field inside the given
-        structure or array, which must be given as a C type name.  
+        structure or array, which must be given as a C type name.
         You can give several field names in case of nested structures.
         You can also give numeric values which correspond to array
         items, in case of an array type.
@@ -309,7 +299,7 @@
         return self._backend.string(cdata, maxlen)
 
     def unpack(self, cdata, length):
-        """Unpack an array of C data of the given length, 
+        """Unpack an array of C data of the given length,
         returning a Python string/unicode/list.
 
         If 'cdata' is a pointer to 'char', returns a byte string.
@@ -461,7 +451,6 @@
         return self._backend.getwinerror(code)
 
     def _pointer_to(self, ctype):
-        from . import model
         with self._lock:
             return model.pointer_cache(self, ctype)
 
@@ -773,7 +762,6 @@
         return backend.load_library(path, flags)
 
 def _make_ffi_library(ffi, libname, flags):
-    import os
     backend = ffi._backend
     backendlib = _load_backend_lib(backend, libname, flags)
     #
@@ -811,7 +799,6 @@
         if accessors_version[0] is ffi._cdef_version:
             return
         #
-        from . import model
         for key, (tp, _) in ffi._parser._declarations.items():
             if not isinstance(tp, model.EnumType):
                 tag, name = key.split(' ', 1)
diff --git a/cffi/cffi_opcode.py b/cffi/cffi_opcode.py
--- a/cffi/cffi_opcode.py
+++ b/cffi/cffi_opcode.py
@@ -1,3 +1,4 @@
+from .error import VerificationError
 
 class CffiOp(object):
     def __init__(self, op, arg):
@@ -19,7 +20,6 @@
                                     % (self.arg,))
             return format_four_bytes(value)
         if isinstance(self.arg, str):
-            from .ffiplatform import VerificationError
             raise VerificationError("cannot emit to Python: %r" % (self.arg,))
         return format_four_bytes((self.arg << 8) | self.op)
 
diff --git a/cffi/commontypes.py b/cffi/commontypes.py
--- a/cffi/commontypes.py
+++ b/cffi/commontypes.py
@@ -1,5 +1,6 @@
 import sys
-from . import api, model
+from . import model
+from .error import FFIError
 
 
 COMMON_TYPES = {}
@@ -31,11 +32,11 @@
         elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result, quals = model.PrimitiveType(cdecl), 0
         elif cdecl == 'set-unicode-needed':
-            raise api.FFIError("The Windows type %r is only available after "
-                               "you call ffi.set_unicode()" % (commontype,))
+            raise FFIError("The Windows type %r is only available after "
+                           "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == cdecl:
-                raise api.FFIError(
+                raise FFIError(
                     "Unsupported type: %r.  Please look at "
         "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
                     "and file an issue if you think this type should really "
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -1,5 +1,6 @@
-from . import api, model
+from . import model
 from .commontypes import COMMON_TYPES, resolve_common_type
+from .error import FFIError, CDefError
 try:
     from . import _pycparser as pycparser
 except ImportError:
@@ -113,7 +114,7 @@
             # grouping variant
             closing = csource.find('}', endpos)
             if closing < 0:
-                raise api.CDefError("'extern \"Python\" {': no '}' found")
+                raise CDefError("'extern \"Python\" {': no '}' found")
             if csource.find('{', endpos + 1, closing) >= 0:
                 raise NotImplementedError("cannot use { } inside a block "
                                           "'extern \"Python\" { ... }'")
@@ -123,7 +124,7 @@
             # non-grouping variant
             semicolon = csource.find(';', endpos)
             if semicolon < 0:
-                raise api.CDefError("'extern \"Python\": no ';' found")
+                raise CDefError("'extern \"Python\": no ';' found")
             parts.append(csource[endpos:semicolon+1])
             csource = csource[semicolon+1:]
         parts.append(' void __cffi_extern_python_stop;')
@@ -288,7 +289,7 @@
             msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
-        raise api.CDefError(msg)
+        raise CDefError(msg)
 
     def parse(self, csource, override=False, packed=False, dllexport=False):
         prev_options = self._options
@@ -318,8 +319,8 @@
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
                     if not decl.name:
-                        raise api.CDefError("typedef does not declare any name",
-                                            decl)
+                        raise CDefError("typedef does not declare any name",
+                                        decl)
                     quals = 0
                     if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
                             and decl.type.type.names[-1] == '__dotdotdot__'):
@@ -337,8 +338,8 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise api.CDefError("unrecognized construct", decl)
-        except api.FFIError as e:
+                    raise CDefError("unrecognized construct", decl)
+        except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
                 e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
@@ -348,7 +349,7 @@
         if key in self._int_constants:
             if self._int_constants[key] == val:
                 return     # ignore identical double declarations
-            raise api.FFIError(
+            raise FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
 
@@ -375,7 +376,7 @@
             elif value == '...':
                 self._declare('macro ' + key, value)
             else:
-                raise api.CDefError(
+                raise CDefError(
                     'only supports one of the following syntax:\n'
                     '  #define %s ...     (literally dot-dot-dot)\n'
                     '  #define %s NUMBER  (with NUMBER an integer'
@@ -410,8 +411,8 @@
             elif isinstance(node, pycparser.c_ast.Enum):
                 self._get_struct_union_enum_type('enum', node)
             elif not decl.name:
-                raise api.CDefError("construct does not declare any variable",
-                                    decl)
+                raise CDefError("construct does not declare any variable",
+                                decl)
             #
             if decl.name:
                 tp, quals = self._get_type_and_quals(node,
@@ -438,7 +439,7 @@
                     self._inside_extern_python = decl.name
                 else:
                     if self._inside_extern_python !='__cffi_extern_python_stop':
-                        raise api.CDefError(
+                        raise CDefError(
                             "cannot declare constants or "
                             "variables with 'extern \"Python\"'")
                     if (quals & model.Q_CONST) and not tp.is_array_type:
@@ -454,7 +455,7 @@
         assert not macros
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
-            raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
+            raise CDefError("unknown identifier '%s'" % (exprnode.name,))
         return self._get_type_and_quals(exprnode.type)
 
     def _declare(self, name, obj, included=False, quals=0):
@@ -463,7 +464,7 @@
             if prevobj is obj and prevquals == quals:
                 return
             if not self._options.get('override'):
-                raise api.FFIError(
+                raise FFIError(
                     "multiple declarations of %s (for interactive usage, "
                     "try cdef(xx, override=True))" % (name,))
         assert '__dotdotdot__' not in name.split()
@@ -551,7 +552,7 @@
                 if ident == 'void':
                     return model.void_type, quals
                 if ident == '__dotdotdot__':
-                    raise api.FFIError(':%d: bad usage of "..."' %
+                    raise FFIError(':%d: bad usage of "..."' %
                             typenode.coord.line)
                 tp0, quals0 = resolve_common_type(self, ident)
                 return tp0, (quals | quals0)
@@ -583,14 +584,14 @@
             return self._get_struct_union_enum_type('union', typenode, name,
                                                     nested=True), 0
         #
-        raise api.FFIError(":%d: bad or unsupported type declaration" %
+        raise FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
     def _parse_function_type(self, typenode, funcname=None):
         params = list(getattr(typenode.args, 'params', []))
         for i, arg in enumerate(params):
             if not hasattr(arg, 'type'):
-                raise api.CDefError("%s arg %d: unknown type '%s'"
+                raise CDefError("%s arg %d: unknown type '%s'"
                     " (if you meant to use the old C syntax of giving"
                     " untyped arguments, it is not supported)"
                     % (funcname or 'in expression', i + 1,
@@ -604,7 +605,7 @@
         if ellipsis:
             params.pop()
             if not params:
-                raise api.CDefError(
+                raise CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
         args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type))
@@ -705,7 +706,7 @@
             return tp
         #
         if tp.fldnames is not None:
-            raise api.CDefError("duplicate declaration of struct %s" % name)
+            raise CDefError("duplicate declaration of struct %s" % name)
         fldnames = []
         fldtypes = []
         fldbitsize = []
@@ -749,7 +750,7 @@
 
     def _make_partial(self, tp, nested):
         if not isinstance(tp, model.StructOrUnion):
-            raise api.CDefError("%s cannot be partial" % (tp,))
+            raise CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
         tp.partial = True
@@ -769,7 +770,7 @@
                     len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
                 return ord(s[-2])
             else:
-                raise api.CDefError("invalid constant %r" % (s,))
+                raise CDefError("invalid constant %r" % (s,))
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '+'):
@@ -788,12 +789,12 @@
             if partial_length_ok:
                 self._partial_length = True
                 return '...'
-            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
-                               "the actual array length in this context"
-                               % exprnode.coord.line)
+            raise FFIError(":%d: unsupported '[...]' here, cannot derive "
+                           "the actual array length in this context"
+                           % exprnode.coord.line)
         #
-        raise api.FFIError(":%d: unsupported expression: expected a "
-                           "simple numeric constant" % exprnode.coord.line)
+        raise FFIError(":%d: unsupported expression: expected a "
+                       "simple numeric constant" % exprnode.coord.line)
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
@@ -843,8 +844,8 @@
             for t in typenames[:-1]:
                 if t not in ['int', 'short', 'long', 'signed',
                              'unsigned', 'char']:
-                    raise api.FFIError(':%d: bad usage of "..."' %
-                                       decl.coord.line)
+                    raise FFIError(':%d: bad usage of "..."' %
+                                   decl.coord.line)
             result = model.UnknownIntegerType(decl.name)
 
         if self._uses_new_feature is None:
diff --git a/cffi/error.py b/cffi/error.py
new file mode 100644
--- /dev/null
+++ b/cffi/error.py
@@ -0,0 +1,20 @@
+
+class FFIError(Exception):
+    pass
+
+class CDefError(Exception):
+    def __str__(self):
+        try:
+            line = 'line %d: ' % (self.args[1].coord.line,)
+        except (AttributeError, TypeError, IndexError):
+            line = ''
+        return '%s%s' % (line, self.args[0])
+
+class VerificationError(Exception):
+    """ An error raised when verification fails
+    """
+
+class VerificationMissing(Exception):
+    """ An error raised when incomplete structures are passed into
+    cdef, but no verification has been done
+    """
diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py
--- a/cffi/ffiplatform.py
+++ b/cffi/ffiplatform.py
@@ -1,14 +1,5 @@
 import sys, os
-
-
-class VerificationError(Exception):
-    """ An error raised when verification fails
-    """
-
-class VerificationMissing(Exception):
-    """ An error raised when incomplete structures are passed into
-    cdef, but no verification has been done
-    """
+from .error import VerificationError, VerificationMissing
 
 
 LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -1,8 +1,8 @@
-import types, sys
+import types
 import weakref
 
 from .lock import allocate_lock
-
+from .error import CDefError, VerificationError, VerificationMissing
 
 # type qualifiers
 Q_CONST    = 0x01
@@ -39,7 +39,6 @@
         replace_with = qualify(quals, replace_with)
         result = result.replace('&', replace_with)
         if '$' in result:
-            from .ffiplatform import VerificationError
             raise VerificationError(
                 "cannot generate '%s' in %s: unknown type name"
                 % (self._get_c_name(), context))
@@ -223,9 +222,8 @@
     is_raw_function = True
 
     def build_backend_type(self, ffi, finishlist):
-        from . import api
-        raise api.CDefError("cannot render the type %r: it is a function "
-                            "type, not a pointer-to-function type" % (self,))
+        raise CDefError("cannot render the type %r: it is a function "
+                        "type, not a pointer-to-function type" % (self,))
 
     def as_function_pointer(self):
         return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi)
@@ -307,9 +305,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         if self.length == '...':
-            from . import api
-            raise api.CDefError("cannot render the type %r: unknown length" %
-                                (self,))
+            raise CDefError("cannot render the type %r: unknown length" %
+                            (self,))
         self.item.get_cached_btype(ffi, finishlist)   # force the item BType
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
@@ -455,13 +452,11 @@
         self.completed = 2
 
     def _verification_error(self, msg):
-        from .ffiplatform import VerificationError
         raise VerificationError(msg)
 
     def check_not_partial(self):
         if self.partial and self.fixedlayout is None:
-            from . import ffiplatform
-            raise ffiplatform.VerificationMissing(self._get_c_name())
+            raise VerificationMissing(self._get_c_name())
 
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
@@ -499,8 +494,7 @@
 
     def check_not_partial(self):
         if self.partial and not self.partial_resolved:
-            from . import ffiplatform
-            raise ffiplatform.VerificationMissing(self._get_c_name())
+            raise VerificationMissing(self._get_c_name())
 
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
@@ -514,7 +508,6 @@
         if self.baseinttype is not None:
             return self.baseinttype.get_cached_btype(ffi, finishlist)
         #
-        from . import api
         if self.enumvalues:
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
@@ -549,8 +542,8 @@
         if (smallest_value >= ((-1) << (8*size2-1)) and
             largest_value < (1 << (8*size2-sign))):
             return btype2
-        raise api.CDefError("%s values don't all fit into either 'long' "
-                            "or 'unsigned long'" % self._get_c_name())
+        raise CDefError("%s values don't all fit into either 'long' "
+                        "or 'unsigned long'" % self._get_c_name())
 
 def unknown_type(name, structname=None):
     if structname is None:

From pypy.commits at gmail.com  Fri Jan 20 03:32:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 20 Jan 2017 00:32:37 -0800 (PST)
Subject: [pypy-commit] cffi default: Move back "import cparser" to
 FFI.__init__, because that module only
Message-ID: <5881cb25.46831c0a.15ff0.8b2f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2863:bcd1faac093e
Date: 2017-01-20 09:32 +0100
http://bitbucket.org/cffi/cffi/changeset/bcd1faac093e/

Log:	Move back "import cparser" to FFI.__init__, because that module only
	needs to be imported if we instantiate FFI (admittedly, very often)

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -1,7 +1,7 @@
 import sys, types
 from .lock import allocate_lock
 from .error import CDefError
-from . import cparser, model
+from . import model
 
 try:
     callable
@@ -60,6 +60,7 @@
             # 'backend=backend_ctypes.CTypesBackend()', but don't
             # rely on it!  It's probably not going to work well.)
 
+        from . import cparser
         self._backend = backend
         self._lock = allocate_lock()
         self._parser = cparser.Parser()
diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py
--- a/cffi/ffiplatform.py
+++ b/cffi/ffiplatform.py
@@ -1,5 +1,5 @@
 import sys, os
-from .error import VerificationError, VerificationMissing
+from .error import VerificationError
 
 
 LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',

From pypy.commits at gmail.com  Fri Jan 20 03:41:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 20 Jan 2017 00:41:20 -0800 (PST)
Subject: [pypy-commit] cffi default: Follow-up on b87441f6f36c
Message-ID: <5881cd30.810b1c0a.681cb.9160@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2864:35e9e354b292
Date: 2017-01-20 09:41 +0100
http://bitbucket.org/cffi/cffi/changeset/35e9e354b292/

Log:	Follow-up on b87441f6f36c

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -1,5 +1,6 @@
 import os, sys, io
 from . import ffiplatform, model
+from .error import VerificationError
 from .cffi_opcode import *
 
 VERSION = "0x2601"
@@ -211,7 +212,7 @@
                 method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in recompile(): %r" % name)
             try:
                 self._current_quals = quals
@@ -354,12 +355,12 @@
                     included_module_name, included_source = (
                         ffi_to_include._assigned_source[:2])
                 except AttributeError:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "ffi object %r includes %r, but the latter has not "
                         "been prepared with set_source()" % (
                             self.ffi, ffi_to_include,))
                 if included_source is None:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "not implemented yet: ffi.include() of a Python-based "
                         "ffi inside a C-based ffi")
                 prnt('  "%s",' % (included_module_name,))
@@ -464,12 +465,12 @@
                 included_module_name, included_source = (
                     ffi_to_include._assigned_source[:2])
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "ffi object %r includes %r, but the latter has not "
                     "been prepared with set_source()" % (
                         self.ffi, ffi_to_include,))
             if included_source is not None:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented yet: ffi.include() of a C-based "
                     "ffi inside a Python-based ffi")
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
@@ -839,7 +840,7 @@
                 prnt('  { %s = &p->%s; (void)tmp; }' % (
                     ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                     fname))
-            except ffiplatform.VerificationError as e:
+            except VerificationError as e:
                 prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname))
@@ -1002,7 +1003,7 @@
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
                             check_value=None):
         if (category, name) in self._seen_constants:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "duplicate declaration of %s '%s'" % (category, name))
         self._seen_constants.add((category, name))
         #
@@ -1101,7 +1102,7 @@
     def _generate_cpy_macro_ctx(self, tp, name):
         if tp == '...':
             if self.target_is_python:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "cannot use the syntax '...' in '#define %s ...' when "
                     "using the ABI mode" % (name,))
             check_value = None
@@ -1234,7 +1235,7 @@
 
     def _generate_cpy_extern_python_ctx(self, tp, name):
         if self.target_is_python:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "cannot use 'extern \"Python\"' in the ABI mode")
         if tp.ellipsis:
             raise NotImplementedError("a vararg function is extern \"Python\"")
@@ -1315,7 +1316,7 @@
         if tp.length is None:
             self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index)
         elif tp.length == '...':
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "type %s badly placed: the '...' array length can only be "
                 "used on global arrays or on fields of structures" % (
                     str(tp).replace('/*...*/', '...'),))
diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -2,7 +2,8 @@
 # DEPRECATED: implementation for ffi.verify()
 #
 import sys, imp
-from . import model, ffiplatform
+from . import model
+from .error import VerificationError
 
 
 class VCPythonEngine(object):
@@ -155,7 +156,7 @@
                                           self.verifier.modulefilename)
             except ImportError as e:
                 error = "importing %r: %s" % (self.verifier.modulefilename, e)
-                raise ffiplatform.VerificationError(error)
+                raise VerificationError(error)
             finally:
                 if hasattr(sys, "setdlopenflags"):
                     sys.setdlopenflags(previous_flags)
@@ -185,7 +186,7 @@
             def __dir__(self):
                 return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
-        if module._cffi_setup(lst, ffiplatform.VerificationError, library):
+        if module._cffi_setup(lst, VerificationError, library):
             import warnings
             warnings.warn("reimporting %r might overwrite older definitions"
                           % (self.verifier.get_module_name()))
@@ -212,7 +213,7 @@
                 method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in verify(): %r" % name)
             try:
                 method(tp, realname)
@@ -485,7 +486,7 @@
                     prnt('  { %s = &p->%s; (void)tmp; }' % (
                         ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                         fname))
-                except ffiplatform.VerificationError as e:
+                except VerificationError as e:
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         prnt('static PyObject *')
@@ -550,7 +551,7 @@
             # check that the layout sizes and offsets match the real ones
             def check(realvalue, expectedvalue, msg):
                 if realvalue != expectedvalue:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "%s (we have %d, but C compiler says %d)"
                         % (msg, expectedvalue, realvalue))
             ffi = self.ffi
@@ -771,7 +772,7 @@
                 BItemType = self.ffi._get_cached_btype(tp.item)
                 length, rest = divmod(size, self.ffi.sizeof(BItemType))
                 if rest != 0:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "bad size: %r does not seem to be an array of %s" %
                         (name, tp.item))
                 tp = tp.resolve_length(length)
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -4,7 +4,8 @@
 import sys, os
 import types
 
-from . import model, ffiplatform
+from . import model
+from .error import VerificationError
 
 
 class VGenericEngine(object):
@@ -102,7 +103,7 @@
                 method = getattr(self, '_generate_gen_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in verify(): %r" % name)
             try:
                 method(tp, realname)
@@ -281,7 +282,7 @@
                     prnt('  { %s = &p->%s; (void)tmp; }' % (
                         ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                         fname))
-                except ffiplatform.VerificationError as e:
+                except VerificationError as e:
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         self.export_symbols.append(layoutfuncname)
@@ -344,7 +345,7 @@
             # check that the layout sizes and offsets match the real ones
             def check(realvalue, expectedvalue, msg):
                 if realvalue != expectedvalue:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "%s (we have %d, but C compiler says %d)"
                         % (msg, expectedvalue, realvalue))
             ffi = self.ffi
@@ -498,7 +499,7 @@
             error = self.ffi.string(p)
             if sys.version_info >= (3,):
                 error = str(error, 'utf-8')
-            raise ffiplatform.VerificationError(error)
+            raise VerificationError(error)
 
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
@@ -591,7 +592,7 @@
                 BItemType = self.ffi._get_cached_btype(tp.item)
                 length, rest = divmod(size, self.ffi.sizeof(BItemType))
                 if rest != 0:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "bad size: %r does not seem to be an array of %s" %
                         (name, tp.item))
                 tp = tp.resolve_length(length)
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -4,6 +4,7 @@
 import sys, os, binascii, shutil, io
 from . import __version_verifier_modules__
 from . import ffiplatform
+from .error import VerificationError
 
 if sys.version_info >= (3, 3):
     import importlib.machinery
@@ -42,7 +43,7 @@
                  ext_package=None, tag='', force_generic_engine=False,
                  source_extension='.c', flags=None, relative_to=None, **kwds):
         if ffi._parser._uses_new_feature:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "feature not supported with ffi.verify(), but only "
                 "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,))
         self.ffi = ffi
@@ -83,7 +84,7 @@
         which can be tweaked beforehand."""
         with self.ffi._lock:
             if self._has_source and file is None:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "source code already written")
             self._write_source(file)
 
@@ -92,7 +93,7 @@
         This produces a dynamic link library in 'self.modulefilename'."""
         with self.ffi._lock:
             if self._has_module:
-                raise ffiplatform.VerificationError("module already compiled")
+                raise VerificationError("module already compiled")
             if not self._has_source:
                 self._write_source()
             self._compile_module()

From pypy.commits at gmail.com  Fri Jan 20 07:33:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 20 Jan 2017 04:33:26 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58820396.ce181c0a.83309.0c6b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r848:4014b8c7aff4
Date: 2017-01-20 13:33 +0100
http://bitbucket.org/pypy/pypy.org/changeset/4014b8c7aff4/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66560 of $105000 (63.4%)
+   $66569 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2390 left
+    we have $2399 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Fri Jan 20 11:13:13 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 20 Jan 2017 08:13:13 -0800 (PST)
Subject: [pypy-commit] pypy default: Import all the cffi code the cpyext C
 parser depends on
Message-ID: <58823719.52a5df0a.b21bc.290f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89678:99e7353778eb
Date: 2017-01-20 16:11 +0000
http://bitbucket.org/pypy/pypy/changeset/99e7353778eb/

Log:	Import all the cffi code the cpyext C parser depends on

diff --git a/pypy/module/cpyext/cmodel.py b/pypy/module/cpyext/cmodel.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/cmodel.py
@@ -0,0 +1,598 @@
+import types
+import weakref
+
+from .error import CDefError, VerificationError, VerificationMissing
+
+# type qualifiers
+Q_CONST    = 0x01
+Q_RESTRICT = 0x02
+Q_VOLATILE = 0x04
+
+def qualify(quals, replace_with):
+    if quals & Q_CONST:
+        replace_with = ' const ' + replace_with.lstrip()
+    if quals & Q_VOLATILE:
+        replace_with = ' volatile ' + replace_with.lstrip()
+    if quals & Q_RESTRICT:
+        # It seems that __restrict is supported by gcc and msvc.
+        # If you hit some different compiler, add a #define in
+        # _cffi_include.h for it (and in its copies, documented there)
+        replace_with = ' __restrict ' + replace_with.lstrip()
+    return replace_with
+
+
+class BaseTypeByIdentity(object):
+    is_array_type = False
+    is_raw_function = False
+
+    def get_c_name(self, replace_with='', context='a C file', quals=0):
+        result = self.c_name_with_marker
+        assert result.count('&') == 1
+        # some logic duplication with ffi.getctype()... :-(
+        replace_with = replace_with.strip()
+        if replace_with:
+            if replace_with.startswith('*') and '&[' in result:
+                replace_with = '(%s)' % replace_with
+            elif not replace_with[0] in '[(':
+                replace_with = ' ' + replace_with
+        replace_with = qualify(quals, replace_with)
+        result = result.replace('&', replace_with)
+        if '$' in result:
+            raise VerificationError(
+                "cannot generate '%s' in %s: unknown type name"
+                % (self._get_c_name(), context))
+        return result
+
+    def _get_c_name(self):
+        return self.c_name_with_marker.replace('&', '')
+
+    def has_c_name(self):
+        return '$' not in self._get_c_name()
+
+    def is_integer_type(self):
+        return False
+
+    def get_cached_btype(self, ffi, finishlist, can_delay=False):
+        try:
+            BType = ffi._cached_btypes[self]
+        except KeyError:
+            BType = self.build_backend_type(ffi, finishlist)
+            BType2 = ffi._cached_btypes.setdefault(self, BType)
+            assert BType2 is BType
+        return BType
+
+    def __repr__(self):
+        return '<%s>' % (self._get_c_name(),)
+
+    def _get_items(self):
+        return [(name, getattr(self, name)) for name in self._attrs_]
+
+
+class BaseType(BaseTypeByIdentity):
+
+    def __eq__(self, other):
+        return (self.__class__ == other.__class__ and
+                self._get_items() == other._get_items())
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __hash__(self):
+        return hash((self.__class__, tuple(self._get_items())))
+
+
+class VoidType(BaseType):
+    _attrs_ = ()
+
+    def __init__(self):
+        self.c_name_with_marker = 'void&'
+
+    def build_backend_type(self, ffi, finishlist):
+        return global_cache(self, ffi, 'new_void_type')
+
+void_type = VoidType()
+
+
+class BasePrimitiveType(BaseType):
+    pass
+
+
+class PrimitiveType(BasePrimitiveType):
+    _attrs_ = ('name',)
+
+    ALL_PRIMITIVE_TYPES = {
+        'char':               'c',
+        'short':              'i',
+        'int':                'i',
+        'long':               'i',
+        'long long':          'i',
+        'signed char':        'i',
+        'unsigned char':      'i',
+        'unsigned short':     'i',
+        'unsigned int':       'i',
+        'unsigned long':      'i',
+        'unsigned long long': 'i',
+        'float':              'f',
+        'double':             'f',
+        'long double':        'f',
+        '_Bool':              'i',
+        # the following types are not primitive in the C sense
+        'wchar_t':            'c',
+        'int8_t':             'i',
+        'uint8_t':            'i',
+        'int16_t':            'i',
+        'uint16_t':           'i',
+        'int32_t':            'i',
+        'uint32_t':           'i',
+        'int64_t':            'i',
+        'uint64_t':           'i',
+        'int_least8_t':       'i',
+        'uint_least8_t':      'i',
+        'int_least16_t':      'i',
+        'uint_least16_t':     'i',
+        'int_least32_t':      'i',
+        'uint_least32_t':     'i',
+        'int_least64_t':      'i',
+        'uint_least64_t':     'i',
+        'int_fast8_t':        'i',
+        'uint_fast8_t':       'i',
+        'int_fast16_t':       'i',
+        'uint_fast16_t':      'i',
+        'int_fast32_t':       'i',
+        'uint_fast32_t':      'i',
+        'int_fast64_t':       'i',
+        'uint_fast64_t':      'i',
+        'intptr_t':           'i',
+        'uintptr_t':          'i',
+        'intmax_t':           'i',
+        'uintmax_t':          'i',
+        'ptrdiff_t':          'i',
+        'size_t':             'i',
+        'ssize_t':            'i',
+        }
+
+    def __init__(self, name):
+        assert name in self.ALL_PRIMITIVE_TYPES
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def is_char_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'c'
+    def is_integer_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
+    def is_float_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+
+    def build_backend_type(self, ffi, finishlist):
+        return global_cache(self, ffi, 'new_primitive_type', self.name)
+
+
+class UnknownIntegerType(BasePrimitiveType):
+    _attrs_ = ('name',)
+
+    def __init__(self, name):
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def is_integer_type(self):
+        return True
+
+    def build_backend_type(self, ffi, finishlist):
+        raise NotImplementedError("integer type '%s' can only be used after "
+                                  "compilation" % self.name)
+
+class UnknownFloatType(BasePrimitiveType):
+    _attrs_ = ('name', )
+
+    def __init__(self, name):
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def build_backend_type(self, ffi, finishlist):
+        raise NotImplementedError("float type '%s' can only be used after "
+                                  "compilation" % self.name)
+
+
+class BaseFunctionType(BaseType):
+    _attrs_ = ('args', 'result', 'ellipsis', 'abi')
+
+    def __init__(self, args, result, ellipsis, abi=None):
+        self.args = args
+        self.result = result
+        self.ellipsis = ellipsis
+        self.abi = abi
+        #
+        reprargs = [arg._get_c_name() for arg in self.args]
+        if self.ellipsis:
+            reprargs.append('...')
+        reprargs = reprargs or ['void']
+        replace_with = self._base_pattern % (', '.join(reprargs),)
+        if abi is not None:
+            replace_with = replace_with[:1] + abi + ' ' + replace_with[1:]
+        self.c_name_with_marker = (
+            self.result.c_name_with_marker.replace('&', replace_with))
+
+
+class RawFunctionType(BaseFunctionType):
+    # Corresponds to a C type like 'int(int)', which is the C type of
+    # a function, but not a pointer-to-function.  The backend has no
+    # notion of such a type; it's used temporarily by parsing.
+    _base_pattern = '(&)(%s)'
+    is_raw_function = True
+
+    def build_backend_type(self, ffi, finishlist):
+        raise CDefError("cannot render the type %r: it is a function "
+                        "type, not a pointer-to-function type" % (self,))
+
+    def as_function_pointer(self):
+        return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi)
+
+
+class FunctionPtrType(BaseFunctionType):
+    _base_pattern = '(*&)(%s)'
+
+    def build_backend_type(self, ffi, finishlist):
+        result = self.result.get_cached_btype(ffi, finishlist)
+        args = []
+        for tp in self.args:
+            args.append(tp.get_cached_btype(ffi, finishlist))
+        abi_args = ()
+        if self.abi == "__stdcall":
+            if not self.ellipsis:    # __stdcall ignored for variadic funcs
+                try:
+                    abi_args = (ffi._backend.FFI_STDCALL,)
+                except AttributeError:
+                    pass
+        return global_cache(self, ffi, 'new_function_type',
+                            tuple(args), result, self.ellipsis, *abi_args)
+
+    def as_raw_function(self):
+        return RawFunctionType(self.args, self.result, self.ellipsis, self.abi)
+
+
+class PointerType(BaseType):
+    _attrs_ = ('totype', 'quals')
+
+    def __init__(self, totype, quals=0):
+        self.totype = totype
+        self.quals = quals
+        extra = qualify(quals, " *&")
+        if totype.is_array_type:
+            extra = "(%s)" % (extra.lstrip(),)
+        self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra)
+
+    def build_backend_type(self, ffi, finishlist):
+        BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True)
+        return global_cache(self, ffi, 'new_pointer_type', BItem)
+
+voidp_type = PointerType(void_type)
+
+def ConstPointerType(totype):
+    return PointerType(totype, Q_CONST)
+
+const_voidp_type = ConstPointerType(void_type)
+
+
+class NamedPointerType(PointerType):
+    _attrs_ = ('totype', 'name')
+
+    def __init__(self, totype, name, quals=0):
+        PointerType.__init__(self, totype, quals)
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+
+class ArrayType(BaseType):
+    _attrs_ = ('item', 'length')
+    is_array_type = True
+
+    def __init__(self, item, length):
+        self.item = item
+        self.length = length
+        #
+        if length is None:
+            brackets = '&[]'
+        elif length == '...':
+            brackets = '&[/*...*/]'
+        else:
+            brackets = '&[%s]' % length
+        self.c_name_with_marker = (
+            self.item.c_name_with_marker.replace('&', brackets))
+
+    def resolve_length(self, newlength):
+        return ArrayType(self.item, newlength)
+
+    def build_backend_type(self, ffi, finishlist):
+        if self.length == '...':
+            raise CDefError("cannot render the type %r: unknown length" %
+                            (self,))
+        self.item.get_cached_btype(ffi, finishlist)   # force the item BType
+        BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
+        return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
+
+char_array_type = ArrayType(PrimitiveType('char'), None)
+
+
+class StructOrUnionOrEnum(BaseTypeByIdentity):
+    _attrs_ = ('name',)
+    forcename = None
+
+    def build_c_name_with_marker(self):
+        name = self.forcename or '%s %s' % (self.kind, self.name)
+        self.c_name_with_marker = name + '&'
+
+    def force_the_name(self, forcename):
+        self.forcename = forcename
+        self.build_c_name_with_marker()
+
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
+
+class StructOrUnion(StructOrUnionOrEnum):
+    fixedlayout = None
+    completed = 0
+    partial = False
+    packed = False
+
+    def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
+        self.name = name
+        self.fldnames = fldnames
+        self.fldtypes = fldtypes
+        self.fldbitsize = fldbitsize
+        self.fldquals = fldquals
+        self.build_c_name_with_marker()
+
+    def has_anonymous_struct_fields(self):
+        if self.fldtypes is None:
+            return False
+        for name, type in zip(self.fldnames, self.fldtypes):
+            if name == '' and isinstance(type, StructOrUnion):
+                return True
+        return False
+
+    def enumfields(self):
+        fldquals = self.fldquals
+        if fldquals is None:
+            fldquals = (0,) * len(self.fldnames)
+        for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes,
+                                              self.fldbitsize, fldquals):
+            if name == '' and isinstance(type, StructOrUnion):
+                # nested anonymous struct/union
+                for result in type.enumfields():
+                    yield result
+            else:
+                yield (name, type, bitsize, quals)
+
+    def force_flatten(self):
+        # force the struct or union to have a declaration that lists
+        # directly all fields returned by enumfields(), flattening
+        # nested anonymous structs/unions.
+        names = []
+        types = []
+        bitsizes = []
+        fldquals = []
+        for name, type, bitsize, quals in self.enumfields():
+            names.append(name)
+            types.append(type)
+            bitsizes.append(bitsize)
+            fldquals.append(quals)
+        self.fldnames = tuple(names)
+        self.fldtypes = tuple(types)
+        self.fldbitsize = tuple(bitsizes)
+        self.fldquals = tuple(fldquals)
+
+    def get_cached_btype(self, ffi, finishlist, can_delay=False):
+        BType = StructOrUnionOrEnum.get_cached_btype(self, ffi, finishlist,
+                                                     can_delay)
+        if not can_delay:
+            self.finish_backend_type(ffi, finishlist)
+        return BType
+
+    def finish_backend_type(self, ffi, finishlist):
+        if self.completed:
+            if self.completed != 2:
+                raise NotImplementedError("recursive structure declaration "
+                                          "for '%s'" % (self.name,))
+            return
+        BType = ffi._cached_btypes[self]
+        #
+        self.completed = 1
+        #
+        if self.fldtypes is None:
+            pass    # not completing it: it's an opaque struct
+            #
+        elif self.fixedlayout is None:
+            fldtypes = [tp.get_cached_btype(ffi, finishlist)
+                        for tp in self.fldtypes]
+            lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
+            sflags = 0
+            if self.packed:
+                sflags = 8    # SF_PACKED
+            ffi._backend.complete_struct_or_union(BType, lst, self,
+                                                  -1, -1, sflags)
+            #
+        else:
+            fldtypes = []
+            fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout
+            for i in range(len(self.fldnames)):
+                fsize = fieldsize[i]
+                ftype = self.fldtypes[i]
+                #
+                if isinstance(ftype, ArrayType) and ftype.length == '...':
+                    # fix the length to match the total size
+                    BItemType = ftype.item.get_cached_btype(ffi, finishlist)
+                    nlen, nrest = divmod(fsize, ffi.sizeof(BItemType))
+                    if nrest != 0:
+                        self._verification_error(
+                            "field '%s.%s' has a bogus size?" % (
+                            self.name, self.fldnames[i] or '{}'))
+                    ftype = ftype.resolve_length(nlen)
+                    self.fldtypes = (self.fldtypes[:i] + (ftype,) +
+                                     self.fldtypes[i+1:])
+                #
+                BFieldType = ftype.get_cached_btype(ffi, finishlist)
+                if isinstance(ftype, ArrayType) and ftype.length is None:
+                    assert fsize == 0
+                else:
+                    bitemsize = ffi.sizeof(BFieldType)
+                    if bitemsize != fsize:
+                        self._verification_error(
+                            "field '%s.%s' is declared as %d bytes, but is "
+                            "really %d bytes" % (self.name,
+                                                 self.fldnames[i] or '{}',
+                                                 bitemsize, fsize))
+                fldtypes.append(BFieldType)
+            #
+            lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs))
+            ffi._backend.complete_struct_or_union(BType, lst, self,
+                                                  totalsize, totalalignment)
+        self.completed = 2
+
+    def _verification_error(self, msg):
+        raise VerificationError(msg)
+
+    def check_not_partial(self):
+        if self.partial and self.fixedlayout is None:
+            raise VerificationMissing(self._get_c_name())
+
+    def build_backend_type(self, ffi, finishlist):
+        self.check_not_partial()
+        finishlist.append(self)
+        #
+        return global_cache(self, ffi, 'new_%s_type' % self.kind,
+                            self.get_official_name(), key=self)
+
+
+class StructType(StructOrUnion):
+    kind = 'struct'
+
+
+class UnionType(StructOrUnion):
+    kind = 'union'
+
+
+class EnumType(StructOrUnionOrEnum):
+    kind = 'enum'
+    partial = False
+    partial_resolved = False
+
+    def __init__(self, name, enumerators, enumvalues, baseinttype=None):
+        self.name = name
+        self.enumerators = enumerators
+        self.enumvalues = enumvalues
+        self.baseinttype = baseinttype
+        self.build_c_name_with_marker()
+
+    def force_the_name(self, forcename):
+        StructOrUnionOrEnum.force_the_name(self, forcename)
+        if self.forcename is None:
+            name = self.get_official_name()
+            self.forcename = '$' + name.replace(' ', '_')
+
+    def check_not_partial(self):
+        if self.partial and not self.partial_resolved:
+            raise VerificationMissing(self._get_c_name())
+
+    def build_backend_type(self, ffi, finishlist):
+        self.check_not_partial()
+        base_btype = self.build_baseinttype(ffi, finishlist)
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
+                            self.enumerators, self.enumvalues,
+                            base_btype, key=self)
+
+    def build_baseinttype(self, ffi, finishlist):
+        if self.baseinttype is not None:
+            return self.baseinttype.get_cached_btype(ffi, finishlist)
+        #
+        if self.enumvalues:
+            smallest_value = min(self.enumvalues)
+            largest_value = max(self.enumvalues)
+        else:
+            import warnings
+            try:
+                # XXX!  The goal is to ensure that the warnings.warn()
+                # will not suppress the warning.  We want to get it
+                # several times if we reach this point several times.
+                __warningregistry__.clear()
+            except NameError:
+                pass
+            warnings.warn("%r has no values explicitly defined; "
+                          "guessing that it is equivalent to 'unsigned int'"
+                          % self._get_c_name())
+            smallest_value = largest_value = 0
+        if smallest_value < 0:   # needs a signed type
+            sign = 1
+            candidate1 = PrimitiveType("int")
+            candidate2 = PrimitiveType("long")
+        else:
+            sign = 0
+            candidate1 = PrimitiveType("unsigned int")
+            candidate2 = PrimitiveType("unsigned long")
+        btype1 = candidate1.get_cached_btype(ffi, finishlist)
+        btype2 = candidate2.get_cached_btype(ffi, finishlist)
+        size1 = ffi.sizeof(btype1)
+        size2 = ffi.sizeof(btype2)
+        if (smallest_value >= ((-1) << (8*size1-1)) and
+            largest_value < (1 << (8*size1-sign))):
+            return btype1
+        if (smallest_value >= ((-1) << (8*size2-1)) and
+            largest_value < (1 << (8*size2-sign))):
+            return btype2
+        raise CDefError("%s values don't all fit into either 'long' "
+                        "or 'unsigned long'" % self._get_c_name())
+
+def unknown_type(name, structname=None):
+    if structname is None:
+        structname = '$%s' % name
+    tp = StructType(structname, None, None, None)
+    tp.force_the_name(name)
+    tp.origin = "unknown_type"
+    return tp
+
+def unknown_ptr_type(name, structname=None):
+    if structname is None:
+        structname = '$$%s' % name
+    tp = StructType(structname, None, None, None)
+    return NamedPointerType(tp, name)
+
+
+def global_cache(srctype, ffi, funcname, *args, **kwds):
+    key = kwds.pop('key', (funcname, args))
+    assert not kwds
+    try:
+        return ffi._backend.__typecache[key]
+    except KeyError:
+        pass
+    except AttributeError:
+        # initialize the __typecache attribute, either at the module level
+        # if ffi._backend is a module, or at the class level if ffi._backend
+        # is some instance.
+        if isinstance(ffi._backend, types.ModuleType):
+            ffi._backend.__typecache = weakref.WeakValueDictionary()
+        else:
+            type(ffi._backend).__typecache = weakref.WeakValueDictionary()
+    try:
+        res = getattr(ffi._backend, funcname)(*args)
+    except NotImplementedError as e:
+        raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e))
+    # note that setdefault() on WeakValueDictionary is not atomic
+    # and contains a rare bug (http://bugs.python.org/issue19542);
+    # we have to use a lock and do it ourselves
+    cache = ffi._backend.__typecache
+    with global_lock:
+        res1 = cache.get(key)
+        if res1 is None:
+            cache[key] = res
+            return res
+        else:
+            return res1
+
+def pointer_cache(ffi, BType):
+    return global_cache('?', ffi, 'new_pointer_type', BType)
+
+def attach_exception_info(e, name):
+    if e.args and type(e.args[0]) is str:
+        e.args = ('%s: %s' % (name, e.args[0]),) + e.args[1:]
diff --git a/pypy/module/cpyext/commontypes.py b/pypy/module/cpyext/commontypes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/commontypes.py
@@ -0,0 +1,80 @@
+import sys
+from . import cmodel as model
+from .error import FFIError
+
+
+COMMON_TYPES = {}
+
+try:
+    # fetch "bool" and all simple Windows types
+    from _cffi_backend import _get_common_types
+    _get_common_types(COMMON_TYPES)
+except ImportError:
+    pass
+
+COMMON_TYPES['FILE'] = model.unknown_type('FILE', '_IO_FILE')
+COMMON_TYPES['bool'] = '_Bool'    # in case we got ImportError above
+
+for _type in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
+    if _type.endswith('_t'):
+        COMMON_TYPES[_type] = _type
+del _type
+
+_CACHE = {}
+
+def resolve_common_type(parser, commontype):
+    try:
+        return _CACHE[commontype]
+    except KeyError:
+        cdecl = COMMON_TYPES.get(commontype, commontype)
+        if not isinstance(cdecl, str):
+            result, quals = cdecl, 0    # cdecl is already a BaseType
+        elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
+            result, quals = model.PrimitiveType(cdecl), 0
+        elif cdecl == 'set-unicode-needed':
+            raise FFIError("The Windows type %r is only available after "
+                           "you call ffi.set_unicode()" % (commontype,))
+        else:
+            if commontype == cdecl:
+                raise FFIError(
+                    "Unsupported type: %r.  Please look at "
+        "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
+                    "and file an issue if you think this type should really "
+                    "be supported." % (commontype,))
+            result, quals = parser.parse_type_and_quals(cdecl)   # recursive
+
+        assert isinstance(result, model.BaseTypeByIdentity)
+        _CACHE[commontype] = result, quals
+        return result, quals
+
+
+# ____________________________________________________________
+# extra types for Windows (most of them are in commontypes.c)
+
+
+def win_common_types():
+    return {
+        "UNICODE_STRING": model.StructType(
+            "_UNICODE_STRING",
+            ["Length",
+             "MaximumLength",
+             "Buffer"],
+            [model.PrimitiveType("unsigned short"),
+             model.PrimitiveType("unsigned short"),
+             model.PointerType(model.PrimitiveType("wchar_t"))],
+            [-1, -1, -1]),
+        "PUNICODE_STRING": "UNICODE_STRING *",
+        "PCUNICODE_STRING": "const UNICODE_STRING *",
+
+        "TBYTE": "set-unicode-needed",
+        "TCHAR": "set-unicode-needed",
+        "LPCTSTR": "set-unicode-needed",
+        "PCTSTR": "set-unicode-needed",
+        "LPTSTR": "set-unicode-needed",
+        "PTSTR": "set-unicode-needed",
+        "PTBYTE": "set-unicode-needed",
+        "PTCHAR": "set-unicode-needed",
+        }
+
+if sys.platform == 'win32':
+    COMMON_TYPES.update(win_common_types())
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -1,12 +1,12 @@
-import sys
 from collections import OrderedDict
-from cffi import api, model
-from cffi.commontypes import COMMON_TYPES, resolve_common_type
+from . import cmodel as model
+from .commontypes import COMMON_TYPES, resolve_common_type
+from .error import FFIError, CDefError
 try:
     from cffi import _pycparser as pycparser
 except ImportError:
     import pycparser
-import weakref, re
+import weakref, re, sys
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rfile import FILEP
 from rpython.rtyper.lltypesystem import rffi, lltype
@@ -161,7 +161,7 @@
             msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
-        raise api.CDefError(msg)
+        raise CDefError(msg)
 
     def parse(self, csource, override=False, packed=False, dllexport=False):
         prev_options = self._options
@@ -190,8 +190,8 @@
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
                     if not decl.name:
-                        raise api.CDefError("typedef does not declare any name",
-                                            decl)
+                        raise CDefError("typedef does not declare any name",
+                                        decl)
                     quals = 0
                     realtype, quals = self._get_type_and_quals(
                         decl.type, name=decl.name, partial_length_ok=True)
@@ -199,8 +199,8 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise api.CDefError("unrecognized construct", decl)
-        except api.FFIError as e:
+                    raise CDefError("unrecognized construct", decl)
+        except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
                 e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
@@ -210,7 +210,7 @@
         if key in self._int_constants:
             if self._int_constants[key] == val:
                 return     # ignore identical double declarations
-            raise api.FFIError(
+            raise FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
 
@@ -259,8 +259,8 @@
             elif isinstance(node, pycparser.c_ast.Enum):
                 self._get_struct_union_enum_type('enum', node)
             elif not decl.name:
-                raise api.CDefError("construct does not declare any variable",
-                                    decl)
+                raise CDefError("construct does not declare any variable",
+                                decl)
             #
             if decl.name:
                 tp, quals = self._get_type_and_quals(node,
@@ -292,7 +292,7 @@
         ast, _, _ = self._parse('void __dummy(\n%s\n);' % cdecl)
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
-            raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
+            raise CDefError("unknown identifier '%s'" % (exprnode.name,))
         return self._get_type_and_quals(exprnode.type)
 
     def _declare(self, name, obj, included=False, quals=0):
@@ -413,14 +413,14 @@
             return self._get_struct_union_enum_type('union', typenode, name,
                                                     nested=True), 0
         #
-        raise api.FFIError(":%d: bad or unsupported type declaration" %
+        raise FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
     def _parse_function_type(self, typenode, funcname=None):
         params = list(getattr(typenode.args, 'params', []))
         for i, arg in enumerate(params):
             if not hasattr(arg, 'type'):
-                raise api.CDefError("%s arg %d: unknown type '%s'"
+                raise CDefError("%s arg %d: unknown type '%s'"
                     " (if you meant to use the old C syntax of giving"
                     " untyped arguments, it is not supported)"
                     % (funcname or 'in expression', i + 1,
@@ -434,7 +434,7 @@
         if ellipsis:
             params.pop()
             if not params:
-                raise api.CDefError(
+                raise CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
         args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type))
@@ -533,7 +533,7 @@
             return tp
         #
         if tp.fldnames is not None:
-            raise api.CDefError("duplicate declaration of struct %s" % name)
+            raise CDefError("duplicate declaration of struct %s" % name)
         fldnames = []
         fldtypes = []
         fldbitsize = []
@@ -570,7 +570,7 @@
 
     def _make_partial(self, tp, nested):
         if not isinstance(tp, model.StructOrUnion):
-            raise api.CDefError("%s cannot be partial" % (tp,))
+            raise CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
         tp.partial = True
@@ -590,7 +590,7 @@
                     len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
                 return ord(s[-2])
             else:
-                raise api.CDefError("invalid constant %r" % (s,))
+                raise CDefError("invalid constant %r" % (s,))
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '+'):
@@ -609,12 +609,12 @@
             if partial_length_ok:
                 self._partial_length = True
                 return '...'
-            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
-                               "the actual array length in this context"
-                               % exprnode.coord.line)
+            raise FFIError(":%d: unsupported '[...]' here, cannot derive "
+                           "the actual array length in this context"
+                           % exprnode.coord.line)
         #
-        raise api.FFIError(":%d: unsupported expression: expected a "
-                           "simple numeric constant" % exprnode.coord.line)
+        raise FFIError(":%d: unsupported expression: expected a "
+                       "simple numeric constant" % exprnode.coord.line)
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
diff --git a/pypy/module/cpyext/error.py b/pypy/module/cpyext/error.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/error.py
@@ -0,0 +1,20 @@
+
+class FFIError(Exception):
+    pass
+
+class CDefError(Exception):
+    def __str__(self):
+        try:
+            line = 'line %d: ' % (self.args[1].coord.line,)
+        except (AttributeError, TypeError, IndexError):
+            line = ''
+        return '%s%s' % (line, self.args[0])
+
+class VerificationError(Exception):
+    """ An error raised when verification fails
+    """
+
+class VerificationMissing(Exception):
+    """ An error raised when incomplete structures are passed into
+    cdef, but no verification has been done
+    """

From pypy.commits at gmail.com  Fri Jan 20 12:03:16 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 20 Jan 2017 09:03:16 -0800 (PST)
Subject: [pypy-commit] pypy default: Refactor typedef'd identifier handling
 in _get_type_and_quals(): move it together with the other identifiers
Message-ID: <588242d4.ec98df0a.ec649.3fb7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89679:5444bd569e8a
Date: 2017-01-20 17:02 +0000
http://bitbucket.org/pypy/pypy/changeset/5444bd569e8a/

Log:	Refactor typedef'd identifier handling in _get_type_and_quals():
	move it together with the other identifiers

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -326,15 +326,6 @@
         return model.PointerType(type, quals)
 
     def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False):
-        # first, dereference typedefs, if we have it already parsed, we're good
-        if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
-            isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
-            len(typenode.type.names) == 1 and
-            ('typedef ' + typenode.type.names[0]) in self._declarations):
-            tp, quals = self._declarations['typedef ' + typenode.type.names[0]]
-            quals |= self._extract_quals(typenode)
-            return tp, quals
-        #
         if isinstance(typenode, pycparser.c_ast.ArrayDecl):
             # array type
             if typenode.dim is None:
@@ -356,6 +347,13 @@
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
             quals = self._extract_quals(typenode)
             type = typenode.type
+            # first, dereference typedefs, if we have it already parsed, we're good
+            if (isinstance(type, pycparser.c_ast.IdentifierType) and
+                len(type.names) == 1 and
+                ('typedef ' + type.names[0]) in self._declarations):
+                tp, base_quals = self._declarations['typedef ' + type.names[0]]
+                quals |= base_quals
+                return tp, quals
             if isinstance(type, pycparser.c_ast.IdentifierType):
                 # assume a primitive type.  get it from .names, but reduce
                 # synonyms to a single chosen combination

From pypy.commits at gmail.com  Fri Jan 20 14:20:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 20 Jan 2017 11:20:51 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe: Add discussion
Message-ID: <58826313.43e61c0a.679ae.bc45@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe
Changeset: r89680:6b25ff8de778
Date: 2017-01-20 20:20 +0100
http://bitbucket.org/pypy/pypy/changeset/6b25ff8de778/

Log:	Add discussion

diff --git a/pypy/doc/discussion/nogil-unsafe.rst b/pypy/doc/discussion/nogil-unsafe.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/discussion/nogil-unsafe.rst
@@ -0,0 +1,136 @@
+===========================================================
+Explicitly removing the GIL (and letting the program crash)
+===========================================================
+
+
+Idea
+----
+
+Python programs normally run with the GIL, meaning only one thread at a
+time can be running Python code.  Of course, threads are still useful in
+Python for various reasons: notably, while a thread is waiting in any
+system call, it releases the GIL, allowing another thread to run Python
+code.
+
+If we allowed unrestricted removal of the GIL, the Python interpreter
+would either crash, or we would need a lot of extra work: either adding
+careful locking everywhere like Jython/IronPython, or use STM.  Both
+options impose an additional runtime overhead everywhere.
+
+The nogil-unsafe branch explores a different approach: the Python
+interpreter is allowed to crash, but the user (i.e. the Python
+programmer) needs to identify parts of the program that are safe to run
+on multiple threads.  The idea so far is that it makes sense to say
+"this code here can run on multiple threads because I carefully checked
+that it is safe".  Such a safety does not automatically extend to other
+parts of the program: it is by default *never* safe to run two threads
+in parallel, *unless* the user says that this particular piece of code
+code can really be run by multiple threads in parallel.  Even if the
+program contains, in two unrelated parts, two pieces of code that are
+each parallelizable, it does not automatically mean that we can run a
+thread in one piece of code in parallel with a thread in the other.
+
+So the model is that every thread runs under a configurable "color", and
+threads can only run together if they are of the same color.  By
+default, all threads have no color at all, and cannot run in parallel
+with any other thread at all (colored or not).  On the other hand,
+several colored threads can run in parallel if the color is the same.
+The idea is that the user identifies a piece of code that can be
+parallelized, creates a color for it, and uses a "with" statement to
+change the color of threads entering and leaving that piece of code.
+
+What can be done in such parallelizable pieces of code?  "Simple enough"
+operations---the details of which should be researched and well
+documented.  Moreover, we can tweak a few parts of the standard library
+to support this general approach, too: for example, when pdb is called,
+it would switch the thread back to colorless (or maybe the Python
+interpreter should do so whenever it calls tracing hooks, or invokves a
+signal handler).
+
+
+Implementation
+--------------
+
+Here is in more details how the GIL works in PyPy.  There are parts that
+are heuristically determined, but it seems to work quite well enough
+nowadays:
+
+- A thread must acquire the GIL to run Python code.
+
+- When a thread does a system call, it releases the GIL in such a way
+  that if the system call completes quickly, it will re-acquire the GIL
+  immediately afterwards, with high probability.  If the system call
+  does not complete quickly, another thread will acquire the GIL and
+  continue.
+
+- When a thread exhausts its "time slice", measured in number of
+  bytecodes run, then it releases the GIL in a different way that
+  explicitly puts this thread at the back of the queue.
+
+The GIL release and re-acquire operations around a system call work like
+this.  First, we pick another thread, more precisely the one that is
+scheduled to run after this one, and call it the "stealer thread".  In
+the stealer thread, instead of purely sleeping, we regularly "poll" the
+value of a boolean global variable.  When the running thread releases
+the GIL it just writes a 0 in this global variable; when it re-acquires
+the GIL it tries to replace it with a 1 again.  So if the running thread
+does only short-running system calls, the global variable will mostly be
+seen as 1 from the stealer thread.  However, if the stealer thread sees
+it as 0, it tries to replace it with 1, and if that succeeds, then the
+GIL was really stolen.
+
+The plan for the nogil-unsafe branch is to keep this logic, but extend
+it for the case where the running thread is colored.  To simplify, let's
+say for now that colored threads don't participate in the "time slice"
+system.  The idea is to replace the boolean global variable with a full
+integer variable, divided in two halves: the higher 48 bits contain the
+color of the running thread (colorless threads all use a different
+unique value); the lower 16 bits contain the number of threads of that
+color currently running.
+
+Logic to release the GIL before a system call:
+
+- in the default branch: ``global_variable = 0;``
+
+- in the nogil-unsafe branch: ``atomically_decrement(integer_variable);``
+
+Logic to acquire the GIL after a system call:
+
+- in the default branch::
+
+     if (xchg(global_variable, 1) != 0)
+         slow_path();
+
+- in the nogil-unsafe branch::
+
+     n = integer_variable + 1;
+     if ((n >> 16) != thread_color || !compare_and_swap(integer_variable, n))
+         slow_path();
+
+This change should come with a minimal performance impact (I guess).
+
+Logic of the stealer thread:
+
+- in the default branch::
+
+     while (xchg(global_variable, 1) != 0) {
+         sleep for 0.1 ms
+     }
+
+- in the nogil-unsafe branch::
+
+     while ((old_value & 0xffff) != 0 ||
+            !compare_and_swap(integer_variable, old_value, (new_color<<16) + 1))
+         sleep for 0.1 ms
+     }
+
+Similar too.  We need however to add logic such that when several
+threads with the same color enter the waiting state, only the first one
+moves on to the queue (where it can become the stealer).  All the other
+ones should instead be suspended, e.g. on an OS "condition variable".
+When the stealing succeeds, it sends a signal on this condition variable
+to wake them all up.  With a simple tweak we could count in advance how
+many such threads there are, and immediately set the integer variable to
+``(new_color << 16) + count``; this avoids the thundering herd effect
+caused by all the threads woken up at the same time, each trying to
+increase the integer variable by one.

From pypy.commits at gmail.com  Fri Jan 20 14:38:06 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 20 Jan 2017 11:38:06 -0800 (PST)
Subject: [pypy-commit] pypy default: Move code one more level down
Message-ID: <5882671e.01af1c0a.67eec.a428@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89681:8c8c83bfd69b
Date: 2017-01-20 19:37 +0000
http://bitbucket.org/pypy/pypy/changeset/8c8c83bfd69b/

Log:	Move code one more level down

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -347,14 +347,12 @@
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
             quals = self._extract_quals(typenode)
             type = typenode.type
-            # first, dereference typedefs, if we have it already parsed, we're good
-            if (isinstance(type, pycparser.c_ast.IdentifierType) and
-                len(type.names) == 1 and
-                ('typedef ' + type.names[0]) in self._declarations):
-                tp, base_quals = self._declarations['typedef ' + type.names[0]]
-                quals |= base_quals
-                return tp, quals
             if isinstance(type, pycparser.c_ast.IdentifierType):
+                # first, dereference typedefs, if we have it already parsed, we're good
+                if (len(type.names) == 1 and
+                    ('typedef ' + type.names[0]) in self._declarations):
+                    tp0, quals0 = self._declarations['typedef ' + type.names[0]]
+                    return tp0, (quals | quals0)
                 # assume a primitive type.  get it from .names, but reduce
                 # synonyms to a single chosen combination
                 names = list(type.names)

From pypy.commits at gmail.com  Sat Jan 21 15:07:28 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 21 Jan 2017 12:07:28 -0800 (PST)
Subject: [pypy-commit] pypy default: Keep typedef information around a bit
 longer
Message-ID: <5883bf80.810b1c0a.681cb.587c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89682:96efa16ddc41
Date: 2017-01-21 19:54 +0000
http://bitbucket.org/pypy/pypy/changeset/96efa16ddc41/

Log:	Keep typedef information around a bit longer

diff --git a/pypy/module/cpyext/cmodel.py b/pypy/module/cpyext/cmodel.py
--- a/pypy/module/cpyext/cmodel.py
+++ b/pypy/module/cpyext/cmodel.py
@@ -192,6 +192,15 @@
         raise NotImplementedError("float type '%s' can only be used after "
                                   "compilation" % self.name)
 
+class DefinedType(BaseType):
+    _attrs_ = ('name', )
+
+    def __init__(self, name, realtype, quals):
+        self.name = name
+        self.realtype = realtype
+        self.quals = quals
+        self.c_name_with_marker = name + '&'
+
 
 class BaseFunctionType(BaseType):
     _attrs_ = ('args', 'result', 'ellipsis', 'abi')
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -189,13 +189,7 @@
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
-                    if not decl.name:
-                        raise CDefError("typedef does not declare any name",
-                                        decl)
-                    quals = 0
-                    realtype, quals = self._get_type_and_quals(
-                        decl.type, name=decl.name, partial_length_ok=True)
-                    self._declare('typedef ' + decl.name, realtype, quals=quals)
+                    self._parse_typedef(decl)
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
@@ -245,6 +239,14 @@
             tag = 'function '
         self._declare(tag + decl.name, tp)
 
+    def _parse_typedef(self, decl):
+        if not decl.name:
+            raise CDefError("typedef does not declare any name", decl)
+        realtype, quals = self._get_type_and_quals(
+            decl.type, name=decl.name, partial_length_ok=True)
+        tp = model.DefinedType(decl.name, realtype, quals)
+        self._declare('typedef ' + decl.name, tp)
+
     def _parse_decl(self, decl):
         node = decl.type
         if isinstance(node, pycparser.c_ast.FuncDecl):
@@ -802,6 +804,8 @@
                 del self._TYPES[name]
 
     def convert_type(self, obj, quals=0):
+        if isinstance(obj, model.DefinedType):
+            return self.convert_type(obj.realtype, obj.quals)
         if isinstance(obj, model.PrimitiveType):
             return cname_to_lltype(obj.name)
         elif isinstance(obj, model.StructType):

From pypy.commits at gmail.com  Sat Jan 21 15:07:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 21 Jan 2017 12:07:30 -0800 (PST)
Subject: [pypy-commit] pypy default: Make sure that the header declarations
 use the same types as were passed to @api_decl
Message-ID: <5883bf82.85b2df0a.c97ab.09eb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89683:7075c0cf6f95
Date: 2017-01-21 20:06 +0000
http://bitbucket.org/pypy/pypy/changeset/7075c0cf6f95/

Log:	Make sure that the header declarations use the same types as were
	passed to @api_decl

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -379,10 +379,14 @@
 
     def get_c_restype(self, c_writer):
         if self.cdecl:
-            return self.cdecl.split(self.c_name)[0].strip()
+            return self.cdecl.tp.result.get_c_name()
         return c_writer.gettype(self.restype).replace('@', '').strip()
 
     def get_c_args(self, c_writer):
+        if self.cdecl:
+            args = [tp.get_c_name('arg%d' % i) for i, tp in
+                enumerate(self.cdecl.tp.args)]
+            return ', '.join(args) or "void"
         args = []
         for i, argtype in enumerate(self.argtypes):
             if argtype is CONST_STRING:
@@ -479,14 +483,15 @@
         return unwrapper
     return decorate
 
-def api_decl(cdecl, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+def api_decl(cdef, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
     def decorate(func):
         func._always_inline_ = 'try'
-        name, FUNC = cts.parse_func(cdecl)
+        cdecl = cts.parse_func(cdef)
+        RESULT = cdecl.get_llresult(cts)
         api_function = ApiFunction(
-            FUNC.ARGS, FUNC.RESULT, func,
-            error=_compute_error(error, FUNC.RESULT), cdecl=cdecl)
-        FUNCTIONS_BY_HEADER[header][name] = api_function
+            cdecl.get_llargs(cts), RESULT, func,
+            error=_compute_error(error, RESULT), cdecl=cdecl)
+        FUNCTIONS_BY_HEADER[header][cdecl.name] = api_function
         unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -856,8 +856,7 @@
         ast, _, _ = self.ctx._parse(cdecl)
         decl = ast.ext[-1]
         tp, quals = self.ctx._get_type_and_quals(decl.type, name=decl.name)
-        FUNCP = self.convert_type(tp.as_function_pointer())
-        return decl.name, FUNCP.TO
+        return FunctionDeclaration(decl.name, tp)
 
     def _freeze_(self):
         if self._frozen:
@@ -881,6 +880,16 @@
         self._frozen = True
         return True
 
+class FunctionDeclaration(object):
+    def __init__(self, name, tp):
+        self.name = name
+        self.tp = tp
+
+    def get_llargs(self, cts):
+        return [cts.convert_type(arg) for arg in self.tp.args]
+
+    def get_llresult(self, cts):
+        return cts.convert_type(self.tp.result)
 
 def parse_source(source, includes=None, headers=None, configure_now=True):
     cts = CTypeSpace(headers=headers, includes=includes)
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -185,10 +185,27 @@
     typedef TestFloatObject* (*func_t)(int, int);
     """
     cts = parse_source(decl)
-    name, FUNC = cts.parse_func("func_t some_func(TestFloatObject*)")
-    assert name == 'some_func'
-    assert FUNC.RESULT == cts.gettype('func_t')
-    assert FUNC.ARGS == (cts.gettype('TestFloatObject *'),)
+    func_decl = cts.parse_func("func_t * some_func(TestFloatObject*)")
+    assert func_decl.name == 'some_func'
+    assert func_decl.get_llresult(cts) == cts.gettype('func_t*')
+    assert func_decl.get_llargs(cts) == [cts.gettype('TestFloatObject *')]
+
+def test_write_func():
+    from ..api import ApiFunction
+    from rpython.translator.c.database import LowLevelDatabase
+    db = LowLevelDatabase()
+    cdef = """
+    typedef ssize_t Py_ssize_t;
+    """
+    cts = parse_source(cdef)
+    cdecl = "Py_ssize_t * some_func(Py_ssize_t*)"
+    decl = cts.parse_func(cdecl)
+    api_function = ApiFunction(
+        decl.get_llargs(cts), decl.get_llresult(cts), lambda space, x: None,
+        cdecl=decl)
+    assert (api_function.get_api_decl('some_func', db) ==
+            "PyAPI_FUNC(Py_ssize_t *) some_func(Py_ssize_t * arg0);")
+
 
 def test_wchar_t():
     cdef = """

From pypy.commits at gmail.com  Sat Jan 21 17:38:23 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 21 Jan 2017 14:38:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5883e2df.c3e31c0a.519d1.77af@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89684:e03a42027bba
Date: 2017-01-21 22:37 +0000
http://bitbucket.org/pypy/pypy/changeset/e03a42027bba/

Log:	hg merge default

diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -310,6 +310,10 @@
                 static = os.path.join(sysroot, dir[1:], static_f)
                 xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f)
 
+            # PyPy extension here: 'shared' usually ends in something
+            # like '.pypy-41.so'.  Try without the '.pypy-41' part too.
+            shared_no_pypy = re.sub(r'[.]pypy[^.]+([.][^.]+)$', r'\1', shared)
+
             # We're second-guessing the linker here, with not much hard
             # data to go on: GCC seems to prefer the shared library, so I'm
             # assuming that *all* Unix C compilers do.  And of course I'm
@@ -320,6 +324,8 @@
                 return xcode_stub
             elif os.path.exists(shared):
                 return shared
+            elif os.path.exists(shared_no_pypy):
+                return shared_no_pypy
             elif os.path.exists(static):
                 return static
 
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -1,13 +1,18 @@
 """High performance data structures
+
+Note that PyPy also contains a built-in module '_collections' which will hide
+this one if compiled in.
+
+THIS ONE IS BOGUS in the sense that it is NOT THREAD-SAFE!  It is provided
+only as documentation nowadays.  Please don't run in production a PyPy
+without the '_collections' built-in module.  The built-in module is
+correctly thread-safe, like it is on CPython.
 """
 #
 # Copied and completed from the sandbox of CPython
 #   (nondist/sandbox/collections/pydeque.py rev 1.1, Raymond Hettinger)
 #
 
-# Note that PyPy also contains a built-in module '_collections' which will hide
-# this one if compiled in.
-
 try:
     from _thread import _get_ident as _thread_ident
 except ImportError:
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -382,10 +382,14 @@
 
     def get_c_restype(self, c_writer):
         if self.cdecl:
-            return self.cdecl.split(self.c_name)[0].strip()
+            return self.cdecl.tp.result.get_c_name()
         return c_writer.gettype(self.restype).replace('@', '').strip()
 
     def get_c_args(self, c_writer):
+        if self.cdecl:
+            args = [tp.get_c_name('arg%d' % i) for i, tp in
+                enumerate(self.cdecl.tp.args)]
+            return ', '.join(args) or "void"
         args = []
         for i, argtype in enumerate(self.argtypes):
             if argtype is CONST_STRING:
@@ -482,14 +486,15 @@
         return unwrapper
     return decorate
 
-def api_decl(cdecl, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+def api_decl(cdef, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
     def decorate(func):
         func._always_inline_ = 'try'
-        name, FUNC = cts.parse_func(cdecl)
+        cdecl = cts.parse_func(cdef)
+        RESULT = cdecl.get_llresult(cts)
         api_function = ApiFunction(
-            FUNC.ARGS, FUNC.RESULT, func,
-            error=_compute_error(error, FUNC.RESULT), cdecl=cdecl)
-        FUNCTIONS_BY_HEADER[header][name] = api_function
+            cdecl.get_llargs(cts), RESULT, func,
+            error=_compute_error(error, RESULT), cdecl=cdecl)
+        FUNCTIONS_BY_HEADER[header][cdecl.name] = api_function
         unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
diff --git a/pypy/module/cpyext/cmodel.py b/pypy/module/cpyext/cmodel.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/cmodel.py
@@ -0,0 +1,607 @@
+import types
+import weakref
+
+from .error import CDefError, VerificationError, VerificationMissing
+
+# type qualifiers
+Q_CONST    = 0x01
+Q_RESTRICT = 0x02
+Q_VOLATILE = 0x04
+
+def qualify(quals, replace_with):
+    if quals & Q_CONST:
+        replace_with = ' const ' + replace_with.lstrip()
+    if quals & Q_VOLATILE:
+        replace_with = ' volatile ' + replace_with.lstrip()
+    if quals & Q_RESTRICT:
+        # It seems that __restrict is supported by gcc and msvc.
+        # If you hit some different compiler, add a #define in
+        # _cffi_include.h for it (and in its copies, documented there)
+        replace_with = ' __restrict ' + replace_with.lstrip()
+    return replace_with
+
+
+class BaseTypeByIdentity(object):
+    is_array_type = False
+    is_raw_function = False
+
+    def get_c_name(self, replace_with='', context='a C file', quals=0):
+        result = self.c_name_with_marker
+        assert result.count('&') == 1
+        # some logic duplication with ffi.getctype()... :-(
+        replace_with = replace_with.strip()
+        if replace_with:
+            if replace_with.startswith('*') and '&[' in result:
+                replace_with = '(%s)' % replace_with
+            elif not replace_with[0] in '[(':
+                replace_with = ' ' + replace_with
+        replace_with = qualify(quals, replace_with)
+        result = result.replace('&', replace_with)
+        if '$' in result:
+            raise VerificationError(
+                "cannot generate '%s' in %s: unknown type name"
+                % (self._get_c_name(), context))
+        return result
+
+    def _get_c_name(self):
+        return self.c_name_with_marker.replace('&', '')
+
+    def has_c_name(self):
+        return '$' not in self._get_c_name()
+
+    def is_integer_type(self):
+        return False
+
+    def get_cached_btype(self, ffi, finishlist, can_delay=False):
+        try:
+            BType = ffi._cached_btypes[self]
+        except KeyError:
+            BType = self.build_backend_type(ffi, finishlist)
+            BType2 = ffi._cached_btypes.setdefault(self, BType)
+            assert BType2 is BType
+        return BType
+
+    def __repr__(self):
+        return '<%s>' % (self._get_c_name(),)
+
+    def _get_items(self):
+        return [(name, getattr(self, name)) for name in self._attrs_]
+
+
+class BaseType(BaseTypeByIdentity):
+
+    def __eq__(self, other):
+        return (self.__class__ == other.__class__ and
+                self._get_items() == other._get_items())
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __hash__(self):
+        return hash((self.__class__, tuple(self._get_items())))
+
+
+class VoidType(BaseType):
+    _attrs_ = ()
+
+    def __init__(self):
+        self.c_name_with_marker = 'void&'
+
+    def build_backend_type(self, ffi, finishlist):
+        return global_cache(self, ffi, 'new_void_type')
+
+void_type = VoidType()
+
+
+class BasePrimitiveType(BaseType):
+    pass
+
+
+class PrimitiveType(BasePrimitiveType):
+    _attrs_ = ('name',)
+
+    ALL_PRIMITIVE_TYPES = {
+        'char':               'c',
+        'short':              'i',
+        'int':                'i',
+        'long':               'i',
+        'long long':          'i',
+        'signed char':        'i',
+        'unsigned char':      'i',
+        'unsigned short':     'i',
+        'unsigned int':       'i',
+        'unsigned long':      'i',
+        'unsigned long long': 'i',
+        'float':              'f',
+        'double':             'f',
+        'long double':        'f',
+        '_Bool':              'i',
+        # the following types are not primitive in the C sense
+        'wchar_t':            'c',
+        'int8_t':             'i',
+        'uint8_t':            'i',
+        'int16_t':            'i',
+        'uint16_t':           'i',
+        'int32_t':            'i',
+        'uint32_t':           'i',
+        'int64_t':            'i',
+        'uint64_t':           'i',
+        'int_least8_t':       'i',
+        'uint_least8_t':      'i',
+        'int_least16_t':      'i',
+        'uint_least16_t':     'i',
+        'int_least32_t':      'i',
+        'uint_least32_t':     'i',
+        'int_least64_t':      'i',
+        'uint_least64_t':     'i',
+        'int_fast8_t':        'i',
+        'uint_fast8_t':       'i',
+        'int_fast16_t':       'i',
+        'uint_fast16_t':      'i',
+        'int_fast32_t':       'i',
+        'uint_fast32_t':      'i',
+        'int_fast64_t':       'i',
+        'uint_fast64_t':      'i',
+        'intptr_t':           'i',
+        'uintptr_t':          'i',
+        'intmax_t':           'i',
+        'uintmax_t':          'i',
+        'ptrdiff_t':          'i',
+        'size_t':             'i',
+        'ssize_t':            'i',
+        }
+
+    def __init__(self, name):
+        assert name in self.ALL_PRIMITIVE_TYPES
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def is_char_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'c'
+    def is_integer_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
+    def is_float_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+
+    def build_backend_type(self, ffi, finishlist):
+        return global_cache(self, ffi, 'new_primitive_type', self.name)
+
+
+class UnknownIntegerType(BasePrimitiveType):
+    _attrs_ = ('name',)
+
+    def __init__(self, name):
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def is_integer_type(self):
+        return True
+
+    def build_backend_type(self, ffi, finishlist):
+        raise NotImplementedError("integer type '%s' can only be used after "
+                                  "compilation" % self.name)
+
+class UnknownFloatType(BasePrimitiveType):
+    _attrs_ = ('name', )
+
+    def __init__(self, name):
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def build_backend_type(self, ffi, finishlist):
+        raise NotImplementedError("float type '%s' can only be used after "
+                                  "compilation" % self.name)
+
+class DefinedType(BaseType):
+    _attrs_ = ('name', )
+
+    def __init__(self, name, realtype, quals):
+        self.name = name
+        self.realtype = realtype
+        self.quals = quals
+        self.c_name_with_marker = name + '&'
+
+
+class BaseFunctionType(BaseType):
+    _attrs_ = ('args', 'result', 'ellipsis', 'abi')
+
+    def __init__(self, args, result, ellipsis, abi=None):
+        self.args = args
+        self.result = result
+        self.ellipsis = ellipsis
+        self.abi = abi
+        #
+        reprargs = [arg._get_c_name() for arg in self.args]
+        if self.ellipsis:
+            reprargs.append('...')
+        reprargs = reprargs or ['void']
+        replace_with = self._base_pattern % (', '.join(reprargs),)
+        if abi is not None:
+            replace_with = replace_with[:1] + abi + ' ' + replace_with[1:]
+        self.c_name_with_marker = (
+            self.result.c_name_with_marker.replace('&', replace_with))
+
+
+class RawFunctionType(BaseFunctionType):
+    # Corresponds to a C type like 'int(int)', which is the C type of
+    # a function, but not a pointer-to-function.  The backend has no
+    # notion of such a type; it's used temporarily by parsing.
+    _base_pattern = '(&)(%s)'
+    is_raw_function = True
+
+    def build_backend_type(self, ffi, finishlist):
+        raise CDefError("cannot render the type %r: it is a function "
+                        "type, not a pointer-to-function type" % (self,))
+
+    def as_function_pointer(self):
+        return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi)
+
+
+class FunctionPtrType(BaseFunctionType):
+    _base_pattern = '(*&)(%s)'
+
+    def build_backend_type(self, ffi, finishlist):
+        result = self.result.get_cached_btype(ffi, finishlist)
+        args = []
+        for tp in self.args:
+            args.append(tp.get_cached_btype(ffi, finishlist))
+        abi_args = ()
+        if self.abi == "__stdcall":
+            if not self.ellipsis:    # __stdcall ignored for variadic funcs
+                try:
+                    abi_args = (ffi._backend.FFI_STDCALL,)
+                except AttributeError:
+                    pass
+        return global_cache(self, ffi, 'new_function_type',
+                            tuple(args), result, self.ellipsis, *abi_args)
+
+    def as_raw_function(self):
+        return RawFunctionType(self.args, self.result, self.ellipsis, self.abi)
+
+
+class PointerType(BaseType):
+    _attrs_ = ('totype', 'quals')
+
+    def __init__(self, totype, quals=0):
+        self.totype = totype
+        self.quals = quals
+        extra = qualify(quals, " *&")
+        if totype.is_array_type:
+            extra = "(%s)" % (extra.lstrip(),)
+        self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra)
+
+    def build_backend_type(self, ffi, finishlist):
+        BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True)
+        return global_cache(self, ffi, 'new_pointer_type', BItem)
+
+voidp_type = PointerType(void_type)
+
+def ConstPointerType(totype):
+    return PointerType(totype, Q_CONST)
+
+const_voidp_type = ConstPointerType(void_type)
+
+
+class NamedPointerType(PointerType):
+    _attrs_ = ('totype', 'name')
+
+    def __init__(self, totype, name, quals=0):
+        PointerType.__init__(self, totype, quals)
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+
+class ArrayType(BaseType):
+    _attrs_ = ('item', 'length')
+    is_array_type = True
+
+    def __init__(self, item, length):
+        self.item = item
+        self.length = length
+        #
+        if length is None:
+            brackets = '&[]'
+        elif length == '...':
+            brackets = '&[/*...*/]'
+        else:
+            brackets = '&[%s]' % length
+        self.c_name_with_marker = (
+            self.item.c_name_with_marker.replace('&', brackets))
+
+    def resolve_length(self, newlength):
+        return ArrayType(self.item, newlength)
+
+    def build_backend_type(self, ffi, finishlist):
+        if self.length == '...':
+            raise CDefError("cannot render the type %r: unknown length" %
+                            (self,))
+        self.item.get_cached_btype(ffi, finishlist)   # force the item BType
+        BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
+        return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
+
+char_array_type = ArrayType(PrimitiveType('char'), None)
+
+
+class StructOrUnionOrEnum(BaseTypeByIdentity):
+    _attrs_ = ('name',)
+    forcename = None
+
+    def build_c_name_with_marker(self):
+        name = self.forcename or '%s %s' % (self.kind, self.name)
+        self.c_name_with_marker = name + '&'
+
+    def force_the_name(self, forcename):
+        self.forcename = forcename
+        self.build_c_name_with_marker()
+
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
+
+class StructOrUnion(StructOrUnionOrEnum):
+    fixedlayout = None
+    completed = 0
+    partial = False
+    packed = False
+
+    def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
+        self.name = name
+        self.fldnames = fldnames
+        self.fldtypes = fldtypes
+        self.fldbitsize = fldbitsize
+        self.fldquals = fldquals
+        self.build_c_name_with_marker()
+
+    def has_anonymous_struct_fields(self):
+        if self.fldtypes is None:
+            return False
+        for name, type in zip(self.fldnames, self.fldtypes):
+            if name == '' and isinstance(type, StructOrUnion):
+                return True
+        return False
+
+    def enumfields(self):
+        fldquals = self.fldquals
+        if fldquals is None:
+            fldquals = (0,) * len(self.fldnames)
+        for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes,
+                                              self.fldbitsize, fldquals):
+            if name == '' and isinstance(type, StructOrUnion):
+                # nested anonymous struct/union
+                for result in type.enumfields():
+                    yield result
+            else:
+                yield (name, type, bitsize, quals)
+
+    def force_flatten(self):
+        # force the struct or union to have a declaration that lists
+        # directly all fields returned by enumfields(), flattening
+        # nested anonymous structs/unions.
+        names = []
+        types = []
+        bitsizes = []
+        fldquals = []
+        for name, type, bitsize, quals in self.enumfields():
+            names.append(name)
+            types.append(type)
+            bitsizes.append(bitsize)
+            fldquals.append(quals)
+        self.fldnames = tuple(names)
+        self.fldtypes = tuple(types)
+        self.fldbitsize = tuple(bitsizes)
+        self.fldquals = tuple(fldquals)
+
+    def get_cached_btype(self, ffi, finishlist, can_delay=False):
+        BType = StructOrUnionOrEnum.get_cached_btype(self, ffi, finishlist,
+                                                     can_delay)
+        if not can_delay:
+            self.finish_backend_type(ffi, finishlist)
+        return BType
+
+    def finish_backend_type(self, ffi, finishlist):
+        if self.completed:
+            if self.completed != 2:
+                raise NotImplementedError("recursive structure declaration "
+                                          "for '%s'" % (self.name,))
+            return
+        BType = ffi._cached_btypes[self]
+        #
+        self.completed = 1
+        #
+        if self.fldtypes is None:
+            pass    # not completing it: it's an opaque struct
+            #
+        elif self.fixedlayout is None:
+            fldtypes = [tp.get_cached_btype(ffi, finishlist)
+                        for tp in self.fldtypes]
+            lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
+            sflags = 0
+            if self.packed:
+                sflags = 8    # SF_PACKED
+            ffi._backend.complete_struct_or_union(BType, lst, self,
+                                                  -1, -1, sflags)
+            #
+        else:
+            fldtypes = []
+            fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout
+            for i in range(len(self.fldnames)):
+                fsize = fieldsize[i]
+                ftype = self.fldtypes[i]
+                #
+                if isinstance(ftype, ArrayType) and ftype.length == '...':
+                    # fix the length to match the total size
+                    BItemType = ftype.item.get_cached_btype(ffi, finishlist)
+                    nlen, nrest = divmod(fsize, ffi.sizeof(BItemType))
+                    if nrest != 0:
+                        self._verification_error(
+                            "field '%s.%s' has a bogus size?" % (
+                            self.name, self.fldnames[i] or '{}'))
+                    ftype = ftype.resolve_length(nlen)
+                    self.fldtypes = (self.fldtypes[:i] + (ftype,) +
+                                     self.fldtypes[i+1:])
+                #
+                BFieldType = ftype.get_cached_btype(ffi, finishlist)
+                if isinstance(ftype, ArrayType) and ftype.length is None:
+                    assert fsize == 0
+                else:
+                    bitemsize = ffi.sizeof(BFieldType)
+                    if bitemsize != fsize:
+                        self._verification_error(
+                            "field '%s.%s' is declared as %d bytes, but is "
+                            "really %d bytes" % (self.name,
+                                                 self.fldnames[i] or '{}',
+                                                 bitemsize, fsize))
+                fldtypes.append(BFieldType)
+            #
+            lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs))
+            ffi._backend.complete_struct_or_union(BType, lst, self,
+                                                  totalsize, totalalignment)
+        self.completed = 2
+
+    def _verification_error(self, msg):
+        raise VerificationError(msg)
+
+    def check_not_partial(self):
+        if self.partial and self.fixedlayout is None:
+            raise VerificationMissing(self._get_c_name())
+
+    def build_backend_type(self, ffi, finishlist):
+        self.check_not_partial()
+        finishlist.append(self)
+        #
+        return global_cache(self, ffi, 'new_%s_type' % self.kind,
+                            self.get_official_name(), key=self)
+
+
+class StructType(StructOrUnion):
+    kind = 'struct'
+
+
+class UnionType(StructOrUnion):
+    kind = 'union'
+
+
+class EnumType(StructOrUnionOrEnum):
+    kind = 'enum'
+    partial = False
+    partial_resolved = False
+
+    def __init__(self, name, enumerators, enumvalues, baseinttype=None):
+        self.name = name
+        self.enumerators = enumerators
+        self.enumvalues = enumvalues
+        self.baseinttype = baseinttype
+        self.build_c_name_with_marker()
+
+    def force_the_name(self, forcename):
+        StructOrUnionOrEnum.force_the_name(self, forcename)
+        if self.forcename is None:
+            name = self.get_official_name()
+            self.forcename = '$' + name.replace(' ', '_')
+
+    def check_not_partial(self):
+        if self.partial and not self.partial_resolved:
+            raise VerificationMissing(self._get_c_name())
+
+    def build_backend_type(self, ffi, finishlist):
+        self.check_not_partial()
+        base_btype = self.build_baseinttype(ffi, finishlist)
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
+                            self.enumerators, self.enumvalues,
+                            base_btype, key=self)
+
+    def build_baseinttype(self, ffi, finishlist):
+        if self.baseinttype is not None:
+            return self.baseinttype.get_cached_btype(ffi, finishlist)
+        #
+        if self.enumvalues:
+            smallest_value = min(self.enumvalues)
+            largest_value = max(self.enumvalues)
+        else:
+            import warnings
+            try:
+                # XXX!  The goal is to ensure that the warnings.warn()
+                # will not suppress the warning.  We want to get it
+                # several times if we reach this point several times.
+                __warningregistry__.clear()
+            except NameError:
+                pass
+            warnings.warn("%r has no values explicitly defined; "
+                          "guessing that it is equivalent to 'unsigned int'"
+                          % self._get_c_name())
+            smallest_value = largest_value = 0
+        if smallest_value < 0:   # needs a signed type
+            sign = 1
+            candidate1 = PrimitiveType("int")
+            candidate2 = PrimitiveType("long")
+        else:
+            sign = 0
+            candidate1 = PrimitiveType("unsigned int")
+            candidate2 = PrimitiveType("unsigned long")
+        btype1 = candidate1.get_cached_btype(ffi, finishlist)
+        btype2 = candidate2.get_cached_btype(ffi, finishlist)
+        size1 = ffi.sizeof(btype1)
+        size2 = ffi.sizeof(btype2)
+        if (smallest_value >= ((-1) << (8*size1-1)) and
+            largest_value < (1 << (8*size1-sign))):
+            return btype1
+        if (smallest_value >= ((-1) << (8*size2-1)) and
+            largest_value < (1 << (8*size2-sign))):
+            return btype2
+        raise CDefError("%s values don't all fit into either 'long' "
+                        "or 'unsigned long'" % self._get_c_name())
+
+def unknown_type(name, structname=None):
+    if structname is None:
+        structname = '$%s' % name
+    tp = StructType(structname, None, None, None)
+    tp.force_the_name(name)
+    tp.origin = "unknown_type"
+    return tp
+
+def unknown_ptr_type(name, structname=None):
+    if structname is None:
+        structname = '$$%s' % name
+    tp = StructType(structname, None, None, None)
+    return NamedPointerType(tp, name)
+
+
+def global_cache(srctype, ffi, funcname, *args, **kwds):
+    key = kwds.pop('key', (funcname, args))
+    assert not kwds
+    try:
+        return ffi._backend.__typecache[key]
+    except KeyError:
+        pass
+    except AttributeError:
+        # initialize the __typecache attribute, either at the module level
+        # if ffi._backend is a module, or at the class level if ffi._backend
+        # is some instance.
+        if isinstance(ffi._backend, types.ModuleType):
+            ffi._backend.__typecache = weakref.WeakValueDictionary()
+        else:
+            type(ffi._backend).__typecache = weakref.WeakValueDictionary()
+    try:
+        res = getattr(ffi._backend, funcname)(*args)
+    except NotImplementedError as e:
+        raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e))
+    # note that setdefault() on WeakValueDictionary is not atomic
+    # and contains a rare bug (http://bugs.python.org/issue19542);
+    # we have to use a lock and do it ourselves
+    cache = ffi._backend.__typecache
+    with global_lock:
+        res1 = cache.get(key)
+        if res1 is None:
+            cache[key] = res
+            return res
+        else:
+            return res1
+
+def pointer_cache(ffi, BType):
+    return global_cache('?', ffi, 'new_pointer_type', BType)
+
+def attach_exception_info(e, name):
+    if e.args and type(e.args[0]) is str:
+        e.args = ('%s: %s' % (name, e.args[0]),) + e.args[1:]
diff --git a/pypy/module/cpyext/commontypes.py b/pypy/module/cpyext/commontypes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/commontypes.py
@@ -0,0 +1,80 @@
+import sys
+from . import cmodel as model
+from .error import FFIError
+
+
+COMMON_TYPES = {}
+
+try:
+    # fetch "bool" and all simple Windows types
+    from _cffi_backend import _get_common_types
+    _get_common_types(COMMON_TYPES)
+except ImportError:
+    pass
+
+COMMON_TYPES['FILE'] = model.unknown_type('FILE', '_IO_FILE')
+COMMON_TYPES['bool'] = '_Bool'    # in case we got ImportError above
+
+for _type in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
+    if _type.endswith('_t'):
+        COMMON_TYPES[_type] = _type
+del _type
+
+_CACHE = {}
+
+def resolve_common_type(parser, commontype):
+    try:
+        return _CACHE[commontype]
+    except KeyError:
+        cdecl = COMMON_TYPES.get(commontype, commontype)
+        if not isinstance(cdecl, str):
+            result, quals = cdecl, 0    # cdecl is already a BaseType
+        elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
+            result, quals = model.PrimitiveType(cdecl), 0
+        elif cdecl == 'set-unicode-needed':
+            raise FFIError("The Windows type %r is only available after "
+                           "you call ffi.set_unicode()" % (commontype,))
+        else:
+            if commontype == cdecl:
+                raise FFIError(
+                    "Unsupported type: %r.  Please look at "
+        "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
+                    "and file an issue if you think this type should really "
+                    "be supported." % (commontype,))
+            result, quals = parser.parse_type_and_quals(cdecl)   # recursive
+
+        assert isinstance(result, model.BaseTypeByIdentity)
+        _CACHE[commontype] = result, quals
+        return result, quals
+
+
+# ____________________________________________________________
+# extra types for Windows (most of them are in commontypes.c)
+
+
+def win_common_types():
+    return {
+        "UNICODE_STRING": model.StructType(
+            "_UNICODE_STRING",
+            ["Length",
+             "MaximumLength",
+             "Buffer"],
+            [model.PrimitiveType("unsigned short"),
+             model.PrimitiveType("unsigned short"),
+             model.PointerType(model.PrimitiveType("wchar_t"))],
+            [-1, -1, -1]),
+        "PUNICODE_STRING": "UNICODE_STRING *",
+        "PCUNICODE_STRING": "const UNICODE_STRING *",
+
+        "TBYTE": "set-unicode-needed",
+        "TCHAR": "set-unicode-needed",
+        "LPCTSTR": "set-unicode-needed",
+        "PCTSTR": "set-unicode-needed",
+        "LPTSTR": "set-unicode-needed",
+        "PTSTR": "set-unicode-needed",
+        "PTBYTE": "set-unicode-needed",
+        "PTCHAR": "set-unicode-needed",
+        }
+
+if sys.platform == 'win32':
+    COMMON_TYPES.update(win_common_types())
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -1,12 +1,12 @@
-import sys
 from collections import OrderedDict
-from cffi import api, model
-from cffi.commontypes import COMMON_TYPES, resolve_common_type
+from . import cmodel as model
+from .commontypes import COMMON_TYPES, resolve_common_type
+from .error import FFIError, CDefError
 try:
     from cffi import _pycparser as pycparser
 except ImportError:
     import pycparser
-import weakref, re
+import weakref, re, sys
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rfile import FILEP
 from rpython.rtyper.lltypesystem import rffi, lltype
@@ -161,7 +161,7 @@
             msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
-        raise api.CDefError(msg)
+        raise CDefError(msg)
 
     def parse(self, csource, override=False, packed=False, dllexport=False):
         prev_options = self._options
@@ -189,18 +189,12 @@
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
-                    if not decl.name:
-                        raise api.CDefError("typedef does not declare any name",
-                                            decl)
-                    quals = 0
-                    realtype, quals = self._get_type_and_quals(
-                        decl.type, name=decl.name, partial_length_ok=True)
-                    self._declare('typedef ' + decl.name, realtype, quals=quals)
+                    self._parse_typedef(decl)
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise api.CDefError("unrecognized construct", decl)
-        except api.FFIError as e:
+                    raise CDefError("unrecognized construct", decl)
+        except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
                 e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
@@ -210,7 +204,7 @@
         if key in self._int_constants:
             if self._int_constants[key] == val:
                 return     # ignore identical double declarations
-            raise api.FFIError(
+            raise FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
 
@@ -245,6 +239,14 @@
             tag = 'function '
         self._declare(tag + decl.name, tp)
 
+    def _parse_typedef(self, decl):
+        if not decl.name:
+            raise CDefError("typedef does not declare any name", decl)
+        realtype, quals = self._get_type_and_quals(
+            decl.type, name=decl.name, partial_length_ok=True)
+        tp = model.DefinedType(decl.name, realtype, quals)
+        self._declare('typedef ' + decl.name, tp)
+
     def _parse_decl(self, decl):
         node = decl.type
         if isinstance(node, pycparser.c_ast.FuncDecl):
@@ -259,8 +261,8 @@
             elif isinstance(node, pycparser.c_ast.Enum):
                 self._get_struct_union_enum_type('enum', node)
             elif not decl.name:
-                raise api.CDefError("construct does not declare any variable",
-                                    decl)
+                raise CDefError("construct does not declare any variable",
+                                decl)
             #
             if decl.name:
                 tp, quals = self._get_type_and_quals(node,
@@ -292,7 +294,7 @@
         ast, _, _ = self._parse('void __dummy(\n%s\n);' % cdecl)
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
-            raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
+            raise CDefError("unknown identifier '%s'" % (exprnode.name,))
         return self._get_type_and_quals(exprnode.type)
 
     def _declare(self, name, obj, included=False, quals=0):
@@ -326,15 +328,6 @@
         return model.PointerType(type, quals)
 
     def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False):
-        # first, dereference typedefs, if we have it already parsed, we're good
-        if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
-            isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
-            len(typenode.type.names) == 1 and
-            ('typedef ' + typenode.type.names[0]) in self._declarations):
-            tp, quals = self._declarations['typedef ' + typenode.type.names[0]]
-            quals |= self._extract_quals(typenode)
-            return tp, quals
-        #
         if isinstance(typenode, pycparser.c_ast.ArrayDecl):
             # array type
             if typenode.dim is None:
@@ -357,6 +350,11 @@
             quals = self._extract_quals(typenode)
             type = typenode.type
             if isinstance(type, pycparser.c_ast.IdentifierType):
+                # first, dereference typedefs, if we have it already parsed, we're good
+                if (len(type.names) == 1 and
+                    ('typedef ' + type.names[0]) in self._declarations):
+                    tp0, quals0 = self._declarations['typedef ' + type.names[0]]
+                    return tp0, (quals | quals0)
                 # assume a primitive type.  get it from .names, but reduce
                 # synonyms to a single chosen combination
                 names = list(type.names)
@@ -413,14 +411,14 @@
             return self._get_struct_union_enum_type('union', typenode, name,
                                                     nested=True), 0
         #
-        raise api.FFIError(":%d: bad or unsupported type declaration" %
+        raise FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
     def _parse_function_type(self, typenode, funcname=None):
         params = list(getattr(typenode.args, 'params', []))
         for i, arg in enumerate(params):
             if not hasattr(arg, 'type'):
-                raise api.CDefError("%s arg %d: unknown type '%s'"
+                raise CDefError("%s arg %d: unknown type '%s'"
                     " (if you meant to use the old C syntax of giving"
                     " untyped arguments, it is not supported)"
                     % (funcname or 'in expression', i + 1,
@@ -434,7 +432,7 @@
         if ellipsis:
             params.pop()
             if not params:
-                raise api.CDefError(
+                raise CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
         args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type))
@@ -533,7 +531,7 @@
             return tp
         #
         if tp.fldnames is not None:
-            raise api.CDefError("duplicate declaration of struct %s" % name)
+            raise CDefError("duplicate declaration of struct %s" % name)
         fldnames = []
         fldtypes = []
         fldbitsize = []
@@ -570,7 +568,7 @@
 
     def _make_partial(self, tp, nested):
         if not isinstance(tp, model.StructOrUnion):
-            raise api.CDefError("%s cannot be partial" % (tp,))
+            raise CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
         tp.partial = True
@@ -590,7 +588,7 @@
                     len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
                 return ord(s[-2])
             else:
-                raise api.CDefError("invalid constant %r" % (s,))
+                raise CDefError("invalid constant %r" % (s,))
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '+'):
@@ -609,12 +607,12 @@
             if partial_length_ok:
                 self._partial_length = True
                 return '...'
-            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
-                               "the actual array length in this context"
-                               % exprnode.coord.line)
+            raise FFIError(":%d: unsupported '[...]' here, cannot derive "
+                           "the actual array length in this context"
+                           % exprnode.coord.line)
         #
-        raise api.FFIError(":%d: unsupported expression: expected a "
-                           "simple numeric constant" % exprnode.coord.line)
+        raise FFIError(":%d: unsupported expression: expected a "
+                       "simple numeric constant" % exprnode.coord.line)
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
@@ -806,6 +804,8 @@
                 del self._TYPES[name]
 
     def convert_type(self, obj, quals=0):
+        if isinstance(obj, model.DefinedType):
+            return self.convert_type(obj.realtype, obj.quals)
         if isinstance(obj, model.PrimitiveType):
             return cname_to_lltype(obj.name)
         elif isinstance(obj, model.StructType):
@@ -856,8 +856,7 @@
         ast, _, _ = self.ctx._parse(cdecl)
         decl = ast.ext[-1]
         tp, quals = self.ctx._get_type_and_quals(decl.type, name=decl.name)
-        FUNCP = self.convert_type(tp.as_function_pointer())
-        return decl.name, FUNCP.TO
+        return FunctionDeclaration(decl.name, tp)
 
     def _freeze_(self):
         if self._frozen:
@@ -881,6 +880,16 @@
         self._frozen = True
         return True
 
+class FunctionDeclaration(object):
+    def __init__(self, name, tp):
+        self.name = name
+        self.tp = tp
+
+    def get_llargs(self, cts):
+        return [cts.convert_type(arg) for arg in self.tp.args]
+
+    def get_llresult(self, cts):
+        return cts.convert_type(self.tp.result)
 
 def parse_source(source, includes=None, headers=None, configure_now=True):
     cts = CTypeSpace(headers=headers, includes=includes)
diff --git a/pypy/module/cpyext/error.py b/pypy/module/cpyext/error.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/error.py
@@ -0,0 +1,20 @@
+
+class FFIError(Exception):
+    pass
+
+class CDefError(Exception):
+    def __str__(self):
+        try:
+            line = 'line %d: ' % (self.args[1].coord.line,)
+        except (AttributeError, TypeError, IndexError):
+            line = ''
+        return '%s%s' % (line, self.args[0])
+
+class VerificationError(Exception):
+    """ An error raised when verification fails
+    """
+
+class VerificationMissing(Exception):
+    """ An error raised when incomplete structures are passed into
+    cdef, but no verification has been done
+    """
diff --git a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h
--- a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h
+++ b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h
@@ -11,6 +11,7 @@
 #define NUMPY_IMPORT_ARRAY_RETVAL
 #endif
 
-#define import_array() {return NUMPY_IMPORT_ARRAY_RETVAL;}
+/* on pypy import_array never fails, so it's just an empty macro */
+#define import_array()
 
 
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -7,14 +7,7 @@
 extern "C" {
 #endif
 
-/* Hack: MSVC doesn't support ssize_t */
-#ifdef _WIN32
-#define ssize_t long
-#endif
 #include <cpyext_object.h>
-#ifdef _WIN32
-#undef ssize_t
-#endif
 
 #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
 #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -1,5 +1,5 @@
 
-typedef ssize_t Py_ssize_t;
+typedef long Py_ssize_t;
 
 #define PyObject_HEAD  \
     Py_ssize_t ob_refcnt;        \
diff --git a/pypy/module/cpyext/test/test_borrow.py b/pypy/module/cpyext/test/test_borrow.py
--- a/pypy/module/cpyext/test/test_borrow.py
+++ b/pypy/module/cpyext/test/test_borrow.py
@@ -12,13 +12,13 @@
                 PyObject *t = PyTuple_New(1);
                 PyObject *f = PyFloat_FromDouble(42.0);
                 PyObject *g = NULL;
-                printf("Refcnt1: %zd\\n", f->ob_refcnt);
+                printf("Refcnt1: %ld\\n", f->ob_refcnt);
                 PyTuple_SetItem(t, 0, f); // steals reference
-                printf("Refcnt2: %zd\\n", f->ob_refcnt);
+                printf("Refcnt2: %ld\\n", f->ob_refcnt);
                 f = PyTuple_GetItem(t, 0); // borrows reference
-                printf("Refcnt3: %zd\\n", f->ob_refcnt);
+                printf("Refcnt3: %ld\\n", f->ob_refcnt);
                 g = PyTuple_GetItem(t, 0); // borrows reference again
-                printf("Refcnt4: %zd\\n", f->ob_refcnt);
+                printf("Refcnt4: %ld\\n", f->ob_refcnt);
                 printf("COMPARE: %i\\n", f == g);
                 fflush(stdout);
                 Py_DECREF(t);
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -185,10 +185,27 @@
     typedef TestFloatObject* (*func_t)(int, int);
     """
     cts = parse_source(decl)
-    name, FUNC = cts.parse_func("func_t some_func(TestFloatObject*)")
-    assert name == 'some_func'
-    assert FUNC.RESULT == cts.gettype('func_t')
-    assert FUNC.ARGS == (cts.gettype('TestFloatObject *'),)
+    func_decl = cts.parse_func("func_t * some_func(TestFloatObject*)")
+    assert func_decl.name == 'some_func'
+    assert func_decl.get_llresult(cts) == cts.gettype('func_t*')
+    assert func_decl.get_llargs(cts) == [cts.gettype('TestFloatObject *')]
+
+def test_write_func():
+    from ..api import ApiFunction
+    from rpython.translator.c.database import LowLevelDatabase
+    db = LowLevelDatabase()
+    cdef = """
+    typedef ssize_t Py_ssize_t;
+    """
+    cts = parse_source(cdef)
+    cdecl = "Py_ssize_t * some_func(Py_ssize_t*)"
+    decl = cts.parse_func(cdecl)
+    api_function = ApiFunction(
+        decl.get_llargs(cts), decl.get_llresult(cts), lambda space, x: None,
+        cdecl=decl)
+    assert (api_function.get_api_decl('some_func', db) ==
+            "PyAPI_FUNC(Py_ssize_t *) some_func(Py_ssize_t * arg0);")
+
 
 def test_wchar_t():
     cdef = """
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -627,7 +627,7 @@
             refcnt_after = true_obj->ob_refcnt;
             Py_DECREF(true_obj);
             Py_DECREF(true_obj);
-            fprintf(stderr, "REFCNT %zd %zd\\n", refcnt, refcnt_after);
+            fprintf(stderr, "REFCNT %ld %ld\\n", refcnt, refcnt_after);
             return PyBool_FromLong(refcnt_after == refcnt + 2);
         }
         static PyObject* foo_bar(PyObject* self, PyObject *args)
@@ -643,7 +643,7 @@
                 return NULL;
             refcnt_after = true_obj->ob_refcnt;
             Py_DECREF(tup);
-            fprintf(stderr, "REFCNT2 %zd %zd %zd\\n", refcnt, refcnt_after,
+            fprintf(stderr, "REFCNT2 %ld %ld %ld\\n", refcnt, refcnt_after,
                     true_obj->ob_refcnt);
             return PyBool_FromLong(refcnt_after == refcnt + 1 &&
                                    refcnt == true_obj->ob_refcnt);
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -726,15 +726,7 @@
             op.setref_base(value)
 
     def _record_op(self, opnum, argboxes, descr=None):
-        from rpython.jit.metainterp.opencoder import FrontendTagOverflow
-
-        try:
-            return self.trace.record_op(opnum, argboxes, descr)
-        except FrontendTagOverflow:
-            # note that with the default settings this one should not
-            # happen - however if we hit that case, we don't get
-            # anything disabled
-            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        return self.trace.record_op(opnum, argboxes, descr)
 
     @specialize.argtype(3)
     def record(self, opnum, argboxes, value, descr=None):
diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py
--- a/rpython/jit/metainterp/opencoder.py
+++ b/rpython/jit/metainterp/opencoder.py
@@ -49,8 +49,12 @@
     way up to lltype.Signed for indexes everywhere
     """
 
-class FrontendTagOverflow(Exception):
-    pass
+def frontend_tag_overflow():
+    # Minor abstraction leak: raise directly the right exception
+    # expected by the rest of the machinery
+    from rpython.jit.metainterp import history
+    from rpython.rlib.jit import Counters
+    raise history.SwitchToBlackhole(Counters.ABORT_TOO_LONG)
 
 class BaseTrace(object):
     pass
@@ -296,7 +300,7 @@
             # grow by 2X
             self._ops = self._ops + [rffi.cast(model.STORAGE_TP, 0)] * len(self._ops)
         if not model.MIN_VALUE <= v <= model.MAX_VALUE:
-            raise FrontendTagOverflow
+            raise frontend_tag_overflow()
         self._ops[self._pos] = rffi.cast(model.STORAGE_TP, v)
         self._pos += 1
 
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -519,6 +519,8 @@
         self.jump_args.append(preamble_op.preamble_op)
 
     def use_box(self, box, preamble_op, optimizer=None):
+        if not self.build_inplace:
+            raise InvalidLoop("Forcing boxes would modify an existing short preamble")
         jump_op = self.short.pop()
         AbstractShortPreambleBuilder.use_box(self, box, preamble_op, optimizer)
         self.short.append(jump_op)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2424,7 +2424,6 @@
         self.staticdata.profiler.start_tracing()
         assert jitdriver_sd is self.jitdriver_sd
         self.staticdata.try_to_free_some_loops()
-        self.create_empty_history()
         try:
             original_boxes = self.initialize_original_boxes(jitdriver_sd, *args)
             return self._compile_and_run_once(original_boxes)
@@ -2438,10 +2437,11 @@
         num_green_args = self.jitdriver_sd.num_green_args
         original_greenkey = original_boxes[:num_green_args]
         self.resumekey = compile.ResumeFromInterpDescr(original_greenkey)
-        self.history.set_inputargs(original_boxes[num_green_args:],
-                                   self.staticdata)
         self.seen_loop_header_for_jdindex = -1
         try:
+            self.create_empty_history()
+            self.history.set_inputargs(original_boxes[num_green_args:],
+                                       self.staticdata)
             self.interpret()
         except SwitchToBlackhole as stb:
             self.run_blackhole_interp_to_cancel_tracing(stb)
@@ -2461,9 +2461,11 @@
         if self.resumekey_original_loop_token is None:
             raise compile.giveup() # should be rare
         self.staticdata.try_to_free_some_loops()
-        inputargs = self.initialize_state_from_guard_failure(key, deadframe)
         try:
+            inputargs = self.initialize_state_from_guard_failure(key, deadframe)
             return self._handle_guard_failure(resumedescr, key, inputargs, deadframe)
+        except SwitchToBlackhole as stb:
+            self.run_blackhole_interp_to_cancel_tracing(stb)
         finally:
             self.resumekey_original_loop_token = None
             self.staticdata.profiler.end_tracing()
@@ -2475,13 +2477,10 @@
         self.seen_loop_header_for_jdindex = -1
         if isinstance(key, compile.ResumeAtPositionDescr):
             self.seen_loop_header_for_jdindex = self.jitdriver_sd.index
-        try:
-            self.prepare_resume_from_failure(deadframe, inputargs, resumedescr)
-            if self.resumekey_original_loop_token is None:   # very rare case
-                raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
-            self.interpret()
-        except SwitchToBlackhole as stb:
-            self.run_blackhole_interp_to_cancel_tracing(stb)
+        self.prepare_resume_from_failure(deadframe, inputargs, resumedescr)
+        if self.resumekey_original_loop_token is None:   # very rare case
+            raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
+        self.interpret()
         assert False, "should always raise"
 
     def run_blackhole_interp_to_cancel_tracing(self, stb):
diff --git a/rpython/jit/metainterp/test/test_opencoder.py b/rpython/jit/metainterp/test/test_opencoder.py
--- a/rpython/jit/metainterp/test/test_opencoder.py
+++ b/rpython/jit/metainterp/test/test_opencoder.py
@@ -1,6 +1,5 @@
 import py
 from rpython.jit.metainterp.opencoder import Trace, untag, TAGINT, TAGBOX
-from rpython.jit.metainterp.opencoder import FrontendTagOverflow
 from rpython.jit.metainterp.resoperation import rop, AbstractResOp
 from rpython.jit.metainterp.history import ConstInt, IntFrontendOp
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
@@ -8,6 +7,7 @@
 from rpython.jit.metainterp.test.strategies import lists_of_operations
 from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest
 from rpython.jit.metainterp.history import TreeLoop, AbstractDescr
+from rpython.jit.metainterp.history import SwitchToBlackhole
 from hypothesis import given, strategies
 
 class JitCode(object):
@@ -209,5 +209,5 @@
     def test_tag_overflow(self):
         t = Trace([], metainterp_sd)
         i0 = FakeOp(100000)
-        py.test.raises(FrontendTagOverflow, t.record_op, rop.FINISH, [i0])
+        py.test.raises(SwitchToBlackhole, t.record_op, rop.FINISH, [i0])
         assert t.unpack() == ([], [])
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_zvector.py
rename from rpython/jit/metainterp/test/test_vector.py
rename to rpython/jit/metainterp/test/test_zvector.py

From pypy.commits at gmail.com  Sun Jan 22 11:02:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 08:02:41 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <5884d7a1.d3811c0a.617f9.5d4b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r849:e655714ca28f
Date: 2017-01-22 17:02 +0100
http://bitbucket.org/pypy/pypy.org/changeset/e655714ca28f/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66569 of $105000 (63.4%)
+   $66570 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 

From pypy.commits at gmail.com  Sun Jan 22 11:35:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 08:35:44 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe: tweak
Message-ID: <5884df60.50a4df0a.f684b.2ea7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe
Changeset: r89685:e8a2d7c06655
Date: 2017-01-21 11:12 +0100
http://bitbucket.org/pypy/pypy/changeset/e8a2d7c06655/

Log:	tweak

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -879,9 +879,11 @@
                     self.nublocks = rthread.get_threadlocal_base()
             else:
                 if llop.get_gil_share_count(lltype.Signed) > 1:
-                    assert old_color == 0
+                    self._gc_unlock()
+                    ll_assert(old_color == 0, "old_color != 0")
                     old_color = llop.get_gil_color(lltype.Signed)
                     llop.set_gil_color(lltype.Void, self.gil_gc_color)
+                    self._gc_lock()
                     continue      # waited, maybe the situation changed
                 minor_collection_count += 1
                 if minor_collection_count == 1:

From pypy.commits at gmail.com  Sun Jan 22 11:35:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 08:35:46 -0800 (PST)
Subject: [pypy-commit] pypy default: fix tests that import test_vector
Message-ID: <5884df62.810b1c0a.681cb.78dc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89686:858ae6e5cbfe
Date: 2017-01-22 17:35 +0100
http://bitbucket.org/pypy/pypy/changeset/858ae6e5cbfe/

Log:	fix tests that import test_vector

diff --git a/rpython/jit/backend/ppc/test/test_ppcvector.py b/rpython/jit/backend/ppc/test/test_ppcvector.py
--- a/rpython/jit/backend/ppc/test/test_ppcvector.py
+++ b/rpython/jit/backend/ppc/test/test_ppcvector.py
@@ -1,10 +1,10 @@
 import py
 from rpython.jit.backend.ppc.test import test_basic
-from rpython.jit.metainterp.test import test_vector
+from rpython.jit.metainterp.test import test_zvector
 from rpython.jit.backend.ppc.detect_feature import detect_vsx
 
 
-class TestBasic(test_basic.JitPPCMixin, test_vector.VectorizeTests):
+class TestBasic(test_basic.JitPPCMixin, test_zvector.VectorizeTests):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_basic.py
     def setup_method(self, method):
diff --git a/rpython/jit/backend/x86/test/test_x86vector.py b/rpython/jit/backend/x86/test/test_x86vector.py
--- a/rpython/jit/backend/x86/test/test_x86vector.py
+++ b/rpython/jit/backend/x86/test/test_x86vector.py
@@ -6,11 +6,11 @@
 from rpython.jit.backend.x86.test import test_basic
 from rpython.jit.backend.x86.test.test_assembler import \
         (TestRegallocPushPop as BaseTestAssembler)
-from rpython.jit.metainterp.test import test_vector
+from rpython.jit.metainterp.test import test_zvector
 from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.backend.detect_cpu import getcpuclass
 
-class TestBasic(test_basic.Jit386Mixin, test_vector.VectorizeTests):
+class TestBasic(test_basic.Jit386Mixin, test_zvector.VectorizeTests):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_basic.py
     def setup_method(self, method):
diff --git a/rpython/jit/backend/zarch/test/test_zarchvector.py b/rpython/jit/backend/zarch/test/test_zarchvector.py
--- a/rpython/jit/backend/zarch/test/test_zarchvector.py
+++ b/rpython/jit/backend/zarch/test/test_zarchvector.py
@@ -1,10 +1,10 @@
 import py
 from rpython.jit.backend.zarch.test import test_basic
-from rpython.jit.metainterp.test import test_vector
+from rpython.jit.metainterp.test import test_zvector
 from rpython.jit.backend.zarch.detect_feature import detect_simd_z
 
 
-class TestBasic(test_basic.JitZARCHMixin, test_vector.VectorizeTests):
+class TestBasic(test_basic.JitZARCHMixin, test_zvector.VectorizeTests):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_basic.py
     def setup_method(self, method):

From pypy.commits at gmail.com  Sun Jan 22 11:37:02 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 08:37:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5884dfae.9989df0a.fde4d.2e99@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89687:82de3a7f19fe
Date: 2017-01-22 17:36 +0100
http://bitbucket.org/pypy/pypy/changeset/82de3a7f19fe/

Log:	hg merge default

diff --git a/rpython/jit/backend/ppc/test/test_ppcvector.py b/rpython/jit/backend/ppc/test/test_ppcvector.py
--- a/rpython/jit/backend/ppc/test/test_ppcvector.py
+++ b/rpython/jit/backend/ppc/test/test_ppcvector.py
@@ -1,10 +1,10 @@
 import py
 from rpython.jit.backend.ppc.test import test_basic
-from rpython.jit.metainterp.test import test_vector
+from rpython.jit.metainterp.test import test_zvector
 from rpython.jit.backend.ppc.detect_feature import detect_vsx
 
 
-class TestBasic(test_basic.JitPPCMixin, test_vector.VectorizeTests):
+class TestBasic(test_basic.JitPPCMixin, test_zvector.VectorizeTests):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_basic.py
     def setup_method(self, method):
diff --git a/rpython/jit/backend/x86/test/test_x86vector.py b/rpython/jit/backend/x86/test/test_x86vector.py
--- a/rpython/jit/backend/x86/test/test_x86vector.py
+++ b/rpython/jit/backend/x86/test/test_x86vector.py
@@ -6,11 +6,11 @@
 from rpython.jit.backend.x86.test import test_basic
 from rpython.jit.backend.x86.test.test_assembler import \
         (TestRegallocPushPop as BaseTestAssembler)
-from rpython.jit.metainterp.test import test_vector
+from rpython.jit.metainterp.test import test_zvector
 from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.backend.detect_cpu import getcpuclass
 
-class TestBasic(test_basic.Jit386Mixin, test_vector.VectorizeTests):
+class TestBasic(test_basic.Jit386Mixin, test_zvector.VectorizeTests):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_basic.py
     def setup_method(self, method):
diff --git a/rpython/jit/backend/zarch/test/test_zarchvector.py b/rpython/jit/backend/zarch/test/test_zarchvector.py
--- a/rpython/jit/backend/zarch/test/test_zarchvector.py
+++ b/rpython/jit/backend/zarch/test/test_zarchvector.py
@@ -1,10 +1,10 @@
 import py
 from rpython.jit.backend.zarch.test import test_basic
-from rpython.jit.metainterp.test import test_vector
+from rpython.jit.metainterp.test import test_zvector
 from rpython.jit.backend.zarch.detect_feature import detect_simd_z
 
 
-class TestBasic(test_basic.JitZARCHMixin, test_vector.VectorizeTests):
+class TestBasic(test_basic.JitZARCHMixin, test_zvector.VectorizeTests):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_basic.py
     def setup_method(self, method):

From pypy.commits at gmail.com  Sun Jan 22 11:44:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 08:44:06 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Copy more closely the logic of CPython
 3.5 for 'closefd' inside FileIO.__init__()
Message-ID: <5884e156.c3e31c0a.519d1.73c4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89688:983e1e1ee62f
Date: 2017-01-22 17:43 +0100
http://bitbucket.org/pypy/pypy/changeset/983e1e1ee62f/

Log:	Copy more closely the logic of CPython 3.5 for 'closefd' inside
	FileIO.__init__()

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -145,7 +145,12 @@
 
     @unwrap_spec(mode=str, closefd=int)
     def descr_init(self, space, w_name, mode='r', closefd=True, w_opener=None):
-        self._close(space)
+        if self.fd >= 0:
+            if self.closefd:
+                self._close(space)
+            else:
+                self.fd = -1
+
         if space.isinstance_w(w_name, space.w_float):
             raise oefmt(space.w_TypeError,
                         "integer argument expected, got float")
@@ -168,48 +173,52 @@
             if fd >= 0:
                 self.fd = fd
                 self.closefd = bool(closefd)
-            elif space.is_none(w_opener):
+            else:
                 self.closefd = True
                 if not closefd:
                     raise oefmt(space.w_ValueError,
                                 "Cannot use closefd=False with file name")
 
-                from pypy.module.posix.interp_posix import dispatch_filename
-                while True:
+                if space.is_none(w_opener):
+                    from pypy.module.posix.interp_posix import dispatch_filename
+                    while True:
+                        try:
+                            self.fd = dispatch_filename(rposix.open)(
+                                space, w_name, flags, 0666)
+                            fd_is_own = True
+                            break
+                        except OSError as e:
+                            wrap_oserror2(space, e, w_name,
+                                          exception_name='w_IOError',
+                                          eintr_retry=True)
+                    if not rposix._WIN32:
+                        try:
+                            _open_inhcache.set_non_inheritable(self.fd)
+                        except OSError as e:
+                            raise wrap_oserror2(space, e, w_name,
+                                                eintr_retry=False)
+                else:
+                    w_fd = space.call_function(w_opener, w_name,
+                                               space.wrap(flags))
                     try:
-                        self.fd = dispatch_filename(rposix.open)(
-                            space, w_name, flags, 0666)
+                        self.fd = space.int_w(w_fd)
+                        if self.fd < 0:
+                            # The opener returned a negative result instead
+                            # of raising an exception
+                            raise oefmt(space.w_ValueError,
+                                        "opener returned %d", self.fd)
                         fd_is_own = True
-                        break
-                    except OSError as e:
-                        wrap_oserror2(space, e, w_name,
-                                      exception_name='w_IOError',
-                                      eintr_retry=True)
-                if not rposix._WIN32:
-                    try:
-                        _open_inhcache.set_non_inheritable(self.fd)
-                    except OSError as e:
-                        raise wrap_oserror2(space, e, w_name, eintr_retry=False)
-            else:
-                w_fd = space.call_function(w_opener, w_name, space.wrap(flags))
-                try:
-                    self.fd = space.int_w(w_fd)
-                    if self.fd < 0:
-                        # The opener returned a negative result instead
-                        # of raising an exception
-                        raise oefmt(space.w_ValueError,
-                                    "opener returned %d", self.fd)
-                    fd_is_own = True
-                except OperationError as e:
-                    if not e.match(space, space.w_TypeError):
-                        raise
-                    raise oefmt(space.w_TypeError,
-                                "expected integer from opener")
-                if not rposix._WIN32:
-                    try:
-                        rposix.set_inheritable(self.fd, False)
-                    except OSError as e:
-                        raise wrap_oserror2(space, e, w_name, eintr_retry=False)
+                    except OperationError as e:
+                        if not e.match(space, space.w_TypeError):
+                            raise
+                        raise oefmt(space.w_TypeError,
+                                    "expected integer from opener")
+                    if not rposix._WIN32:
+                        try:
+                            rposix.set_inheritable(self.fd, False)
+                        except OSError as e:
+                            raise wrap_oserror2(space, e, w_name,
+                                                eintr_retry=False)
 
             try:
                 st = os.fstat(self.fd)

From pypy.commits at gmail.com  Sun Jan 22 11:59:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 08:59:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Check for closed files before doing
 peek(). Also, remove _check_init
Message-ID: <5884e505.849c1c0a.f4df3.738c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89689:e56a1d211ebe
Date: 2017-01-22 17:56 +0100
http://bitbucket.org/pypy/pypy/changeset/e56a1d211ebe/

Log:	Check for closed files before doing peek(). Also, remove _check_init
	right before _check_closed because the latter does the former too

diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -305,11 +305,10 @@
 
     @unwrap_spec(pos=r_longlong, whence=int)
     def seek_w(self, space, pos, whence=0):
-        self._check_init(space)
+        self._check_closed(space, "seek of closed file")
         if whence not in (0, 1, 2):
             raise oefmt(space.w_ValueError,
                         "whence must be between 0 and 2, not %d", whence)
-        self._check_closed(space, "seek of closed file")
         check_seekable_w(space, self.w_raw)
         if whence != 2 and self.readable:
             # Check if seeking leaves us inside the current buffer, so as to
@@ -462,7 +461,6 @@
     # Read methods
 
     def read_w(self, space, w_size=None):
-        self._check_init(space)
         self._check_closed(space, "read of closed file")
         size = convert_size(space, w_size)
 
@@ -482,7 +480,7 @@
 
     @unwrap_spec(size=int)
     def peek_w(self, space, size=0):
-        self._check_init(space)
+        self._check_closed(space, "peek of closed file")
         with self.lock:
             if self.writable:
                 self._flush_and_rewind_unlocked(space)
@@ -509,7 +507,6 @@
 
     @unwrap_spec(size=int)
     def read1_w(self, space, size):
-        self._check_init(space)
         self._check_closed(space, "read of closed file")
 
         if size < 0:
@@ -690,7 +687,6 @@
         return None
 
     def readline_w(self, space, w_limit=None):
-        self._check_init(space)
         self._check_closed(space, "readline of closed file")
 
         limit = convert_size(space, w_limit)
@@ -766,7 +762,6 @@
             self.read_end = self.pos
 
     def write_w(self, space, w_data):
-        self._check_init(space)
         self._check_closed(space, "write to closed file")
         data = space.getarg_w('y*', w_data).as_str()
         size = len(data)
@@ -873,7 +868,6 @@
         return space.wrap(written)
 
     def flush_w(self, space):
-        self._check_init(space)
         self._check_closed(space, "flush of closed file")
         with self.lock:
             self._flush_and_rewind_unlocked(space)

From pypy.commits at gmail.com  Sun Jan 22 13:30:45 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 22 Jan 2017 10:30:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Remove a define that is very confusing on
 Python 3
Message-ID: <5884fa55.92d41c0a.8f565.9996@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89690:37af509195dc
Date: 2017-01-22 18:30 +0000
http://bitbucket.org/pypy/pypy/changeset/37af509195dc/

Log:	Remove a define that is very confusing on Python 3

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -115,8 +115,6 @@
 
 #include <cpyext_typeobject.h>
 
-#define PyObject_Bytes PyObject_Str
-
 /* Flag bits for printing: */
 #define Py_PRINT_RAW    1       /* No string quotes etc. */
 

From pypy.commits at gmail.com  Sun Jan 22 14:28:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 11:28:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: A branch to do f-strings
 from PEP 498 "Literal String Interpolation".
Message-ID: <588507d6.8a281c0a.de22b.a915@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89691:29a1a30c2afd
Date: 2017-01-22 18:20 +0100
http://bitbucket.org/pypy/pypy/changeset/29a1a30c2afd/

Log:	A branch to do f-strings from PEP 498 "Literal String
	Interpolation". Even though they are a 3.6 feature, I see only
	benefits in doing it in 3.5. It can be disabled or not merged in
	py3.5 if we decide otherwise.


From pypy.commits at gmail.com  Sun Jan 22 14:28:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 11:28:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: in-progress
Message-ID: <588507d8.8aa3df0a.aa975.679b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89692:533c8eeffd5b
Date: 2017-01-22 20:26 +0100
http://bitbucket.org/pypy/pypy/changeset/533c8eeffd5b/

Log:	in-progress

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1670,6 +1670,10 @@
             return Num.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Str):
             return Str.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_FormattedValue):
+            return FormattedValue.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_JoinedStr):
+            return JoinedStr.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Bytes):
             return Bytes.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_NameConstant):
@@ -2554,6 +2558,98 @@
 State.ast_type('Str', 'expr', ['s'])
 
 
+class FormattedValue(expr):
+
+    def __init__(self, value, conversion, format_spec, lineno, col_offset):
+        self.value = value
+        self.conversion = conversion
+        self.format_spec = format_spec
+        expr.__init__(self, lineno, col_offset)
+
+    def walkabout(self, visitor):
+        visitor.visit_FormattedValue(self)
+
+    def mutate_over(self, visitor):
+        self.value = self.value.mutate_over(visitor)
+        if self.format_spec:
+            self.format_spec = self.format_spec.mutate_over(visitor)
+        return visitor.visit_FormattedValue(self)
+
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_FormattedValue)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        w_conversion = space.wrap(self.conversion)  # int
+        space.setattr(w_node, space.wrap('conversion'), w_conversion)
+        w_format_spec = self.format_spec.to_object(space) if self.format_spec is not None else space.w_None  # expr
+        space.setattr(w_node, space.wrap('format_spec'), w_format_spec)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        w_value = get_field(space, w_node, 'value', False)
+        w_conversion = get_field(space, w_node, 'conversion', True)
+        w_format_spec = get_field(space, w_node, 'format_spec', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
+        _conversion = space.int_w(w_conversion)
+        _format_spec = expr.from_object(space, w_format_spec)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
+        return FormattedValue(_value, _conversion, _format_spec, _lineno, _col_offset)
+
+State.ast_type('FormattedValue', 'expr', ['value', 'conversion', 'format_spec'])
+
+
+class JoinedStr(expr):
+
+    def __init__(self, values, lineno, col_offset):
+        self.values = values
+        expr.__init__(self, lineno, col_offset)
+
+    def walkabout(self, visitor):
+        visitor.visit_JoinedStr(self)
+
+    def mutate_over(self, visitor):
+        if self.values:
+            visitor._mutate_sequence(self.values)
+        return visitor.visit_JoinedStr(self)
+
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_JoinedStr)
+        if self.values is None:
+            values_w = []
+        else:
+            values_w = [node.to_object(space) for node in self.values] # expr
+        w_values = space.newlist(values_w)
+        space.setattr(w_node, space.wrap('values'), w_values)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        w_values = get_field(space, w_node, 'values', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        values_w = space.unpackiterable(w_values)
+        _values = [expr.from_object(space, w_item) for w_item in values_w]
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
+        return JoinedStr(_values, _lineno, _col_offset)
+
+State.ast_type('JoinedStr', 'expr', ['values'])
+
+
 class Bytes(expr):
 
     def __init__(self, s, lineno, col_offset):
@@ -3924,6 +4020,10 @@
         return self.default_visitor(node)
     def visit_Str(self, node):
         return self.default_visitor(node)
+    def visit_FormattedValue(self, node):
+        return self.default_visitor(node)
+    def visit_JoinedStr(self, node):
+        return self.default_visitor(node)
     def visit_Bytes(self, node):
         return self.default_visitor(node)
     def visit_NameConstant(self, node):
@@ -4153,6 +4253,14 @@
     def visit_Str(self, node):
         pass
 
+    def visit_FormattedValue(self, node):
+        node.value.walkabout(self)
+        if node.format_spec:
+            node.format_spec.walkabout(self)
+
+    def visit_JoinedStr(self, node):
+        self.visit_sequence(node.values)
+
     def visit_Bytes(self, node):
         pass
 
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1189,7 +1189,58 @@
             value = self.handle_expr(node.get_child(i+2))
             i += 3
         return (i,key,value)
-    
+
+    def _add_constant_string(self, joined_pieces, w_string, atom_node):
+        space = self.space
+        is_unicode = space.isinstance_w(w_string, space.w_unicode)
+        # Implement implicit string concatenation.
+        if joined_pieces:
+            prev = joined_pieces[-1]
+            if is_unicode and isinstance(prev, ast.Str):
+                w_string = space.add(prev.s, w_string)
+                del joined_pieces[-1]
+            elif not is_unicode and isinstance(prev, ast.Bytes):
+                w_string = space.add(prev.s, w_string)
+                del joined_pieces[-1]
+        node = ast.Str if is_unicode else ast.Bytes
+        joined_pieces.append(node(w_string, atom_node.get_lineno(),
+                                            atom_node.get_column()))
+
+    def _f_string_expr(self, joined_pieces, u, start, atom_node):
+        # Note: a f-string is kept as a single literal up to here.
+        # At this point only, we recursively call the AST compiler
+        # on all the '{expr}' parts.  The 'expr' part is not parsed
+        # or even tokenized together with the rest of the source code!
+        ...
+
+    def _parse_f_string(self, joined_pieces, w_string, atom_node):
+        space = self.space
+        u = space.unicode_w(w_string)
+        conversion = -1     # the conversion char.  -1 if not specified.
+        nested_depth = 0    # nesting level for braces/parens/brackets in exprs
+        start = 0
+        p1 = u.find(u'{')
+        p2 = u.find(u'}')
+        while p1 >= 0 or p2 >= 0:
+            if p1 >= 0 and (p2 < 0 or p1 < p2):
+                pn = p1 + 1
+                if pn < len(u) and u[pn] == u'{':    # '{{' => single '{'
+                    self._add_constant_string(space.newunicode(u[start:pn]))
+                    start = pn + 1
+                else:
+                    start = self._f_string_expr(joined_pieces, u, pn, atom_node)
+                p1 = u.find(u'{', start)
+            else:
+                assert p2 >= 0 and (p1 < 0 or p2 < p1)
+                pn = p2 + 1
+                if pn < len(u) and u[pn] == u'}':    # '}}' => single '}'
+                    self._add_constant_string(space.newunicode(u[start:pn]))
+                    start = pn + 1
+                else:
+                    self.error("unexpected '}' in f-string", atom_node)
+                p2 = u.find(u'}', start)
+        self._add_constant_string(space.newunicode(u[start:]))
+
     def handle_atom(self, atom_node):
         first_child = atom_node.get_child(0)
         first_child_type = first_child.type
@@ -1207,35 +1258,45 @@
                                 first_child.get_column())
             return ast.NameConstant(w_singleton, first_child.get_lineno(),
                                 first_child.get_column())
+        #
         elif first_child_type == tokens.STRING:
             space = self.space
             encoding = self.compile_info.encoding
-            try:
-                sub_strings_w = [
-                    parsestring.parsestr(
+            joined_pieces = []
+            for i in range(atom_node.num_children()):
+                try:
+                    w_next, saw_f = parsestring.parsestr(
                             space, encoding, atom_node.get_child(i).get_value())
-                        for i in range(atom_node.num_children())]
-            except error.OperationError as e:
-                if not (e.match(space, space.w_UnicodeError) or
-                        e.match(space, space.w_ValueError)):
-                    raise
-                # Unicode/ValueError in literal: turn into SyntaxError
-                self.error(e.errorstr(space), atom_node)
-                sub_strings_w = [] # please annotator
-            # Implement implicit string concatenation.
-            w_string = sub_strings_w[0]
-            for i in range(1, len(sub_strings_w)):
-                try:
-                    w_string = space.add(w_string, sub_strings_w[i])
                 except error.OperationError as e:
-                    if not e.match(space, space.w_TypeError):
+                    if not (e.match(space, space.w_UnicodeError) or
+                            e.match(space, space.w_ValueError)):
                         raise
+                    # Unicode/ValueError in literal: turn into SyntaxError
+                    raise self.error(e.errorstr(space), atom_node)
+                if not saw_f:
+                    self._add_constant_string(joined_pieces, w_next, atom_node)
+                else:
+                    self._parse_f_string(joined_pieces, w_next, atom_node)
+            if len(joined_pieces) == 1:   # <= the common path
+                return joined_pieces[0]   # ast.Str, Bytes or FormattedValue
+            # with more than one piece, it is a combination of Str and
+            # FormattedValue pieces---if there is a Bytes, then we got
+            # an invalid mixture of bytes and unicode literals
+            for node in joined_pieces:
+                if isinstance(node, ast.Bytes):
                     self.error("cannot mix bytes and nonbytes literals",
-                              atom_node)
-                # UnicodeError in literal: turn into SyntaxError
-            strdata = space.isinstance_w(w_string, space.w_unicode)
-            node = ast.Str if strdata else ast.Bytes
-            return node(w_string, atom_node.get_lineno(), atom_node.get_column())
+                               atom_node)
+            # remove empty Strs
+            values = [node for node in joined_pieces
+                           if not (isinstance(node, ast.Str) and not node.s)]
+            if len(values) > 1:
+                return ast.JoinedStr(values)
+            elif len(values) == 1:
+                return values[0]
+            else:
+                assert len(joined_pieces) > 0    # but all empty strings
+                return joined_pieces[0]
+        #
         elif first_child_type == tokens.NUMBER:
             num_value = self.parse_number(first_child.get_value())
             return ast.Num(num_value, atom_node.get_lineno(), atom_node.get_column())
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1384,3 +1384,9 @@
         code, blocks = generate_function_code(source, self.space)
         # there is a stack computation error
         assert blocks[0].instructions[3].arg == 0
+
+    def test_fstring(self):
+        source = """def f(x):
+            return f'ab{x}cd'
+        """
+        code, blocks = generate_function_code(source, self.space)
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -70,6 +70,8 @@
          | Call(expr func, expr* args, keyword* keywords)
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
+         | FormattedValue(expr value, int? conversion, expr? format_spec)
+         | JoinedStr(expr* values)
          | Bytes(bytes s)
          -- PyPy mod. first argument name must not be value
          | NameConstant(singleton single)
diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py
--- a/pypy/interpreter/pyparser/dfa_generated.py
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -23,7 +23,7 @@
      '8': 6, '9': 6, ':': 15, ';': 15,
      '<': 10, '=': 14, '>': 9, '@': 14,
      'A': 1, 'B': 2, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'E': 1, 'F': 2, 'G': 1, 'H': 1,
      'I': 1, 'J': 1, 'K': 1, 'L': 1,
      'M': 1, 'N': 1, 'O': 1, 'P': 1,
      'Q': 1, 'R': 3, 'S': 1, 'T': 1,
@@ -31,7 +31,7 @@
      'Y': 1, 'Z': 1, '[': 15, '\\': 19,
      ']': 15, '^': 14, '_': 1, '`': 15,
      'a': 1, 'b': 2, 'c': 1, 'd': 1,
-     'e': 1, 'f': 1, 'g': 1, 'h': 1,
+     'e': 1, 'f': 2, 'g': 1, 'h': 1,
      'i': 1, 'j': 1, 'k': 1, 'l': 1,
      'm': 1, 'n': 1, 'o': 1, 'p': 1,
      'q': 1, 'r': 3, 's': 1, 't': 1,
@@ -78,14 +78,14 @@
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 4, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'E': 1, 'F': 4, 'G': 1, 'H': 1,
      'I': 1, 'J': 1, 'K': 1, 'L': 1,
      'M': 1, 'N': 1, 'O': 1, 'P': 1,
      'Q': 1, 'R': 1, 'S': 1, 'T': 1,
      'U': 1, 'V': 1, 'W': 1, 'X': 1,
      'Y': 1, 'Z': 1, '_': 1, 'a': 1,
      'b': 4, 'c': 1, 'd': 1, 'e': 1,
-     'f': 1, 'g': 1, 'h': 1, 'i': 1,
+     'f': 4, 'g': 1, 'h': 1, 'i': 1,
      'j': 1, 'k': 1, 'l': 1, 'm': 1,
      'n': 1, 'o': 1, 'p': 1, 'q': 1,
      'r': 1, 's': 1, 't': 1, 'u': 1,
diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -152,9 +152,9 @@
         return group(states,
                      chain(states,
                            maybe(states, groupStr(states, "rR")),
-                           maybe(states, groupStr(states, "bB"))),
+                           maybe(states, groupStr(states, "bBfF"))),
                      chain(states,
-                           maybe(states, groupStr(states, "bB")),
+                           maybe(states, groupStr(states, "bBfF")),
                            maybe(states, groupStr(states, "rR"))),
                      maybe(states, groupStr(states, "uU")))
     # ____________________________________________________________
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -5,7 +5,8 @@
 
 
 def parsestr(space, encoding, s):
-    """Parses a string or unicode literal, and return a wrapped value.
+    """Parses a string or unicode literal, and return a pair
+    (wrapped value, f_string_flag).
 
     If encoding=None, the source string is ascii only.
     In other cases, the source string is in utf-8 encoding.
@@ -23,6 +24,7 @@
     rawmode = False
     unicode_literal = True
     saw_u = False
+    saw_f = False
 
     # string decoration handling
     if quote == 'b' or quote == 'B':
@@ -37,6 +39,10 @@
         ps += 1
         quote = s[ps]
         rawmode = True
+    elif quote == 'f' or quote == 'F':
+        ps += 1
+        quote = s[ps]
+        saw_f = True
 
     if not saw_u:
         if quote == 'r' or quote == 'R':
@@ -47,6 +53,10 @@
             ps += 1
             quote = s[ps]
             unicode_literal = False
+        elif quote == 'f' or quote == 'F':
+            ps += 1
+            quote = s[ps]
+            saw_f = True
 
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
@@ -64,6 +74,10 @@
                                         'unmatched triple quotes in literal')
         q -= 2
 
+    if saw_f:
+        # forbid any '\' inside '{' and '}' pairs
+        pass # XXX DO IT
+
     if unicode_literal and not rawmode: # XXX Py_UnicodeFlag is ignored for now
         if encoding is None:
             assert 0 <= ps <= q
@@ -71,7 +85,7 @@
         else:
             substr = decode_unicode_utf8(space, s, ps, q)
         v = unicodehelper.decode_unicode_escape(space, substr)
-        return space.wrap(v)
+        return space.wrap(v), saw_f
 
     assert 0 <= ps <= q
     substr = s[ps : q]
@@ -85,13 +99,13 @@
 
     if rawmode or '\\' not in substr:
         if not unicode_literal:
-            return space.newbytes(substr)
+            return space.newbytes(substr), saw_f
         else:
             v = unicodehelper.decode_utf8(space, substr)
-            return space.wrap(v)
+            return space.wrap(v), saw_f
 
     v = PyString_DecodeEscape(space, substr, 'strict', encoding)
-    return space.newbytes(v)
+    return space.newbytes(v), saw_f
 
 def decode_unicode_utf8(space, s, ps, q):
     # ****The Python 2.7 version, producing UTF-32 escapes****
diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py
--- a/pypy/interpreter/pyparser/pytokenize.py
+++ b/pypy/interpreter/pyparser/pytokenize.py
@@ -27,10 +27,12 @@
            'R' : None,
            "u" : None,
            "U" : None,
+           'f' : None,
+           'F' : None,
            'b' : None,
            'B' : None}
 
-for uniPrefix in ("", "b", "B"):
+for uniPrefix in ("", "b", "B", "f", "F"):
     for rawPrefix in ("", "r", "R"):
         prefix_1 = uniPrefix + rawPrefix
         prefix_2 = rawPrefix + uniPrefix
@@ -55,6 +57,11 @@
 for t in ("'''", '"""',
           "r'''", 'r"""', "R'''", 'R"""',
           "u'''", 'u"""', "U'''", 'U"""',
+          "f'''", 'f"""', "F'''", 'F"""',
+          "fr'''", 'fr"""', "Fr'''", 'Fr"""',
+          "fR'''", 'fR"""', "FR'''", 'FR"""',
+          "rf'''", 'rf"""', "rF'''", 'rF"""',
+          "Rf'''", 'Rf"""', "RF'''", 'RF"""',
           "b'''", 'b"""', "B'''", 'B"""',
           "br'''", 'br"""', "Br'''", 'Br"""',
           "bR'''", 'bR"""', "BR'''", 'BR"""',
@@ -65,6 +72,11 @@
 for t in ("'", '"',
           "r'", 'r"', "R'", 'R"',
           "u'", 'u"', "U'", 'U"',
+          "f'", 'f"', "F'", 'F"',
+          "fr'", 'fr"', "Fr'", 'Fr"',
+          "fR'", 'fR"', "FR'", 'FR"',
+          "rf'", 'rf"', "rF'", 'rF"',
+          "Rf'", 'Rf"', "RF'", 'RF"',
           "b'", 'b"', "B'", 'B"',
           "br'", 'br"', "Br'", 'Br"',
           "bR'", 'bR"', "BR'", 'BR"',

From pypy.commits at gmail.com  Sun Jan 22 15:07:36 2017
From: pypy.commits at gmail.com (gutworth)
Date: Sun, 22 Jan 2017 12:07:36 -0800 (PST)
Subject: [pypy-commit] pypy default: replicate obscure CPython package
 loading behavior
Message-ID: <58851108.ce181c0a.83309.c408@mx.google.com>

Author: Benjamin Peterson <benjamin at python.org>
Branch: 
Changeset: r89693:b289308aa50b
Date: 2017-01-22 12:04 -0800
http://bitbucket.org/pypy/pypy/changeset/b289308aa50b/

Log:	replicate obscure CPython package loading behavior

	1. When searching for an __init__ file, CPython checks for
	__init__.py and __init__.py[co] but not __init__.so or __init__.pyw.

	2. CPython considers any __init__\.py[oc]? entry it can stat
	successfully to qualify a directory as a package. This means
	__init__.py need not be a regular file and may be something weird
	like /dev/null (a character device). The behavior is less strange in
	Python 3, where __init__.py are required to be real files.

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -64,13 +64,26 @@
         space.call_method(w_stderr, "write", space.wrap(message))
 
 def file_exists(path):
-    """Tests whether the given path is an existing regular file."""
+    "Test whether the given path is an existing regular file."
     return os.path.isfile(path) and case_ok(path)
 
+def path_exists(path):
+    "Test whether the given path exists."
+    return os.path.exists(path) and case_ok(path)
+
 def has_so_extension(space):
     return (space.config.objspace.usemodules.cpyext or
             space.config.objspace.usemodules._cffi_backend)
 
+def has_init_module(space, filepart):
+    "Return True if the directory filepart qualifies as a package."
+    init = os.path.join(filepart, "__init__")
+    if path_exists(init + ".py"):
+        return True
+    if space.config.objspace.lonepycfiles and path_exists(init + ".pyc"):
+        return True
+    return False
+
 def find_modtype(space, filepart):
     """Check which kind of module to import for the given filepart,
     which is a path without extension.  Returns PY_SOURCE, PY_COMPILED or
@@ -565,9 +578,7 @@
             filepart = os.path.join(path, partname)
             log_pyverbose(space, 2, "# trying %s" % (filepart,))
             if os.path.isdir(filepart) and case_ok(filepart):
-                initfile = os.path.join(filepart, '__init__')
-                modtype, _, _ = find_modtype(space, initfile)
-                if modtype in (PY_SOURCE, PY_COMPILED):
+                if has_init_module(space, filepart):
                     return FindInfo(PKG_DIRECTORY, filepart, None)
                 else:
                     msg = ("Not importing directory '%s' missing __init__.py" %
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -108,7 +108,7 @@
     # create compiled/x.py and a corresponding pyc file
     p = setuppkg("compiled", x = "x = 84")
     if conftest.option.runappdirect:
-        import marshal, stat, struct, os, imp
+        import marshal, stat, struct, imp
         code = py.code.Source(p.join("x.py").read()).compile()
         s3 = marshal.dumps(code)
         s2 = struct.pack("<i", os.stat(str(p.join("x.py")))[stat.ST_MTIME])
@@ -138,6 +138,15 @@
         pass
     p.join('x.py').rename(p.join('x.pyw'))
 
+    if hasattr(p, "mksymlinkto"):
+        p = root.join("devnullpkg")
+        p.ensure(dir=True)
+        p.join("__init__.py").mksymlinkto(os.devnull)
+
+    p = root.join("onlypyw")
+    p.ensure(dir=True)
+    p.join("__init__.pyw")
+
     return str(root)
 
 def _load_source_module(space, w_modname, w_mod, *args, **kwds):
@@ -795,6 +804,15 @@
             reload(sys)
         assert not output
 
+    def test_dir_with_only_pyw(self):
+        def imp():
+            import onlypyw
+        raises(ImportError, imp)
+
+    @pytest.mark.skipif(not hasattr(py.path.local, "mksymlinkto"), reason="requires symlinks")
+    def test_dev_null_init_file(self):
+        import devnullpkg
+
 
 class TestAbi:
     def test_abi_tag(self):

From pypy.commits at gmail.com  Sun Jan 22 15:07:38 2017
From: pypy.commits at gmail.com (gutworth)
Date: Sun, 22 Jan 2017 12:07:38 -0800 (PST)
Subject: [pypy-commit] pypy default: fix typo
Message-ID: <5885110a.8dae1c0a.32137.b8c8@mx.google.com>

Author: Benjamin Peterson <benjamin at python.org>
Branch: 
Changeset: r89694:21d3185f73a1
Date: 2017-01-22 12:07 -0800
http://bitbucket.org/pypy/pypy/changeset/21d3185f73a1/

Log:	fix typo

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2226,7 +2226,7 @@
         #
         # 'threshold_objects_made_old', is used inside comparisons
         # with 'size_objects_made_old' to know when we must do
-        # several major GC steps (i.e. several consecurive calls
+        # several major GC steps (i.e. several consecutive calls
         # to the present function).  Here is the target that
         # we try to aim to: either (A1) or (A2)
         #

From pypy.commits at gmail.com  Sun Jan 22 15:31:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 12:31:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: in-progress: turn a simple
 f-string to AST
Message-ID: <588516ba.d30f1c0a.a91c0.bd06@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89695:4b57696884e6
Date: 2017-01-22 21:31 +0100
http://bitbucket.org/pypy/pypy/changeset/4b57696884e6/

Log:	in-progress: turn a simple f-string to AST

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -7,9 +7,9 @@
 from rpython.rlib.objectmodel import always_inline, we_are_translated
 
 
-def ast_from_node(space, node, compile_info):
+def ast_from_node(space, node, compile_info, recursive_parser=None):
     """Turn a parse tree, node, to AST."""
-    ast = ASTBuilder(space, node, compile_info).build_ast()
+    ast = ASTBuilder(space, node, compile_info, recursive_parser).build_ast()
     #
     # When we are not translated, we send this ast to validate_ast.
     # The goal is to check that validate_ast doesn't crash on valid
@@ -54,10 +54,11 @@
 
 class ASTBuilder(object):
 
-    def __init__(self, space, n, compile_info):
+    def __init__(self, space, n, compile_info, recursive_parser=None):
         self.space = space
         self.compile_info = compile_info
         self.root_node = n
+        self.recursive_parser = recursive_parser
 
     def build_ast(self):
         """Convert an top level parse tree node into an AST mod."""
@@ -1206,40 +1207,93 @@
         joined_pieces.append(node(w_string, atom_node.get_lineno(),
                                             atom_node.get_column()))
 
-    def _f_string_expr(self, joined_pieces, u, start, atom_node):
+    def _f_constant_string(self, joined_pieces, u, atom_node):
+        self._add_constant_string(joined_pieces, self.space.newunicode(u),
+                                  atom_node)
+
+    def _f_string_compile(self, source, atom_node):
         # Note: a f-string is kept as a single literal up to here.
         # At this point only, we recursively call the AST compiler
         # on all the '{expr}' parts.  The 'expr' part is not parsed
         # or even tokenized together with the rest of the source code!
-        ...
+        from pypy.interpreter.pyparser import pyparse
+
+        if self.recursive_parser is None:
+            self.error("internal error: parser not available for parsing "
+                       "the expressions inside the f-string", atom_node)
+        source = source.encode('utf-8')
+
+        info = pyparse.CompileInfo("<fstring>", "eval",
+                                   consts.PyCF_SOURCE_IS_UTF8 |
+                                   consts.PyCF_IGNORE_COOKIE,
+                                   optimize=self.compile_info.optimize)
+        parse_tree = self.recursive_parser.parse_source(source, info)
+        return ast_from_node(self.space, parse_tree, info)
+
+    def _f_string_expr(self, joined_pieces, u, start, atom_node):
+        conversion = -1     # the conversion char.  -1 if not specified.
+        nested_depth = 0    # nesting level for braces/parens/brackets in exprs
+        p = start
+        while p < len(u):
+            ch = u[p]
+            p += 1
+            if ch in u'[{(':
+                nested_depth += 1
+            elif nested_depth > 0 and ch in u']})':
+                nested_depth -= 1
+            elif nested_depth == 0 and ch in u'!:}':
+                # special-case '!='
+                if ch == u'!' and p < len(u) and u[p] == u'=':
+                    continue
+                break     # normal way out of this loop
+            # XXX forbid comment, but how?
+        else:
+            raise self.error("f-string: unterminated '{' expression")
+        if nested_depth > 0:
+            self.error("f-string: mismatched '(', '{' or '['")
+        if ch == u'!':
+            XXX
+        if ch == u':':
+            XXX
+        assert ch == u'}'
+        end_f_string = p
+        p -= 1      # drop the final '}'
+        assert p >= start
+        expr = self._f_string_compile(u[start:p], atom_node)
+        assert isinstance(expr, ast.Expression)
+        joined_pieces.append(expr.body)
+        return end_f_string
 
     def _parse_f_string(self, joined_pieces, w_string, atom_node):
         space = self.space
         u = space.unicode_w(w_string)
-        conversion = -1     # the conversion char.  -1 if not specified.
-        nested_depth = 0    # nesting level for braces/parens/brackets in exprs
         start = 0
         p1 = u.find(u'{')
-        p2 = u.find(u'}')
-        while p1 >= 0 or p2 >= 0:
-            if p1 >= 0 and (p2 < 0 or p1 < p2):
-                pn = p1 + 1
-                if pn < len(u) and u[pn] == u'{':    # '{{' => single '{'
-                    self._add_constant_string(space.newunicode(u[start:pn]))
+        while True:
+            if p1 < 0:
+                p1 = len(u)
+            p2 = u.find(u'}', start, p1)
+            if p2 >= 0:
+                pn = p2 + 1
+                if pn < len(u) and u[pn] == u'}':    # '}}' => single '}'
+                    self._f_constant_string(joined_pieces, u[start:pn],
+                                            atom_node)
                     start = pn + 1
                 else:
-                    start = self._f_string_expr(joined_pieces, u, pn, atom_node)
-                p1 = u.find(u'{', start)
+                    self.error("f-string: unexpected '}'", atom_node)
+                continue
+            if p1 == len(u):
+                self._f_constant_string(joined_pieces, u[start:], atom_node)
+                break     # no more '{' or '}' left
+            pn = p1 + 1
+            if pn < len(u) and u[pn] == u'{':    # '{{' => single '{'
+                self._f_constant_string(joined_pieces, u[start:pn], atom_node)
+                start = pn + 1
             else:
-                assert p2 >= 0 and (p1 < 0 or p2 < p1)
-                pn = p2 + 1
-                if pn < len(u) and u[pn] == u'}':    # '}}' => single '}'
-                    self._add_constant_string(space.newunicode(u[start:pn]))
-                    start = pn + 1
-                else:
-                    self.error("unexpected '}' in f-string", atom_node)
-                p2 = u.find(u'}', start)
-        self._add_constant_string(space.newunicode(u[start:]))
+                assert u[p1] == u'{'
+                start = self._f_string_expr(joined_pieces, u, pn, atom_node)
+                assert u[start - 1] == u'}'
+            p1 = u.find(u'{', start)
 
     def handle_atom(self, atom_node):
         first_child = atom_node.get_child(0)
@@ -1290,11 +1344,12 @@
             values = [node for node in joined_pieces
                            if not (isinstance(node, ast.Str) and not node.s)]
             if len(values) > 1:
-                return ast.JoinedStr(values)
+                return ast.JoinedStr(values, atom_node.get_lineno(),
+                                             atom_node.get_column())
             elif len(values) == 1:
                 return values[0]
             else:
-                assert len(joined_pieces) > 0    # but all empty strings
+                assert len(joined_pieces) > 0    # they are all empty strings
                 return joined_pieces[0]
         #
         elif first_child_type == tokens.NUMBER:
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -20,7 +20,7 @@
     p = pyparse.PythonParser(space)
     info = pyparse.CompileInfo("<test>", 'exec')
     cst = p.parse_source(expr, info)
-    ast = astbuilder.ast_from_node(space, cst, info)
+    ast = astbuilder.ast_from_node(space, cst, info, recursive_parser=p)
     function_ast = optimize.optimize_ast(space, ast.body[0], info)
     function_ast = ast.body[0]
     assert isinstance(function_ast, FunctionDef)
diff --git a/pypy/interpreter/pycompiler.py b/pypy/interpreter/pycompiler.py
--- a/pypy/interpreter/pycompiler.py
+++ b/pypy/interpreter/pycompiler.py
@@ -150,7 +150,8 @@
         space = self.space
         try:
             parse_tree = self.parser.parse_source(source, info)
-            mod = astbuilder.ast_from_node(space, parse_tree, info)
+            mod = astbuilder.ast_from_node(space, parse_tree, info,
+                                           recursive_parser=self.parser)
         except parseerror.TabError as e:
             raise OperationError(space.w_TabError,
                                  e.wrap_info(space))
diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py
--- a/pypy/module/parser/pyparser.py
+++ b/pypy/module/parser/pyparser.py
@@ -9,9 +9,10 @@
 
 
 class W_STType(W_Root):
-    def __init__(self, tree, mode):
+    def __init__(self, tree, mode, recursive_parser=None):
         self.tree = tree
         self.mode = mode
+        self.recursive_parser = recursive_parser
 
     @specialize.arg(3)
     def _build_app_tree(self, space, node, seq_maker, with_lineno, with_column):
@@ -52,7 +53,7 @@
     def descr_compile(self, space, filename="<syntax-tree>"):
         info = pyparse.CompileInfo(filename, self.mode)
         try:
-            ast = ast_from_node(space, self.tree, info)
+            ast = ast_from_node(space, self.tree, info, self.recursive_parser)
             result = compile_ast(space, ast, info)
         except error.IndentationError as e:
             raise OperationError(space.w_IndentationError,
@@ -82,7 +83,7 @@
     except error.SyntaxError as e:
         raise OperationError(space.w_SyntaxError,
                              e.wrap_info(space))
-    return space.wrap(W_STType(tree, mode))
+    return space.wrap(W_STType(tree, mode, recursive_parser=parser))
 
 
 @unwrap_spec(source=str)

From pypy.commits at gmail.com  Sun Jan 22 15:47:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 12:47:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: ast -> bytecode, minimal
Message-ID: <58851a6d.63afdf0a.3e222.82f4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89696:f23c6257063a
Date: 2017-01-22 21:47 +0100
http://bitbucket.org/pypy/pypy/changeset/f23c6257063a/

Log:	ast -> bytecode, minimal

diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -214,6 +214,9 @@
 def_op('BUILD_TUPLE_UNPACK', 152)
 def_op('BUILD_SET_UNPACK', 153)
 
+def_op('FORMAT_VALUE', 155)
+def_op('BUILD_STRING', 157)
+
 # pypy modification, experimental bytecode
 def_op('LOOKUP_METHOD', 201)          # Index in name list
 hasname.append(201)
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1232,6 +1232,7 @@
 
     def _f_string_expr(self, joined_pieces, u, start, atom_node):
         conversion = -1     # the conversion char.  -1 if not specified.
+        format_spec = None
         nested_depth = 0    # nesting level for braces/parens/brackets in exprs
         p = start
         while p < len(u):
@@ -1261,7 +1262,10 @@
         assert p >= start
         expr = self._f_string_compile(u[start:p], atom_node)
         assert isinstance(expr, ast.Expression)
-        joined_pieces.append(expr.body)
+        fval = ast.FormattedValue(expr.body, conversion, format_spec,
+                                  atom_node.get_lineno(),
+                                  atom_node.get_column())
+        joined_pieces.append(fval)
         return end_f_string
 
     def _parse_f_string(self, joined_pieces, w_string, atom_node):
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1491,6 +1491,16 @@
             sub.value.walkabout(self)
         self._compile_slice(sub.slice, sub.ctx)
 
+    def visit_JoinedStr(self, joinedstr):
+        self.update_position(joinedstr.lineno)
+        for node in joinedstr.values:
+            node.walkabout(self)
+        self.emit_op_arg(ops.BUILD_STRING, len(joinedstr.values))
+
+    def visit_FormattedValue(self, fmt):
+        fmt.value.walkabout(self)
+        self.emit_op_arg(ops.FORMAT_VALUE, 0)
+
 
 class TopLevelCodeGenerator(PythonCodeGenerator):
 
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -447,3 +447,11 @@
             node.single is not space.w_True and
             node.single is not space.w_False):
             raise ValidationError("singleton must be True, False, or None")
+
+    def visit_JoinedStr(self, node):
+        self._validate_exprs(node.values)
+
+    def visit_FormattedValue(self, node):
+        self._validate_expr(node.value)
+        if node.format_spec:
+            self._validate_expr(node.format_spec)

From pypy.commits at gmail.com  Sun Jan 22 16:48:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 13:48:40 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: bytecode interpreter
Message-ID: <588528b8.0f96df0a.7bd56.8e57@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89697:fa0095403f9d
Date: 2017-01-22 22:48 +0100
http://bitbucket.org/pypy/pypy/changeset/fa0095403f9d/

Log:	bytecode interpreter

diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -214,8 +214,8 @@
 def_op('BUILD_TUPLE_UNPACK', 152)
 def_op('BUILD_SET_UNPACK', 153)
 
-def_op('FORMAT_VALUE', 155)
-def_op('BUILD_STRING', 157)
+def_op('FORMAT_VALUE', 155)   # in CPython 3.6, but available in PyPy from 3.5
+def_op('BUILD_STRING', 157)   # in CPython 3.6, but available in PyPy from 3.5
 
 # pypy modification, experimental bytecode
 def_op('LOOKUP_METHOD', 201)          # Index in name list
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -758,6 +758,13 @@
 def _compute_CALL_METHOD(arg):
     return -_num_args(arg) - 1
 
+def _compute_FORMAT_VALUE(arg):
+    #if arg contains some flag: return -1
+    return 0
+
+def _compute_BUILD_STRING(arg):
+    return 1 - arg
+
 
 _stack_effect_computers = {}
 for name, func in globals().items():
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1286,18 +1286,18 @@
                 else:
                     self.error("f-string: unexpected '}'", atom_node)
                 continue
+            self._f_constant_string(joined_pieces, u[start:p1], atom_node)
             if p1 == len(u):
-                self._f_constant_string(joined_pieces, u[start:], atom_node)
                 break     # no more '{' or '}' left
             pn = p1 + 1
             if pn < len(u) and u[pn] == u'{':    # '{{' => single '{'
-                self._f_constant_string(joined_pieces, u[start:pn], atom_node)
-                start = pn + 1
+                start = pn
+                p1 = u.find(u'{', start + 1)
             else:
                 assert u[p1] == u'{'
                 start = self._f_string_expr(joined_pieces, u, pn, atom_node)
                 assert u[start - 1] == u'}'
-            p1 = u.find(u'{', start)
+                p1 = u.find(u'{', start)
 
     def handle_atom(self, atom_node):
         first_child = atom_node.get_child(0)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -12,7 +12,7 @@
     p = pyparse.PythonParser(space)
     info = pyparse.CompileInfo("<test>", mode)
     cst = p.parse_source(expr, info)
-    ast = astbuilder.ast_from_node(space, cst, info)
+    ast = astbuilder.ast_from_node(space, cst, info, recursive_parser=p)
     return codegen.compile_ast(space, ast, info)
 
 def generate_function_code(expr, space):
@@ -1162,6 +1162,9 @@
         """
         yield self.st, source, "f()", 43
 
+    def test_fstring(self):
+        yield self.st, """x = 42; z = f'ab{x}cd'""", 'z', 'ab42cd'
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -434,6 +434,10 @@
                 self.GET_AITER(oparg, next_instr)
             elif opcode == opcodedesc.GET_ANEXT.index:
                 self.GET_ANEXT(oparg, next_instr)
+            elif opcode == opcodedesc.FORMAT_VALUE.index:
+                self.FORMAT_VALUE(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_STRING.index:
+                self.BUILD_STRING(oparg, next_instr)
             else:
                 self.MISSING_OPCODE(oparg, next_instr)
 
@@ -1607,6 +1611,23 @@
                         "from __anext__: %T", w_next_iter)
         self.pushvalue(w_awaitable)
 
+    def FORMAT_VALUE(self, oparg, next_instr):
+        space = self.space
+        w_value = self.popvalue()
+        w_res = space.format(w_value, space.newunicode(u''))
+        self.pushvalue(w_res)
+
+    @jit.unroll_safe
+    def BUILD_STRING(self, itemcount, next_instr):
+        space = self.space
+        lst = []
+        for i in range(itemcount-1, -1, -1):
+            w_item = self.peekvalue(i)
+            lst.append(space.unicode_w(w_item))
+        self.dropvalues(itemcount)
+        w_res = space.newunicode(u''.join(lst))
+        self.pushvalue(w_res)
+
 ### ____________________________________________________________ ###
 
 class ExitFrame(Exception):

From pypy.commits at gmail.com  Sun Jan 22 17:07:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 14:07:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: conversions (!s, !r, !a)
Message-ID: <58852d30.0d1a1c0a.7ab00.fa87@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89698:19d22e69d149
Date: 2017-01-22 23:07 +0100
http://bitbucket.org/pypy/pypy/changeset/19d22e69d149/

Log:	conversions (!s, !r, !a)

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1249,18 +1249,26 @@
                 break     # normal way out of this loop
             # XXX forbid comment, but how?
         else:
-            raise self.error("f-string: unterminated '{' expression")
+            ch = u'\x00'
+        #
         if nested_depth > 0:
-            self.error("f-string: mismatched '(', '{' or '['")
+            self.error("f-string: mismatched '(', '{' or '['", atom_node)
+        end_expression = p - 1
         if ch == u'!':
-            XXX
+            if p + 1 < len(u):
+                conversion = ord(u[p])
+                ch = u[p + 1]
+                p += 2
+            if conversion not in (ord('s'), ord('r'), ord('a')):
+                self.error("f-string: invalid conversion character: "
+                           "expected 's', 'r', or 'a'", atom_node)
         if ch == u':':
             XXX
-        assert ch == u'}'
+        if ch != u'}':
+            self.error("f-string: expecting '}'", atom_node)
         end_f_string = p
-        p -= 1      # drop the final '}'
-        assert p >= start
-        expr = self._f_string_compile(u[start:p], atom_node)
+        assert end_expression >= start
+        expr = self._f_string_compile(u[start:end_expression], atom_node)
         assert isinstance(expr, ast.Expression)
         fval = ast.FormattedValue(expr.body, conversion, format_spec,
                                   atom_node.get_lineno(),
@@ -1284,7 +1292,7 @@
                                             atom_node)
                     start = pn + 1
                 else:
-                    self.error("f-string: unexpected '}'", atom_node)
+                    self.error("f-string: single '}' is not allowed", atom_node)
                 continue
             self._f_constant_string(joined_pieces, u[start:p1], atom_node)
             if p1 == len(u):
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1499,7 +1499,11 @@
 
     def visit_FormattedValue(self, fmt):
         fmt.value.walkabout(self)
-        self.emit_op_arg(ops.FORMAT_VALUE, 0)
+        arg = 0
+        if fmt.conversion == ord('s'): arg = consts.FVC_STR
+        if fmt.conversion == ord('r'): arg = consts.FVC_REPR
+        if fmt.conversion == ord('a'): arg = consts.FVC_ASCII
+        self.emit_op_arg(ops.FORMAT_VALUE, arg)
 
 
 class TopLevelCodeGenerator(PythonCodeGenerator):
diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -33,3 +33,12 @@
 PyCF_IGNORE_COOKIE = 0x0800
 PyCF_ACCEPT_NULL_BYTES = 0x10000000   # PyPy only, for compile()
 PyCF_FOUND_ENCODING = 0x20000000      # PyPy only, for pytokenizer
+
+# Masks and values used by FORMAT_VALUE opcode
+FVC_MASK      = 0x3
+FVC_NONE      = 0x0
+FVC_STR       = 0x1
+FVC_REPR      = 0x2
+FVC_ASCII     = 0x3
+FVS_MASK      = 0x4
+FVS_HAVE_SPEC = 0x4
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1164,6 +1164,15 @@
 
     def test_fstring(self):
         yield self.st, """x = 42; z = f'ab{x}cd'""", 'z', 'ab42cd'
+        yield self.st, """z = f'{{'""", 'z', '{'
+        yield self.st, """z = f'}}'""", 'z', '}'
+        yield self.st, """z = f'x{{y'""", 'z', 'x{y'
+        yield self.st, """z = f'x}}y'""", 'z', 'x}y'
+
+        yield self.st, """x = 'hi'; z = f'{x}'""", 'z', 'hi'
+        yield self.st, """x = 'hi'; z = f'{x!s}'""", 'z', 'hi'
+        yield self.st, """x = 'hi'; z = f'{x!r}'""", 'z', "'hi'"
+        yield self.st, """x = 'hi'; z = f'{x!a}'""", 'z', "'hi'"
 
 
 class AppTestCompiler:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1612,8 +1612,19 @@
         self.pushvalue(w_awaitable)
 
     def FORMAT_VALUE(self, oparg, next_instr):
+        from pypy.interpreter.astcompiler import consts
         space = self.space
         w_value = self.popvalue()
+        #
+        conversion = oparg & consts.FVC_MASK
+        if conversion == consts.FVC_STR:
+            w_value = space.str(w_value)
+        elif conversion == consts.FVC_REPR:
+            w_value = space.repr(w_value)
+        elif conversion == consts.FVC_ASCII:
+            from pypy.objspace.std.unicodeobject import ascii_from_object
+            w_value = ascii_from_object(space, w_value)
+        #
         w_res = space.format(w_value, space.newunicode(u''))
         self.pushvalue(w_res)
 

From pypy.commits at gmail.com  Sun Jan 22 17:31:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 14:31:27 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: better error checking;
 and found a more reasonable way to forbid comments than CPython uses
Message-ID: <588532bf.9aa2df0a.4ef06.9df4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89699:3cc879e93827
Date: 2017-01-22 23:30 +0100
http://bitbucket.org/pypy/pypy/changeset/3cc879e93827/

Log:	better error checking; and found a more reasonable way to forbid
	comments than CPython uses

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1218,14 +1218,22 @@
         # or even tokenized together with the rest of the source code!
         from pypy.interpreter.pyparser import pyparse
 
+        # complain if 'source' is only whitespace or an empty string
+        for c in source:
+            if c not in ' \t\n\r\v\f':
+                break
+        else:
+            self.error("f-string: empty expression not allowed", atom_node)
+
         if self.recursive_parser is None:
             self.error("internal error: parser not available for parsing "
                        "the expressions inside the f-string", atom_node)
-        source = source.encode('utf-8')
+        source = '(%s)' % source.encode('utf-8')
 
         info = pyparse.CompileInfo("<fstring>", "eval",
                                    consts.PyCF_SOURCE_IS_UTF8 |
-                                   consts.PyCF_IGNORE_COOKIE,
+                                   consts.PyCF_IGNORE_COOKIE |
+                                   consts.PyCF_REFUSE_COMMENTS,
                                    optimize=self.compile_info.optimize)
         parse_tree = self.recursive_parser.parse_source(source, info)
         return ast_from_node(self.space, parse_tree, info)
@@ -1247,7 +1255,6 @@
                 if ch == u'!' and p < len(u) and u[p] == u'=':
                     continue
                 break     # normal way out of this loop
-            # XXX forbid comment, but how?
         else:
             ch = u'\x00'
         #
diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -33,6 +33,7 @@
 PyCF_IGNORE_COOKIE = 0x0800
 PyCF_ACCEPT_NULL_BYTES = 0x10000000   # PyPy only, for compile()
 PyCF_FOUND_ENCODING = 0x20000000      # PyPy only, for pytokenizer
+PyCF_REFUSE_COMMENTS = 0x40000000     # PyPy only, for f-strings
 
 # Masks and values used by FORMAT_VALUE opcode
 FVC_MASK      = 0x3
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1174,6 +1174,15 @@
         yield self.st, """x = 'hi'; z = f'{x!r}'""", 'z', "'hi'"
         yield self.st, """x = 'hi'; z = f'{x!a}'""", 'z', "'hi'"
 
+        yield self.st, """x = 'hi'; z = f'''{\nx}'''""", 'z', 'hi'
+
+    def test_fstring_error(self):
+        raises(SyntaxError, self.run, "f'{}'")
+        raises(SyntaxError, self.run, "f'{   \t   }'")
+        raises(SyntaxError, self.run, "f'{5#}'")
+        raises(SyntaxError, self.run, "f'{5)#}'")
+        raises(SyntaxError, self.run, "f'''{5)\n#}'''")
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -187,6 +187,9 @@
                 continue
             if line[pos] == '#':
                 # skip full-line comment, but still check that it is valid utf-8
+                if flags & consts.PyCF_REFUSE_COMMENTS:
+                    raise TokenError("comments not allowed here",
+                                     line, lnum, pos, token_list)
                 if not verify_utf8(line):
                     raise bad_utf8("comment",
                                    line, lnum, pos, token_list, flags)
@@ -257,6 +260,9 @@
                     last_comment = ''
                 elif initial == '#':
                     # skip comment, but still check that it is valid utf-8
+                    if flags & consts.PyCF_REFUSE_COMMENTS:
+                        raise TokenError("comments not allowed here",
+                                         line, lnum, start, token_list)
                     if not verify_utf8(token):
                         raise bad_utf8("comment",
                                        line, lnum, start, token_list, flags)

From pypy.commits at gmail.com  Sun Jan 22 17:59:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 14:59:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: {...:format_spec}
Message-ID: <5885394e.9aa2df0a.4ef06.a3dd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89700:7b27b30dddca
Date: 2017-01-22 23:58 +0100
http://bitbucket.org/pypy/pypy/changeset/7b27b30dddca/

Log:	{...:format_spec}

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -759,7 +759,8 @@
     return -_num_args(arg) - 1
 
 def _compute_FORMAT_VALUE(arg):
-    #if arg contains some flag: return -1
+    if (arg & consts.FVS_MASK) == consts.FVS_HAVE_SPEC:
+        return -1
     return 0
 
 def _compute_BUILD_STRING(arg):
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1238,7 +1238,7 @@
         parse_tree = self.recursive_parser.parse_source(source, info)
         return ast_from_node(self.space, parse_tree, info)
 
-    def _f_string_expr(self, joined_pieces, u, start, atom_node):
+    def _f_string_expr(self, joined_pieces, u, start, atom_node, rec=0):
         conversion = -1     # the conversion char.  -1 if not specified.
         format_spec = None
         nested_depth = 0    # nesting level for braces/parens/brackets in exprs
@@ -1270,7 +1270,14 @@
                 self.error("f-string: invalid conversion character: "
                            "expected 's', 'r', or 'a'", atom_node)
         if ch == u':':
-            XXX
+            if rec >= 2:
+                self.error("f-string: expressions nested too deeply", atom_node)
+            subpieces = []
+            p = self._parse_f_string(subpieces, u, p, atom_node, rec + 1)
+            format_spec = self._f_string_to_ast_node(subpieces, atom_node)
+            ch = u[p] if p >= 0 else u'\x00'
+            p += 1
+
         if ch != u'}':
             self.error("f-string: expecting '}'", atom_node)
         end_f_string = p
@@ -1283,36 +1290,50 @@
         joined_pieces.append(fval)
         return end_f_string
 
-    def _parse_f_string(self, joined_pieces, w_string, atom_node):
+    def _parse_f_string(self, joined_pieces, u, start, atom_node, rec=0):
         space = self.space
-        u = space.unicode_w(w_string)
-        start = 0
-        p1 = u.find(u'{')
+        p1 = u.find(u'{', start)
+        prestart = start
         while True:
             if p1 < 0:
                 p1 = len(u)
             p2 = u.find(u'}', start, p1)
             if p2 >= 0:
+                self._f_constant_string(joined_pieces, u[prestart:p2],
+                                        atom_node)
                 pn = p2 + 1
                 if pn < len(u) and u[pn] == u'}':    # '}}' => single '}'
-                    self._f_constant_string(joined_pieces, u[start:pn],
-                                            atom_node)
                     start = pn + 1
-                else:
-                    self.error("f-string: single '}' is not allowed", atom_node)
-                continue
-            self._f_constant_string(joined_pieces, u[start:p1], atom_node)
+                    prestart = pn
+                    continue
+                return p2     # found a single '}', stop here
+            self._f_constant_string(joined_pieces, u[prestart:p1], atom_node)
             if p1 == len(u):
-                break     # no more '{' or '}' left
+                return -1     # no more '{' or '}' left
             pn = p1 + 1
             if pn < len(u) and u[pn] == u'{':    # '{{' => single '{'
-                start = pn
-                p1 = u.find(u'{', start + 1)
+                start = pn + 1
+                prestart = pn
             else:
                 assert u[p1] == u'{'
-                start = self._f_string_expr(joined_pieces, u, pn, atom_node)
+                start = self._f_string_expr(joined_pieces, u, pn,
+                                            atom_node, rec)
                 assert u[start - 1] == u'}'
-                p1 = u.find(u'{', start)
+                prestart = start
+            p1 = u.find(u'{', start)
+
+    def _f_string_to_ast_node(self, joined_pieces, atom_node):
+        # remove empty Strs
+        values = [node for node in joined_pieces
+                       if not (isinstance(node, ast.Str) and not node.s)]
+        if len(values) > 1:
+            return ast.JoinedStr(values, atom_node.get_lineno(),
+                                         atom_node.get_column())
+        elif len(values) == 1:
+            return values[0]
+        else:
+            assert len(joined_pieces) > 0    # they are all empty strings
+            return joined_pieces[0]
 
     def handle_atom(self, atom_node):
         first_child = atom_node.get_child(0)
@@ -1349,7 +1370,12 @@
                 if not saw_f:
                     self._add_constant_string(joined_pieces, w_next, atom_node)
                 else:
-                    self._parse_f_string(joined_pieces, w_next, atom_node)
+                    p = self._parse_f_string(joined_pieces,
+                                             space.unicode_w(w_next), 0,
+                                             atom_node)
+                    if p != -1:
+                        self.error("f-string: single '}' is not allowed",
+                                   atom_node)
             if len(joined_pieces) == 1:   # <= the common path
                 return joined_pieces[0]   # ast.Str, Bytes or FormattedValue
             # with more than one piece, it is a combination of Str and
@@ -1359,17 +1385,7 @@
                 if isinstance(node, ast.Bytes):
                     self.error("cannot mix bytes and nonbytes literals",
                                atom_node)
-            # remove empty Strs
-            values = [node for node in joined_pieces
-                           if not (isinstance(node, ast.Str) and not node.s)]
-            if len(values) > 1:
-                return ast.JoinedStr(values, atom_node.get_lineno(),
-                                             atom_node.get_column())
-            elif len(values) == 1:
-                return values[0]
-            else:
-                assert len(joined_pieces) > 0    # they are all empty strings
-                return joined_pieces[0]
+            return self._f_string_to_ast_node(joined_pieces, atom_node)
         #
         elif first_child_type == tokens.NUMBER:
             num_value = self.parse_number(first_child.get_value())
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1503,6 +1503,9 @@
         if fmt.conversion == ord('s'): arg = consts.FVC_STR
         if fmt.conversion == ord('r'): arg = consts.FVC_REPR
         if fmt.conversion == ord('a'): arg = consts.FVC_ASCII
+        if fmt.format_spec is not None:
+            arg |= consts.FVS_HAVE_SPEC
+            fmt.format_spec.walkabout(self)
         self.emit_op_arg(ops.FORMAT_VALUE, arg)
 
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1176,6 +1176,9 @@
 
         yield self.st, """x = 'hi'; z = f'''{\nx}'''""", 'z', 'hi'
 
+        yield self.st, """x = 'hi'; z = f'{x:5}'""", 'z', 'hi   '
+        yield self.st, """x = 42;   z = f'{x:5}'""", 'z', '   42'
+
     def test_fstring_error(self):
         raises(SyntaxError, self.run, "f'{}'")
         raises(SyntaxError, self.run, "f'{   \t   }'")
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1614,6 +1614,11 @@
     def FORMAT_VALUE(self, oparg, next_instr):
         from pypy.interpreter.astcompiler import consts
         space = self.space
+        #
+        if (oparg & consts.FVS_MASK) == consts.FVS_HAVE_SPEC:
+            w_spec = self.popvalue()
+        else:
+            w_spec = space.newunicode(u'')
         w_value = self.popvalue()
         #
         conversion = oparg & consts.FVC_MASK
@@ -1625,7 +1630,7 @@
             from pypy.objspace.std.unicodeobject import ascii_from_object
             w_value = ascii_from_object(space, w_value)
         #
-        w_res = space.format(w_value, space.newunicode(u''))
+        w_res = space.format(w_value, w_spec)
         self.pushvalue(w_res)
 
     @jit.unroll_safe

From pypy.commits at gmail.com  Sun Jan 22 18:08:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 22 Jan 2017 15:08:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: more tests
Message-ID: <58853b53.2ea6df0a.2d470.a44b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89701:75a8124d1219
Date: 2017-01-23 00:07 +0100
http://bitbucket.org/pypy/pypy/changeset/75a8124d1219/

Log:	more tests

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1168,6 +1168,8 @@
         yield self.st, """z = f'}}'""", 'z', '}'
         yield self.st, """z = f'x{{y'""", 'z', 'x{y'
         yield self.st, """z = f'x}}y'""", 'z', 'x}y'
+        yield self.st, """z = f'{{{4*10}}}'""", 'z', '{40}'
+        yield self.st, r"""z = fr'x={4*10}\n'""", 'z', 'x=40\\n'
 
         yield self.st, """x = 'hi'; z = f'{x}'""", 'z', 'hi'
         yield self.st, """x = 'hi'; z = f'{x!s}'""", 'z', 'hi'

From pypy.commits at gmail.com  Sun Jan 22 19:03:49 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 22 Jan 2017 16:03:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Implement PyObject_Bytes
Message-ID: <58854865.01af1c0a.67eec.d545@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89702:742da0d33a6a
Date: 2017-01-23 00:03 +0000
http://bitbucket.org/pypy/pypy/changeset/742da0d33a6a/

Log:	Implement PyObject_Bytes

diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t, slot_function,
+    PyVarObject, Py_buffer, size_t, slot_function, api_decl, cts,
     PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
@@ -11,6 +11,7 @@
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
 from pypy.objspace.std.typeobject import W_TypeObject
+from pypy.objspace.std.bytesobject import invoke_bytes_method
 from pypy.interpreter.error import OperationError, oefmt
 import pypy.module.__builtin__.operation as operation
 
@@ -247,6 +248,19 @@
         return space.wrap("<NULL>")
     return space.str(w_obj)
 
+ at api_decl("PyObject * PyObject_Bytes(PyObject *v)", cts)
+def PyObject_Bytes(space, w_obj):
+    if w_obj is None:
+        return space.newbytes("<NULL>")
+    if space.type(w_obj) is space.w_bytes:
+        return w_obj
+    w_result = invoke_bytes_method(space, w_obj)
+    if w_result is not None:
+        return w_result
+    # return PyBytes_FromObject(space, w_obj)
+    buffer = space.buffer_w(w_obj, space.BUF_FULL_RO)
+    return space.newbytes(buffer.as_str())
+
 @cpython_api([PyObject], PyObject)
 def PyObject_Repr(space, w_obj):
     """Compute a string representation of object o.  Returns the string
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -296,6 +296,20 @@
         a = module.empty_format('hello')
         assert isinstance(a, str)
 
+    def test_Bytes(self):
+        class sub1(bytes):
+            pass
+        class sub2(bytes):
+            def __bytes__(self):
+                return self
+        module = self.import_extension('test_Bytes', [
+            ('asbytes', 'METH_O',
+             """
+                return PyObject_Bytes(args);
+             """)])
+        assert type(module.asbytes(sub1(b''))) is bytes
+        assert type(module.asbytes(sub2(b''))) is sub2
+
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
     PyBuffer_FillInfo populates the fields of a Py_buffer from its arguments.

From pypy.commits at gmail.com  Sun Jan 22 20:53:33 2017
From: pypy.commits at gmail.com (sirtom67)
Date: Sun, 22 Jan 2017 17:53:33 -0800 (PST)
Subject: [pypy-commit] cffi sirtom67/float_complex: unskip the complex test
 in c/test_c.py. It still doesn't pass but gets past a couple points
Message-ID: <5885621d.05a81c0a.830fe.faa5@mx.google.com>

Author: Tom Krauss <thomas.p.krauss at gmail.com>
Branch: sirtom67/float_complex
Changeset: r2865:d3cb148ff6fa
Date: 2017-01-22 19:36 -0600
http://bitbucket.org/cffi/cffi/changeset/d3cb148ff6fa/

Log:	unskip the complex test in c/test_c.py. It still doesn't pass but
	gets past a couple points

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -899,6 +899,25 @@
     _write_raw_data(long double);
 }
 
+#define _write_raw_complex_data(type)                      \
+    do {                                                   \
+        if (size == 2*sizeof(type)) {                      \
+            type r = (type)source.real;                    \
+            memcpy(target, &r, sizeof(type));              \
+            type i = (type)source.imag;                    \
+            memcpy(target+sizeof(type), &i, sizeof(type)); \
+            return;                                        \
+        }                                                  \
+    } while(0)
+
+static void
+write_raw_complex_data(char *target, Py_complex source, int size)
+{
+    _write_raw_complex_data(float);
+    _write_raw_complex_data(double);
+    Py_FatalError("write_raw_complex_data: bad float size");
+}
+
 static PyObject *
 new_simple_cdata(char *data, CTypeDescrObject *ct)
 {
@@ -3574,6 +3593,40 @@
         }
         return (PyObject *)cd;
     }
+
+
+    else if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        /* cast to a complex */
+        Py_complex value;
+        PyObject *io;
+        printf("_cffi_backend.c do_cast  ct->ct_size=%ld\n", ct->ct_size);
+        if (CData_Check(ob)) {
+            CDataObject *cdsrc = (CDataObject *)ob;
+
+            if (!(cdsrc->c_type->ct_flags & CT_PRIMITIVE_ANY))
+                goto cannot_cast;
+            io = convert_to_object(cdsrc->c_data, cdsrc->c_type);
+            if (io == NULL)
+                return NULL;
+        }
+        else {
+            io = ob;
+            Py_INCREF(io);
+        }
+
+        value = PyComplex_AsCComplex(io);
+        Py_DECREF(io);
+        if (value.real == -1.0 && value.imag == 0.0 && PyErr_Occurred())
+            return NULL;
+
+        cd = _new_casted_primitive(ct);
+        if (cd != NULL) {
+            write_raw_complex_data(cd->c_data, value, ct->ct_size);
+        }
+        return (PyObject *)cd;
+    }
+
+
     else {
         PyErr_Format(PyExc_TypeError, "cannot cast to ctype '%s'",
                      ct->ct_name);
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -183,10 +183,9 @@
         py.test.raises(TypeError, cast, p, None)
 
 def test_complex_types():
-    py.test.skip("later")
     INF = 1E200 * 1E200
-    for name in ["float", "double"]:
-        p = new_primitive_type("_Complex " + name)
+    for name in ["float"]:  #, "double"]:
+        p = new_primitive_type(name + " _Complex")
         assert bool(cast(p, 0))
         assert bool(cast(p, INF))
         assert bool(cast(p, -INF))

From pypy.commits at gmail.com  Mon Jan 23 04:13:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jan 2017 01:13:09 -0800 (PST)
Subject: [pypy-commit] cffi default: Change the Windows-friendly version to
 be both Windows- and
Message-ID: <5885c925.b296df0a.ebe67.3df0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2866:a06c29d41c30
Date: 2017-01-23 10:12 +0100
http://bitbucket.org/cffi/cffi/changeset/a06c29d41c30/

Log:	Change the Windows-friendly version to be both Windows- and POSIX-
	friendly (previously it *only* compiled on Windows)

diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst
--- a/doc/source/embedding.rst
+++ b/doc/source/embedding.rst
@@ -52,13 +52,25 @@
     /* file plugin.h, Windows-friendly version */
     typedef struct { int x, y; } point_t;
 
-    /* When including this file from ffibuilder.set_source(),
-       this macro is defined to __declspec(dllexport).  When
-       including this file directly from your C program, we
-       define it to __declspec(dllimport) instead. */
+    /* When including this file from ffibuilder.set_source(), the
+       following macro is defined to '__declspec(dllexport)'.  When
+       including this file directly from your C program, we define
+       it to 'extern __declspec(dllimport)' instead.
+
+       With non-MSVC compilers we simply define it to 'extern'.
+       (The 'extern' is needed for sharing global variables;
+       functions would be fine without it.  The macros always
+       include 'extern': you must not repeat it when using the
+       macros later.)
+    */
     #ifndef CFFI_DLLEXPORT
-    #  define CFFI_DLLEXPORT __declspec(dllimport)
+    #  if defined(_MSC_VER)
+    #    define CFFI_DLLEXPORT  extern __declspec(dllimport)
+    #  else
+    #    define CFFI_DLLEXPORT  extern
+    #  endif
     #endif
+
     CFFI_DLLEXPORT int do_stuff(point_t *);
 
 .. code-block:: python

From pypy.commits at gmail.com  Mon Jan 23 05:01:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jan 2017 02:01:52 -0800 (PST)
Subject: [pypy-commit] pypy default: update README
Message-ID: <5885d490.d18bdf0a.b3653.52ca@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89703:a231c4776760
Date: 2017-01-23 11:01 +0100
http://bitbucket.org/pypy/pypy/changeset/a231c4776760/

Log:	update README

diff --git a/rpython/rlib/rvmprof/README.txt b/rpython/rlib/rvmprof/README.txt
--- a/rpython/rlib/rvmprof/README.txt
+++ b/rpython/rlib/rvmprof/README.txt
@@ -2,8 +2,6 @@
 VMProf: a profiler for RPython VMs
 ==================================
 
-**Limited to 64-bit Linux.**
-
 
 from rpython.rlib import rvmprof
 
@@ -11,85 +9,36 @@
 Your VM must have an interpreter for "code" objects, which can be
 of any RPython instance.
 
-Use this decorator on the mainloop of the interpreter, to tell vmprof
+Use this as a decorator on the mainloop of the interpreter, to tell vmprof
 when you enter and leave a "code" object:
 
     def vmprof_execute_code(name, get_code_fn, result_class=None):
-        """Decorator to be used on the function that interprets a code object.
 
-        'name' must be a unique name.
-
-        'get_code_fn(*args)' is called to extract the code object from the
-        arguments given to the decorated function.
-
-        The original function can return None, an integer, or an instance.
-        In the latter case (only), 'result_class' must be set.
-
-        NOTE: for now, this assumes that the decorated functions only takes
-        instances or plain integer arguments, and at most 5 of them
-        (including 'self' if applicable).
-        """
+See its docstring in rvmprof.py.
 
 
 The class of code objects needs to be registered by calling the
-following function (once, before translation):
+function ``rpython.rlib.rvmprof.register_code_object_class()``
+(once, before translation).  It is a global function in __init__.py,
+but see the docstring of the method in rvmprof.py.
 
-    def register_code_object_class(CodeClass, full_name_func):
-        """NOT_RPYTHON
-        Register statically the class 'CodeClass' as containing user
-        code objects.
 
-        full_name_func() is a function called at runtime with an
-        instance of CodeClass and it should return a string.  This
-        is the string stored in the vmprof file identifying the code
-        object.  It can be directly an unbound method of CodeClass.
-        IMPORTANT: the name returned must be at most MAX_FUNC_NAME
-        characters long, and with exactly 3 colons, i.e. of the form
+To support adding new code objects at run-time, whenever a code object is
+instantiated, call the function ``rpython.rlib.rvmprof.register_code()``.
 
-            class:func_name:func_line:filename
+If you need JIT support, you also need to add a jitdriver method
+``get_unique_id(*greenkey)``, where you call
+``rpython.rlib.rvmprof.get_unique_code()``.
 
-        where 'class' is 'py' for PyPy.
 
-        Instances of the CodeClass will have a new attribute called
-        '_vmprof_unique_id', but that's managed internally.
-        """
-
-
-To support adding new code objects at run-time, whenever a code
-object is instantiated, call this function:
-
-    @specialize.argtype(1)
-    def register_code(code, name):
-        """Register the code object.  Call when a new code object is made.
-        """
-
-    @specialize.argtype(1)
-    def get_unique_id(code):
-        """Return the internal unique ID of a code object.  Can only be
-        called after register_code().  Call this in the jitdriver's
-        method 'get_unique_id(*greenkey)'.  Always returns 0 if we
-        didn't call register_code_object_class() on the class.
-        """
-
-
-Enable the profiler with:
+Enable/disable the profiler at runtime with:
 
     def enable(fileno, interval):
-        """Enable vmprof.  Writes go to the given 'fileno'.
-        The sampling interval is given by 'interval' as a number of
-        seconds, as a float which must be smaller than 1.0.
-        Raises VMProfError if something goes wrong.
-        """
-
+    def disable():
 
 The file descriptor must remain open until the profiler is disabled.
 The profiler must be disabled before the program exit, otherwise the
-file is incompletely written.  Disable it with:
+file is incompletely written.
 
-    def disable():
-        """Disable vmprof.
-        Raises VMProfError if something goes wrong.
-        """
-
-You should close the file descriptor afterwards; it is not
-automatically closed.
+You should close the file descriptor after disabling the profiler; it is
+not automatically closed.

From pypy.commits at gmail.com  Mon Jan 23 05:47:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jan 2017 02:47:56 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix: allow get_unique_id=... on a
 jitdriver whose 'is_recursive' status
Message-ID: <5885df5c.28a8df0a.8397e.e60f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89704:142f2ad47039
Date: 2017-01-23 11:42 +0100
http://bitbucket.org/pypy/pypy/changeset/142f2ad47039/

Log:	Fix: allow get_unique_id=... on a jitdriver whose 'is_recursive'
	status will be set to True later (when it is the only jitdriver in
	the program).

diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -52,12 +52,19 @@
         jd.jitdriver.is_recursive = True
     else:
         count_recursive = 0
+        invalid = 0
         for jd in warmrunnerdesc.jitdrivers_sd:
             count_recursive += jd.jitdriver.is_recursive
+            invalid += (jd.jitdriver.has_unique_id and
+                           not jd.jitdriver.is_recursive)
         if count_recursive == 0:
             raise Exception("if you have more than one jitdriver, at least"
                 " one of them has to be marked with is_recursive=True,"
                 " none found")
+        if invalid > 0:
+            raise Exception("found %d jitdriver(s) with 'get_unique_id=...' "
+                            "specified but without 'is_recursive=True'" %
+                            (invalid,))
     for jd in warmrunnerdesc.jitdrivers_sd:
         jd.warmstate.set_param_inlining(inline)
         jd.warmstate.set_param_vec(vec)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -637,8 +637,6 @@
             raise AttributeError("no 'greens' or 'reds' supplied")
         if virtualizables is not None:
             self.virtualizables = virtualizables
-        if get_unique_id is not None:
-            assert is_recursive, "get_unique_id and is_recursive must be specified at the same time"
         for v in self.virtualizables:
             assert v in self.reds
         # if reds are automatic, they won't be passed to jit_merge_point, so
@@ -653,6 +651,7 @@
         assert set_jitcell_at is None, "set_jitcell_at no longer used"
         self.get_printable_location = get_printable_location
         self.get_location = get_location
+        self.has_unique_id = (get_unique_id is not None)
         if get_unique_id is None:
             get_unique_id = lambda *args: 0
         self.get_unique_id = get_unique_id

From pypy.commits at gmail.com  Mon Jan 23 12:11:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jan 2017 09:11:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: See
 https://hg.python.org/cpython/rev/9791c5d55f52: it's just another
Message-ID: <5886395d.f0a6df0a.45aae.5378@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89705:7ade09ee3ac5
Date: 2017-01-23 18:11 +0100
http://bitbucket.org/pypy/pypy/changeset/7ade09ee3ac5/

Log:	See https://hg.python.org/cpython/rev/9791c5d55f52: it's just
	another similar case

diff --git a/lib-python/3/test/test_asyncio/test_events.py b/lib-python/3/test/test_asyncio/test_events.py
--- a/lib-python/3/test/test_asyncio/test_events.py
+++ b/lib-python/3/test/test_asyncio/test_events.py
@@ -825,9 +825,15 @@
         server = self.loop.run_until_complete(f)
         self.assertEqual(len(server.sockets), 1)
         sock = server.sockets[0]
-        self.assertFalse(
-            sock.getsockopt(
-                socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        try:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        except OSError:
+            raise unittest.SkipTest(
+                "Python's socket module was compiled using modern headers "
+                "thus defining SO_REUSEPORT but this process is running "
+                "under an older kernel that does not support SO_REUSEPORT.")
         server.close()
 
         test_utils.run_briefly(self.loop)

From pypy.commits at gmail.com  Mon Jan 23 12:40:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jan 2017 09:40:25 -0800 (PST)
Subject: [pypy-commit] pypy default: I spent the last 2 hours trying to make
 sense out of a
Message-ID: <58864009.b6a9df0a.dabfc.10fd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89706:2bf0191fb21d
Date: 2017-01-23 18:39 +0100
http://bitbucket.org/pypy/pypy/changeset/2bf0191fb21d/

Log:	I spent the last 2 hours trying to make sense out of a "jit-log-
	noopt-loop" section before realizing it was really a bridge

diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -16,7 +16,11 @@
         if not have_debug_prints():
             return
         inputargs, ops = self._unpack_trace(trace)
-        self.log_loop(inputargs, ops, memo=memo)
+        debug_start("jit-log-noopt")
+        debug_print("# Traced loop or bridge with", len(ops), "ops")
+        logops = self._log_operations(inputargs, ops, None, memo)
+        debug_stop("jit-log-noopt")
+        return logops
 
     def _unpack_trace(self, trace):
         ops = []
@@ -28,6 +32,7 @@
     def log_loop(self, inputargs, operations, number=0, type=None,
                  ops_offset=None, name='', memo=None):
         if type is None:
+            # XXX this case not normally used any more, I think
             debug_start("jit-log-noopt-loop")
             debug_print("# Loop", number, '(%s)' % name, ":", "noopt",
                         "with", len(operations), "ops")
@@ -58,6 +63,7 @@
     def log_bridge(self, inputargs, operations, extra=None,
                    descr=None, ops_offset=None, memo=None):
         if extra == "noopt":
+            # XXX this case no longer used
             debug_start("jit-log-noopt-bridge")
             debug_print("# bridge out of Guard",
                         "0x%x" % compute_unique_id(descr),

From pypy.commits at gmail.com  Mon Jan 23 14:22:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jan 2017 11:22:58 -0800 (PST)
Subject: [pypy-commit] cffi default: bump version number to 1.10.0
Message-ID: <58865812.4d821c0a.cb48e.a4b1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2867:4fcdf992d623
Date: 2017-01-23 10:19 +0100
http://bitbucket.org/cffi/cffi/changeset/4fcdf992d623/

Log:	bump version number to 1.10.0

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.9.2"
+#define CFFI_VERSION  "1.10.0"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,9 +12,9 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.9.2", ("This test_c.py file is for testing a version"
-                                " of cffi that differs from the one that we"
-                                " get from 'import _cffi_backend'")
+assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+                                 " of cffi that differs from the one that we"
+                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
     type_or_class = "type"
     mandatory_b_prefix = ''
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.9.2"
-__version_info__ = (1, 9, 2)
+__version__ = "1.10.0"
+__version_info__ = (1, 10, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.9.2"
+                               "\ncompiled with cffi version: 1.10.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -45,9 +45,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.9'
+version = '1.10'
 # The full version, including alpha/beta/rc tags.
-release = '1.9.2'
+release = '1.10.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,7 +51,7 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.9.2.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.10.0.tar.gz
 
    - MD5: ...
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -183,7 +183,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.9.2',
+        version='1.10.0',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}
diff --git a/testing/cffi0/test_version.py b/testing/cffi0/test_version.py
--- a/testing/cffi0/test_version.py
+++ b/testing/cffi0/test_version.py
@@ -28,7 +28,7 @@
     content = open(p).read()
     #
     v = cffi.__version__
-    assert ("version = '%s'\n" % v[:3]) in content
+    assert ("version = '%s'\n" % v[:4]) in content
     assert ("release = '%s'\n" % v) in content
 
 def test_doc_version_file():

From pypy.commits at gmail.com  Mon Jan 23 14:23:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jan 2017 11:23:00 -0800 (PST)
Subject: [pypy-commit] cffi default: Version fix
Message-ID: <58865814.6f98df0a.d4ac7.3f13@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2868:b0204d9d8b96
Date: 2017-01-23 15:37 +0100
http://bitbucket.org/cffi/cffi/changeset/b0204d9d8b96/

Log:	Version fix

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3744,7 +3744,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))

From pypy.commits at gmail.com  Mon Jan 23 14:23:02 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jan 2017 11:23:02 -0800 (PST)
Subject: [pypy-commit] cffi default: issue300: return _Bool as Python
 booleans; related fixes
Message-ID: <58865816.07addf0a.2e292.36a6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2869:d05a37bd9b1f
Date: 2017-01-23 20:22 +0100
http://bitbucket.org/cffi/cffi/changeset/d05a37bd9b1f/

Log:	issue300: return _Bool as Python booleans; related fixes

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -970,8 +970,23 @@
         /*READ(data, ct->ct_size)*/
         value = read_raw_unsigned_data(data, ct->ct_size);
 
-        if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG)
+        if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) {
+            if (ct->ct_flags & CT_IS_BOOL) {
+                PyObject *x;
+                switch ((int)value) {
+                case 0: x = Py_False; break;
+                case 1: x = Py_True; break;
+                default:
+                    PyErr_Format(PyExc_ValueError,
+                                 "got a _Bool of value %d, expected 0 or 1",
+                                 (int)value);
+                    return NULL;
+                }
+                Py_INCREF(x);
+                return x;
+            }
             return PyInt_FromLong((long)value);
+        }
         else
             return PyLong_FromUnsignedLongLong(value);
     }
@@ -1208,6 +1223,20 @@
 }
 
 static int
+must_be_array_of_zero_or_one(const char *data, Py_ssize_t n)
+{
+    Py_ssize_t i;
+    for (i = 0; i < n; i++) {
+        if (((unsigned char)data[i]) > 1) {
+            PyErr_SetString(PyExc_ValueError,
+                "an array of _Bool can only contain \\x00 or \\x01");
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static int
 convert_array_from_object(char *data, CTypeDescrObject *ct, PyObject *init)
 {
     /* used by convert_from_object(), and also to decode lists/tuples/unicodes
@@ -1254,6 +1283,9 @@
             if (n != ct->ct_length)
                 n++;
             srcdata = PyBytes_AS_STRING(init);
+            if (ctitem->ct_flags & CT_IS_BOOL)
+                if (must_be_array_of_zero_or_one(srcdata, n) < 0)
+                    return -1;
             memcpy(data, srcdata, n);
             return 0;
         }
@@ -1424,12 +1456,15 @@
         unsigned PY_LONG_LONG value = _my_PyLong_AsUnsignedLongLong(init, 1);
         if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())
             return -1;
-        if (ct->ct_flags & CT_IS_BOOL)
-            if (value & ~1)      /* value != 0 && value != 1 */
+        if (ct->ct_flags & CT_IS_BOOL) {
+            if (value > 1ULL)      /* value != 0 && value != 1 */
                 goto overflow;
-        write_raw_integer_data(buf, value, ct->ct_size);
-        if (value != read_raw_unsigned_data(buf, ct->ct_size))
-            goto overflow;
+        }
+        else {
+            write_raw_integer_data(buf, value, ct->ct_size);
+            if (value != read_raw_unsigned_data(buf, ct->ct_size))
+                goto overflow;
+        }
         write_raw_integer_data(data, value, ct->ct_size);
         return 0;
     }
@@ -2544,6 +2579,10 @@
             length = PyBytes_GET_SIZE(init) + 1;
 #else
             *output_data = PyBytes_AS_STRING(init);
+            if (ctitem->ct_flags & CT_IS_BOOL)
+                if (must_be_array_of_zero_or_one(*output_data,
+                                                 PyBytes_GET_SIZE(init)) < 0)
+                    return -1;
             return 0;
 #endif
         }
@@ -5740,7 +5779,8 @@
     if (cd->c_type->ct_itemdescr != NULL &&
         cd->c_type->ct_itemdescr->ct_flags & (CT_PRIMITIVE_CHAR |
                                               CT_PRIMITIVE_SIGNED |
-                                              CT_PRIMITIVE_UNSIGNED)) {
+                                              CT_PRIMITIVE_UNSIGNED) &&
+        !(cd->c_type->ct_itemdescr->ct_flags & CT_IS_BOOL)) {
         Py_ssize_t length = maxlen;
         if (cd->c_data == NULL) {
             PyObject *s = cdata_repr(cd);
@@ -5904,7 +5944,8 @@
             else if (itemsize == sizeof(short))       casenum = 1;
             else if (itemsize == sizeof(signed char)) casenum = 0;
         }
-        else if (ctitem->ct_flags & CT_PRIMITIVE_UNSIGNED) {
+        else if ((ctitem->ct_flags & (CT_PRIMITIVE_UNSIGNED | CT_IS_BOOL))
+                 == CT_PRIMITIVE_UNSIGNED) {
             /* Note: we never pick case 6 if sizeof(int) == sizeof(long),
                so that case 6 below can assume that the 'unsigned int' result
                would always fit in a 'signed long'. */
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -896,6 +896,15 @@
     py.test.raises(OverflowError, f, 128, 0)
     py.test.raises(OverflowError, f, 0, 128)
 
+def test_call_function_0_pretend_bool_result():
+    BSignedChar = new_primitive_type("signed char")
+    BBool = new_primitive_type("_Bool")
+    BFunc0 = new_function_type((BSignedChar, BSignedChar), BBool, False)
+    f = cast(BFunc0, _testfunc(0))
+    assert f(40, -39) is True
+    assert f(40, -40) is False
+    py.test.raises(ValueError, f, 40, 2)
+
 def test_call_function_1():
     BInt = new_primitive_type("int")
     BLong = new_primitive_type("long")
@@ -1058,6 +1067,17 @@
     res = f(b"foo")
     assert res == 1000 * ord(b'f')
 
+def test_call_function_23_bool_array():
+    # declaring the function as int(_Bool*)
+    BBool = new_primitive_type("_Bool")
+    BBoolP = new_pointer_type(BBool)
+    BInt = new_primitive_type("int")
+    BFunc23 = new_function_type((BBoolP,), BInt, False)
+    f = cast(BFunc23, _testfunc(23))
+    res = f(b"\x01\x01")
+    assert res == 1000
+    py.test.raises(ValueError, f, b"\x02\x02")
+
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
@@ -2628,13 +2648,38 @@
     py.test.raises(OverflowError, newp, BBoolP, 2)
     py.test.raises(OverflowError, newp, BBoolP, -1)
     BCharP = new_pointer_type(new_primitive_type("char"))
-    p = newp(BCharP, b'X')
+    p = newp(BCharP, b'\x01')
     q = cast(BBoolP, p)
-    assert q[0] == ord(b'X')
+    assert q[0] is True
+    p = newp(BCharP, b'\x00')
+    q = cast(BBoolP, p)
+    assert q[0] is False
     py.test.raises(TypeError, string, cast(BBool, False))
     BDouble = new_primitive_type("double")
     assert int(cast(BBool, cast(BDouble, 0.1))) == 1
     assert int(cast(BBool, cast(BDouble, 0.0))) == 0
+    BBoolA = new_array_type(BBoolP, None)
+    p = newp(BBoolA, b'\x01\x00')
+    assert p[0] is True
+    assert p[1] is False
+
+def test_bool_forbidden_cases():
+    BBool = new_primitive_type("_Bool")
+    BBoolP = new_pointer_type(BBool)
+    BBoolA = new_array_type(BBoolP, None)
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    p = newp(BCharP, b'X')
+    q = cast(BBoolP, p)
+    py.test.raises(ValueError, "q[0]")
+    py.test.raises(TypeError, newp, BBoolP, b'\x00')
+    assert newp(BBoolP, 0)[0] is False
+    assert newp(BBoolP, 1)[0] is True
+    py.test.raises(OverflowError, newp, BBoolP, 2)
+    py.test.raises(OverflowError, newp, BBoolP, -1)
+    py.test.raises(ValueError, newp, BBoolA, b'\x00\x01\x02')
+    py.test.raises(OverflowError, newp, BBoolA, [0, 1, 2])
+    py.test.raises(TypeError, string, newp(BBoolP, 1))
+    py.test.raises(TypeError, string, newp(BBoolA, [1]))
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
@@ -3674,7 +3719,7 @@
             ("int16_t", [-2**15, 2**15-1]),
             ("int32_t", [-2**31, 2**31-1]),
             ("int64_t", [-2**63, 2**63-1]),
-            ("_Bool", [0, 1]),
+            ("_Bool", [False, True]),
             ("float", [0.0, 10.5]),
             ("double", [12.34, 56.78]),
             ]:
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -597,7 +597,8 @@
 |   integers    | an integer or anything | a Python int or  | int(), bool()  |
 |   and enums   | on which int() works   | long, depending  | `(******)`     |
 |   `(*****)`   | (but not a float!).    | on the type      |                |
-|               | Must be within range.  |                  |                |
+|               | Must be within range.  | (ver. 1.10: or a |                |
+|               |                        | bool)            |                |
 +---------------+------------------------+------------------+----------------+
 |   ``char``    | a string of length 1   | a string of      | int(), bool()  |
 |               | or another <cdata char>| length 1         |                |
@@ -637,12 +638,13 @@
 |               | items                  |                  |``[]`` `(****)`,|
 |               |                        |                  |``+``, ``-``    |
 +---------------+------------------------+                  +----------------+
-|  ``char[]``   | same as arrays, or a   |                  | len(), iter(), |
-|               | Python string          |                  | ``[]``, ``+``, |
-|               |                        |                  | ``-``          |
+| ``char[]``,   | same as arrays, or a   |                  | len(), iter(), |
+| ``un/signed`` | Python byte string     |                  | ``[]``, ``+``, |
+| ``char[]``,   |                        |                  | ``-``          |
+| ``_Bool[]``   |                        |                  |                |
 +---------------+------------------------+                  +----------------+
 | ``wchar_t[]`` | same as arrays, or a   |                  | len(), iter(), |
-|               | Python unicode         |                  | ``[]``,        |
+|               | Python unicode string  |                  | ``[]``,        |
 |               |                        |                  | ``+``, ``-``   |
 |               |                        |                  |                |
 +---------------+------------------------+------------------+----------------+
@@ -727,6 +729,14 @@
    *New in version 1.7.*  In previous versions, it only worked on
    pointers; for primitives it always returned True.
 
+   *New in version 1.10:*  The C type ``_Bool`` or ``bool`` converts to
+   Python booleans now.  You get an exception if a C ``_Bool`` happens
+   to contain a value different from 0 and 1 (this case triggers
+   undefined behavior in C; if you really have to interface with a
+   library relying on this, don't use ``_Bool`` in the CFFI side).
+   Also, when converting from a byte string to a ``_Bool[]``, only the
+   bytes ``\x00`` and ``\x01`` are accepted.
+
 .. _file:
 
 Support for FILE
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -31,6 +31,19 @@
   been fixed and the unicode strings don't support the memoryview
   interface any more.)
 
+* The C type ``_Bool`` or ``bool`` now converts to a Python boolean
+  when reading, instead of the content of the byte as an integer.  The
+  change here is mostly what occurs if the byte happens to contain a
+  value different from 0 and 1.  Previously, it would just return it;
+  with this change, CFFI raises an exception in this case.  But this
+  case means "undefined behavior" in C; if you really have to interface
+  with a library relying on this, don't use ``_Bool`` in the CFFI side.
+  Also, it is still valid to use a byte string as initializer for a
+  ``_Bool[]``, but now it must only contain ``\x00`` or ``\x01``.  As an
+  aside, ``ffi.string()`` no longer works on ``_Bool[]`` (but it never
+  made much sense, as this function stops on the first zero).
+
+
 v1.9
 ====
 

From pypy.commits at gmail.com  Mon Jan 23 14:33:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jan 2017 11:33:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: an extra case
Message-ID: <58865a7b.1282df0a.6360e.3d6b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89707:f4cdc59d88b9
Date: 2017-01-23 20:32 +0100
http://bitbucket.org/pypy/pypy/changeset/f4cdc59d88b9/

Log:	an extra case

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1180,6 +1180,7 @@
 
         yield self.st, """x = 'hi'; z = f'{x:5}'""", 'z', 'hi   '
         yield self.st, """x = 42;   z = f'{x:5}'""", 'z', '   42'
+        yield self.st, """x = 2; z = f'{5:{x:+1}0}'""", 'z', (' ' * 18 + '+5')
 
     def test_fstring_error(self):
         raises(SyntaxError, self.run, "f'{}'")

From pypy.commits at gmail.com  Mon Jan 23 14:55:47 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 23 Jan 2017 11:55:47 -0800 (PST)
Subject: [pypy-commit] cffi default: tweak
Message-ID: <58865fc3.0c8e1c0a.eb77f.03c5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r2870:7c43a8dc8393
Date: 2017-01-23 21:53 +0200
http://bitbucket.org/cffi/cffi/changeset/7c43a8dc8393/

Log:	tweak

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -21,8 +21,9 @@
 To run tests under CPython, run::
 
     pip install pytest     # if you don't have py.test already
+    pip install pycparser
     python setup.py build_ext -f -i
-    py.test c/ test/
+    py.test c/ testing/
 
 If you run in another directory (either the tests or another program),
 you should use the environment variable ``PYTHONPATH=/path`` to point

From pypy.commits at gmail.com  Mon Jan 23 15:17:10 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 23 Jan 2017 12:17:10 -0800 (PST)
Subject: [pypy-commit] pypy default: Make astcompiler code more
 annotator-friendly
Message-ID: <588664c6.07addf0a.2e292.4a1a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89708:584f6da010f3
Date: 2017-01-23 19:11 +0000
http://bitbucket.org/pypy/pypy/changeset/584f6da010f3/

Log:	Make astcompiler code more annotator-friendly

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1,6 +1,8 @@
 # Generated by tools/asdl_py.py
 from rpython.tool.pairtype import extendabletype
 from rpython.tool.sourcetools import func_with_new_name
+from rpython.rlib.signature import signature, finishsigs
+from rpython.rlib import types
 
 from pypy.interpreter import typedef
 from pypy.interpreter.baseobjspace import W_Root
@@ -32,6 +34,7 @@
 
 class AST(object):
     __metaclass__ = extendabletype
+    _attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")
@@ -139,7 +142,7 @@
         self.w_AST = space.gettypeobject(W_AST.typedef)
         for (name, base, fields, attributes) in self.AST_TYPES:
             self.make_new_type(space, name, base, fields, attributes)
-        
+
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
@@ -151,7 +154,7 @@
             space.setitem_str(w_dict, "_attributes",
                               space.newtuple([space.wrap(a) for a in attributes]))
         w_type = space.call_function(
-            space.w_type, 
+            space.w_type,
             space.wrap(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
@@ -1883,8 +1886,8 @@
 
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
-        if self.generators:
-            visitor._mutate_sequence(self.generators)
+        for i in range(len(self.generators)):
+            self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_ListComp(self)
 
     def to_object(self, space):
@@ -1931,8 +1934,8 @@
 
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
-        if self.generators:
-            visitor._mutate_sequence(self.generators)
+        for i in range(len(self.generators)):
+            self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_SetComp(self)
 
     def to_object(self, space):
@@ -1981,8 +1984,8 @@
     def mutate_over(self, visitor):
         self.key = self.key.mutate_over(visitor)
         self.value = self.value.mutate_over(visitor)
-        if self.generators:
-            visitor._mutate_sequence(self.generators)
+        for i in range(len(self.generators)):
+            self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_DictComp(self)
 
     def to_object(self, space):
@@ -2033,8 +2036,8 @@
 
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
-        if self.generators:
-            visitor._mutate_sequence(self.generators)
+        for i in range(len(self.generators)):
+            self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_GeneratorExp(self)
 
     def to_object(self, space):
@@ -3106,6 +3109,7 @@
     _NotIn,
 ]
 
+ at finishsigs
 class comprehension(AST):
 
     def __init__(self, target, iter, ifs):
@@ -3113,6 +3117,7 @@
         self.iter = iter
         self.ifs = ifs
 
+    @signature(types.self(), types.any(), returns=types.self())
     def mutate_over(self, visitor):
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -7,6 +7,7 @@
 # you figure out a way to remove them, great, but try a translation first,
 # please.
 
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.astcompiler import ast, assemble, symtable, consts, misc
 from pypy.interpreter.astcompiler import optimize # For side effects
 from pypy.interpreter.pyparser.error import SyntaxError
@@ -1027,11 +1028,17 @@
 
     def visit_ListComp(self, lc):
         self.update_position(lc.lineno)
-        if len(lc.generators) != 1 or lc.generators[0].ifs:
+        if len(lc.generators) == 1:
+            comp = lc.generators[0]
+            assert isinstance(comp, ast.comprehension)
+            if comp.ifs:
+                single = False
+                self.emit_op_arg(ops.BUILD_LIST, 0)
+            else:
+                single = True
+        else:
             single = False
             self.emit_op_arg(ops.BUILD_LIST, 0)
-        else:
-            single = True
         self._listcomp_generator(lc.generators, 0, lc.elt, single=single)
 
     def _comp_generator(self, node, generators, gen_index):
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.error import OperationError
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.runicode import MAXUNICODE
+from rpython.rlib.objectmodel import specialize
 
 
 def optimize_ast(space, tree, compile_info):
@@ -165,6 +166,7 @@
         self.space = space
         self.compile_info = compile_info
 
+    @specialize.argtype(1)
     def default_visitor(self, node):
         return node
 
diff --git a/rpython/rlib/types.py b/rpython/rlib/types.py
--- a/rpython/rlib/types.py
+++ b/rpython/rlib/types.py
@@ -76,8 +76,8 @@
     return model.SomeDict(dictdef)
 
 
-def instance(cls):
-    return lambda bookkeeper: model.SomeInstance(bookkeeper.getuniqueclassdef(cls))
+def instance(cls, can_be_None=False):
+    return lambda bookkeeper: model.SomeInstance(bookkeeper.getuniqueclassdef(cls), can_be_None=can_be_None)
 
 
 class SelfTypeMarker(object):

From pypy.commits at gmail.com  Mon Jan 23 16:37:32 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 23 Jan 2017 13:37:32 -0800 (PST)
Subject: [pypy-commit] cffi warn-no-ref-kept: proof-of-concept issue a
 warning if assigning a dead object to a struct field
Message-ID: <5886779c.85b2df0a.c97ab.6a66@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: warn-no-ref-kept
Changeset: r2871:ce17083bac2b
Date: 2017-01-23 23:36 +0200
http://bitbucket.org/cffi/cffi/changeset/ce17083bac2b/

Log:	proof-of-concept issue a warning if assigning a dead object to a
	struct field

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2503,6 +2503,13 @@
 {
     CFieldObject *cf;
     CTypeDescrObject *ct = cd->c_type;
+    if (value && (CDataOwn_Check(value) || Py_TYPE(value) == &CDataGCP_Type) && (value->ob_refcnt == 1))
+    {
+        if (PyErr_WarnEx(PyExc_UserWarning, 
+            "setting a struct field with a non-referenced value, expect trouble", 1))
+                return -1;
+    }
+
 
     if (ct->ct_flags & CT_POINTER)
         ct = ct->ct_itemdescr;
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -128,6 +128,27 @@
         alloc5 = ffi.new_allocator(myalloc5)
         py.test.raises(MemoryError, alloc5, "int[5]")
 
+    def test_no_ref_kept(self):
+        import gc, numpy as np
+        ffi = FFI(backend=self.Backend())
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", b"X" * size)
+        def myfree(raw):
+            seen.append(raw)
+        ffi.cdef('typedef struct {char * buf;} vstruct;')
+        alloc = ffi.new_allocator(myalloc, myfree)
+        vstruct = ffi.new('vstruct[1]')
+        b = np.empty(10, dtype='uint8')
+        vstruct[0].buf = ffi.cast('char*', b.ctypes.data)
+        with pytest.warns(UserWarning):
+            vstruct[0].buf = alloc('char[]', chr(100) * 100)
+        for i in range(10):
+            gc.collect()    
+            if len(seen) > 1:
+                break
+        assert len(seen) == 2
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
@@ -505,3 +526,5 @@
             py.test.raises(TypeError, cd)
             py.test.raises(TypeError, cd, ffi.NULL)
             py.test.raises(TypeError, cd, ffi.typeof("void *"))
+
+

From pypy.commits at gmail.com  Mon Jan 23 17:38:08 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 23 Jan 2017 14:38:08 -0800 (PST)
Subject: [pypy-commit] pypy default: Backport py3.5 asdl.py
Message-ID: <588685d0.9989df0a.fde4d.7f9c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89709:f585a82772af
Date: 2017-01-23 22:37 +0000
http://bitbucket.org/pypy/pypy/changeset/f585a82772af/

Log:	Backport py3.5 asdl.py

diff too long, truncating to 2000 out of 2152 lines

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1,8 +1,6 @@
 # Generated by tools/asdl_py.py
 from rpython.tool.pairtype import extendabletype
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.signature import signature, finishsigs
-from rpython.rlib import types
 
 from pypy.interpreter import typedef
 from pypy.interpreter.baseobjspace import W_Root
@@ -10,10 +8,9 @@
 from pypy.interpreter.gateway import interp2app
 
 
-def raise_attriberr(space, w_obj, name):
-    raise oefmt(space.w_AttributeError,
-                "'%T' object has no attribute '%s'", w_obj, name)
-
+def raise_required_value(space, w_obj, name):
+    raise oefmt(space.w_ValueError,
+                "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
     if not (space.isinstance_w(w_obj, space.w_str) or
@@ -34,7 +31,6 @@
 
 class AST(object):
     __metaclass__ = extendabletype
-    _attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")
@@ -266,6 +262,8 @@
     def from_object(space, w_node):
         w_body = get_field(space, w_node, 'body', False)
         _body = expr.from_object(space, w_body)
+        if _body is None:
+            raise_required_value(space, w_node, 'body')
         return Expression(_body)
 
 State.ast_type('Expression', 'mod', ['body'])
@@ -417,7 +415,11 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _name = space.realstr_w(w_name)
+        if _name is None:
+            raise_required_value(space, w_node, 'name')
         _args = arguments.from_object(space, w_args)
+        if _args is None:
+            raise_required_value(space, w_node, 'args')
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(w_decorator_list)
@@ -487,6 +489,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _name = space.realstr_w(w_name)
+        if _name is None:
+            raise_required_value(space, w_node, 'name')
         bases_w = space.unpackiterable(w_bases)
         _bases = [expr.from_object(space, w_item) for w_item in bases_w]
         body_w = space.unpackiterable(w_body)
@@ -620,6 +624,8 @@
         targets_w = space.unpackiterable(w_targets)
         _targets = [expr.from_object(space, w_item) for w_item in targets_w]
         _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Assign(_targets, _value, _lineno, _col_offset)
@@ -665,8 +671,14 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _target = expr.from_object(space, w_target)
+        if _target is None:
+            raise_required_value(space, w_node, 'target')
         _op = operator.from_object(space, w_op)
+        if _op is None:
+            raise_required_value(space, w_node, 'op')
         _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return AugAssign(_target, _op, _value, _lineno, _col_offset)
@@ -721,6 +733,8 @@
         values_w = space.unpackiterable(w_values)
         _values = [expr.from_object(space, w_item) for w_item in values_w]
         _nl = space.bool_w(w_nl)
+        if _nl is None:
+            raise_required_value(space, w_node, 'nl')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Print(_dest, _values, _nl, _lineno, _col_offset)
@@ -782,7 +796,11 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _target = expr.from_object(space, w_target)
+        if _target is None:
+            raise_required_value(space, w_node, 'target')
         _iter = expr.from_object(space, w_iter)
+        if _iter is None:
+            raise_required_value(space, w_node, 'iter')
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         orelse_w = space.unpackiterable(w_orelse)
@@ -843,6 +861,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _test = expr.from_object(space, w_test)
+        if _test is None:
+            raise_required_value(space, w_node, 'test')
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         orelse_w = space.unpackiterable(w_orelse)
@@ -903,6 +923,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _test = expr.from_object(space, w_test)
+        if _test is None:
+            raise_required_value(space, w_node, 'test')
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         orelse_w = space.unpackiterable(w_orelse)
@@ -959,6 +981,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _context_expr = expr.from_object(space, w_context_expr)
+        if _context_expr is None:
+            raise_required_value(space, w_node, 'context_expr')
         _optional_vars = expr.from_object(space, w_optional_vars)
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
@@ -1175,6 +1199,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _test = expr.from_object(space, w_test)
+        if _test is None:
+            raise_required_value(space, w_node, 'test')
         _msg = expr.from_object(space, w_msg)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
@@ -1318,6 +1344,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _body = expr.from_object(space, w_body)
+        if _body is None:
+            raise_required_value(space, w_node, 'body')
         _globals = expr.from_object(space, w_globals)
         _locals = expr.from_object(space, w_locals)
         _lineno = space.int_w(w_lineno)
@@ -1396,6 +1424,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Expr(_value, _lineno, _col_offset)
@@ -1591,6 +1621,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _op = boolop.from_object(space, w_op)
+        if _op is None:
+            raise_required_value(space, w_node, 'op')
         values_w = space.unpackiterable(w_values)
         _values = [expr.from_object(space, w_item) for w_item in values_w]
         _lineno = space.int_w(w_lineno)
@@ -1638,8 +1670,14 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _left = expr.from_object(space, w_left)
+        if _left is None:
+            raise_required_value(space, w_node, 'left')
         _op = operator.from_object(space, w_op)
+        if _op is None:
+            raise_required_value(space, w_node, 'op')
         _right = expr.from_object(space, w_right)
+        if _right is None:
+            raise_required_value(space, w_node, 'right')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return BinOp(_left, _op, _right, _lineno, _col_offset)
@@ -1680,7 +1718,11 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _op = unaryop.from_object(space, w_op)
+        if _op is None:
+            raise_required_value(space, w_node, 'op')
         _operand = expr.from_object(space, w_operand)
+        if _operand is None:
+            raise_required_value(space, w_node, 'operand')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return UnaryOp(_op, _operand, _lineno, _col_offset)
@@ -1722,7 +1764,11 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _args = arguments.from_object(space, w_args)
+        if _args is None:
+            raise_required_value(space, w_node, 'args')
         _body = expr.from_object(space, w_body)
+        if _body is None:
+            raise_required_value(space, w_node, 'body')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Lambda(_args, _body, _lineno, _col_offset)
@@ -1769,8 +1815,14 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _test = expr.from_object(space, w_test)
+        if _test is None:
+            raise_required_value(space, w_node, 'test')
         _body = expr.from_object(space, w_body)
+        if _body is None:
+            raise_required_value(space, w_node, 'body')
         _orelse = expr.from_object(space, w_orelse)
+        if _orelse is None:
+            raise_required_value(space, w_node, 'orelse')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return IfExp(_test, _body, _orelse, _lineno, _col_offset)
@@ -1886,8 +1938,8 @@
 
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
-        for i in range(len(self.generators)):
-            self.generators[i] = self.generators[i].mutate_over(visitor)
+        if self.generators:
+            visitor._mutate_sequence(self.generators)
         return visitor.visit_ListComp(self)
 
     def to_object(self, space):
@@ -1913,6 +1965,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _elt = expr.from_object(space, w_elt)
+        if _elt is None:
+            raise_required_value(space, w_node, 'elt')
         generators_w = space.unpackiterable(w_generators)
         _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
         _lineno = space.int_w(w_lineno)
@@ -1934,8 +1988,8 @@
 
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
-        for i in range(len(self.generators)):
-            self.generators[i] = self.generators[i].mutate_over(visitor)
+        if self.generators:
+            visitor._mutate_sequence(self.generators)
         return visitor.visit_SetComp(self)
 
     def to_object(self, space):
@@ -1961,6 +2015,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _elt = expr.from_object(space, w_elt)
+        if _elt is None:
+            raise_required_value(space, w_node, 'elt')
         generators_w = space.unpackiterable(w_generators)
         _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
         _lineno = space.int_w(w_lineno)
@@ -1984,8 +2040,8 @@
     def mutate_over(self, visitor):
         self.key = self.key.mutate_over(visitor)
         self.value = self.value.mutate_over(visitor)
-        for i in range(len(self.generators)):
-            self.generators[i] = self.generators[i].mutate_over(visitor)
+        if self.generators:
+            visitor._mutate_sequence(self.generators)
         return visitor.visit_DictComp(self)
 
     def to_object(self, space):
@@ -2014,7 +2070,11 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _key = expr.from_object(space, w_key)
+        if _key is None:
+            raise_required_value(space, w_node, 'key')
         _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         generators_w = space.unpackiterable(w_generators)
         _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
         _lineno = space.int_w(w_lineno)
@@ -2036,8 +2096,8 @@
 
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
-        for i in range(len(self.generators)):
-            self.generators[i] = self.generators[i].mutate_over(visitor)
+        if self.generators:
+            visitor._mutate_sequence(self.generators)
         return visitor.visit_GeneratorExp(self)
 
     def to_object(self, space):
@@ -2063,6 +2123,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _elt = expr.from_object(space, w_elt)
+        if _elt is None:
+            raise_required_value(space, w_node, 'elt')
         generators_w = space.unpackiterable(w_generators)
         _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
         _lineno = space.int_w(w_lineno)
@@ -2156,6 +2218,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _left = expr.from_object(space, w_left)
+        if _left is None:
+            raise_required_value(space, w_node, 'left')
         ops_w = space.unpackiterable(w_ops)
         _ops = [cmpop.from_object(space, w_item) for w_item in ops_w]
         comparators_w = space.unpackiterable(w_comparators)
@@ -2228,6 +2292,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _func = expr.from_object(space, w_func)
+        if _func is None:
+            raise_required_value(space, w_node, 'func')
         args_w = space.unpackiterable(w_args)
         _args = [expr.from_object(space, w_item) for w_item in args_w]
         keywords_w = space.unpackiterable(w_keywords)
@@ -2270,6 +2336,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Repr(_value, _lineno, _col_offset)
@@ -2305,6 +2373,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _n = w_n
+        if _n is None:
+            raise_required_value(space, w_node, 'n')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Num(_n, _lineno, _col_offset)
@@ -2340,6 +2410,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _s = check_string(space, w_s)
+        if _s is None:
+            raise_required_value(space, w_node, 's')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Str(_s, _lineno, _col_offset)
@@ -2384,8 +2456,14 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         _attr = space.realstr_w(w_attr)
+        if _attr is None:
+            raise_required_value(space, w_node, 'attr')
         _ctx = expr_context.from_object(space, w_ctx)
+        if _ctx is None:
+            raise_required_value(space, w_node, 'ctx')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Attribute(_value, _attr, _ctx, _lineno, _col_offset)
@@ -2431,8 +2509,14 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         _slice = slice.from_object(space, w_slice)
+        if _slice is None:
+            raise_required_value(space, w_node, 'slice')
         _ctx = expr_context.from_object(space, w_ctx)
+        if _ctx is None:
+            raise_required_value(space, w_node, 'ctx')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Subscript(_value, _slice, _ctx, _lineno, _col_offset)
@@ -2472,7 +2556,11 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _id = space.realstr_w(w_id)
+        if _id is None:
+            raise_required_value(space, w_node, 'id')
         _ctx = expr_context.from_object(space, w_ctx)
+        if _ctx is None:
+            raise_required_value(space, w_node, 'ctx')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Name(_id, _ctx, _lineno, _col_offset)
@@ -2520,6 +2608,8 @@
         elts_w = space.unpackiterable(w_elts)
         _elts = [expr.from_object(space, w_item) for w_item in elts_w]
         _ctx = expr_context.from_object(space, w_ctx)
+        if _ctx is None:
+            raise_required_value(space, w_node, 'ctx')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return List(_elts, _ctx, _lineno, _col_offset)
@@ -2567,6 +2657,8 @@
         elts_w = space.unpackiterable(w_elts)
         _elts = [expr.from_object(space, w_item) for w_item in elts_w]
         _ctx = expr_context.from_object(space, w_ctx)
+        if _ctx is None:
+            raise_required_value(space, w_node, 'ctx')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Tuple(_elts, _ctx, _lineno, _col_offset)
@@ -2602,6 +2694,8 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _value = w_value
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Const(_value, _lineno, _col_offset)
@@ -2808,6 +2902,8 @@
     def from_object(space, w_node):
         w_value = get_field(space, w_node, 'value', False)
         _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         return Index(_value)
 
 State.ast_type('Index', 'slice', ['value'])
@@ -3109,7 +3205,6 @@
     _NotIn,
 ]
 
- at finishsigs
 class comprehension(AST):
 
     def __init__(self, target, iter, ifs):
@@ -3117,7 +3212,6 @@
         self.iter = iter
         self.ifs = ifs
 
-    @signature(types.self(), types.any(), returns=types.self())
     def mutate_over(self, visitor):
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
@@ -3148,7 +3242,11 @@
         w_iter = get_field(space, w_node, 'iter', False)
         w_ifs = get_field(space, w_node, 'ifs', False)
         _target = expr.from_object(space, w_target)
+        if _target is None:
+            raise_required_value(space, w_node, 'target')
         _iter = expr.from_object(space, w_iter)
+        if _iter is None:
+            raise_required_value(space, w_node, 'iter')
         ifs_w = space.unpackiterable(w_ifs)
         _ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
         return comprehension(_target, _iter, _ifs)
@@ -3307,7 +3405,11 @@
         w_arg = get_field(space, w_node, 'arg', False)
         w_value = get_field(space, w_node, 'value', False)
         _arg = space.realstr_w(w_arg)
+        if _arg is None:
+            raise_required_value(space, w_node, 'arg')
         _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         return keyword(_arg, _value)
 
 State.ast_type('keyword', 'AST', ['arg', 'value'])
@@ -3337,6 +3439,8 @@
         w_name = get_field(space, w_node, 'name', False)
         w_asname = get_field(space, w_node, 'asname', True)
         _name = space.realstr_w(w_name)
+        if _name is None:
+            raise_required_value(space, w_node, 'name')
         _asname = space.str_or_None_w(w_asname)
         return alias(_name, _asname)
 
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -1,6 +1,6 @@
 -- ASDL's five builtin types are identifier, int, string, object, bool
 
-module Python version "$Revision: 43614 $"
+module Python
 {
 	mod = Module(stmt* body)
 	    | Interactive(stmt* body)
diff --git a/pypy/interpreter/astcompiler/tools/asdl.py b/pypy/interpreter/astcompiler/tools/asdl.py
--- a/pypy/interpreter/astcompiler/tools/asdl.py
+++ b/pypy/interpreter/astcompiler/tools/asdl.py
@@ -1,243 +1,53 @@
-"""An implementation of the Zephyr Abstract Syntax Definition Language.
+#-------------------------------------------------------------------------------
+# Parser for ASDL [1] definition files. Reads in an ASDL description and parses
+# it into an AST that describes it.
+#
+# The EBNF we're parsing here: Figure 1 of the paper [1]. Extended to support
+# modules and attributes after a product. Words starting with Capital letters
+# are terminals. Literal tokens are in "double quotes". Others are
+# non-terminals. Id is either TokenId or ConstructorId.
+#
+# module        ::= "module" Id "{" [definitions] "}"
+# definitions   ::= { TypeId "=" type }
+# type          ::= product | sum
+# product       ::= fields ["attributes" fields]
+# fields        ::= "(" { field, "," } field ")"
+# field         ::= TypeId ["?" | "*"] [Id]
+# sum           ::= constructor { "|" constructor } ["attributes" fields]
+# constructor   ::= ConstructorId [fields]
+#
+# [1] "The Zephyr Abstract Syntax Description Language" by Wang, et. al. See
+#     http://asdl.sourceforge.net/
+#-------------------------------------------------------------------------------
+from collections import namedtuple
+import re
 
-See http://asdl.sourceforge.net/ and
-http://www.cs.princeton.edu/~danwang/Papers/dsl97/dsl97-abstract.html.
+__all__ = [
+    'builtin_types', 'parse', 'AST', 'Module', 'Type', 'Constructor',
+    'Field', 'Sum', 'Product', 'VisitorBase', 'Check', 'check']
 
-Only supports top level module decl, not view.  I'm guessing that view
-is intended to support the browser and I'm not interested in the
-browser.
+# The following classes define nodes into which the ASDL description is parsed.
+# Note: this is a "meta-AST". ASDL files (such as Python.asdl) describe the AST
+# structure used by a programming language. But ASDL files themselves need to be
+# parsed. This module parses ASDL files and uses a simple AST to represent them.
+# See the EBNF at the top of the file to understand the logical connection
+# between the various node types.
 
-Changes for Python: Add support for module versions
-"""
+builtin_types = {'identifier', 'string', 'bytes', 'int', 'bool', 'object',
+                 'singleton'}
 
-import os
-import traceback
+class AST:
+    def __repr__(self):
+        raise NotImplementedError
 
-import spark
-
-class Token(object):
-    # spark seems to dispatch in the parser based on a token's
-    # type attribute
-    def __init__(self, type, lineno):
-        self.type = type
-        self.lineno = lineno
-
-    def __str__(self):
-        return self.type
+class Module(AST):
+    def __init__(self, name, dfns):
+        self.name = name
+        self.dfns = dfns
+        self.types = {type.name: type.value for type in dfns}
 
     def __repr__(self):
-        return str(self)
-
-class Id(Token):
-    def __init__(self, value, lineno):
-        self.type = 'Id'
-        self.value = value
-        self.lineno = lineno
-
-    def __str__(self):
-        return self.value
-
-class String(Token):
-    def __init__(self, value, lineno):
-        self.type = 'String'
-        self.value = value
-        self.lineno = lineno
-
-class ASDLSyntaxError(Exception):
-
-    def __init__(self, lineno, token=None, msg=None):
-        self.lineno = lineno
-        self.token = token
-        self.msg = msg
-
-    def __str__(self):
-        if self.msg is None:
-            return "Error at '%s', line %d" % (self.token, self.lineno)
-        else:
-            return "%s, line %d" % (self.msg, self.lineno)
-
-class ASDLScanner(spark.GenericScanner, object):
-
-    def tokenize(self, input):
-        self.rv = []
-        self.lineno = 1
-        super(ASDLScanner, self).tokenize(input)
-        return self.rv
-
-    def t_id(self, s):
-        r"[\w\.]+"
-        # XXX doesn't distinguish upper vs. lower, which is
-        # significant for ASDL.
-        self.rv.append(Id(s, self.lineno))
-
-    def t_string(self, s):
-        r'"[^"]*"'
-        self.rv.append(String(s, self.lineno))
-
-    def t_xxx(self, s): # not sure what this production means
-        r"<="
-        self.rv.append(Token(s, self.lineno))
-
-    def t_punctuation(self, s):
-        r"[\{\}\*\=\|\(\)\,\?\:]"
-        self.rv.append(Token(s, self.lineno))
-
-    def t_comment(self, s):
-        r"\-\-[^\n]*"
-        pass
-
-    def t_newline(self, s):
-        r"\n"
-        self.lineno += 1
-
-    def t_whitespace(self, s):
-        r"[ \t]+"
-        pass
-
-    def t_default(self, s):
-        r" . +"
-        raise ValueError("unmatched input: %s" % `s`)
-
-class ASDLParser(spark.GenericParser, object):
-    def __init__(self):
-        super(ASDLParser, self).__init__("module")
-
-    def typestring(self, tok):
-        return tok.type
-
-    def error(self, tok):
-        raise ASDLSyntaxError(tok.lineno, tok)
-
-    def p_module_0(self, (module, name, version, _0, _1)):
-        " module ::= Id Id version { } "
-        if module.value != "module":
-            raise ASDLSyntaxError(module.lineno,
-                                  msg="expected 'module', found %s" % module)
-        return Module(name, None, version)
-
-    def p_module(self, (module, name, version, _0, definitions, _1)):
-        " module ::= Id Id version { definitions } "
-        if module.value != "module":
-            raise ASDLSyntaxError(module.lineno,
-                                  msg="expected 'module', found %s" % module)
-        return Module(name, definitions, version)
-
-    def p_version(self, (version, V)):
-        "version ::= Id String"
-        if version.value != "version":
-            raise ASDLSyntaxError(version.lineno,
-                                msg="expected 'version', found %" % version)
-        return V
-
-    def p_definition_0(self, (definition,)):
-        " definitions ::= definition "
-        return definition
-
-    def p_definition_1(self, (definitions, definition)):
-        " definitions ::= definition definitions "
-        return definitions + definition
-
-    def p_definition(self, (id, _, type)):
-        " definition ::= Id = type "
-        return [Type(id, type)]
-
-    def p_type_0(self, (product,)):
-        " type ::= product "
-        return product
-
-    def p_type_1(self, (sum,)):
-        " type ::= sum "
-        return Sum(sum)
-
-    def p_type_2(self, (sum, id, _0, attributes, _1)):
-        " type ::= sum Id ( fields ) "
-        if id.value != "attributes":
-            raise ASDLSyntaxError(id.lineno,
-                                  msg="expected attributes, found %s" % id)
-        if attributes:
-            attributes.reverse()
-        return Sum(sum, attributes)
-
-    def p_product(self, (_0, fields, _1)):
-        " product ::= ( fields ) "
-        # XXX can't I just construct things in the right order?
-        fields.reverse()
-        return Product(fields)
-
-    def p_sum_0(self, (constructor,)):
-        " sum ::= constructor "
-        return [constructor]
-
-    def p_sum_1(self, (constructor, _, sum)):
-        " sum ::= constructor | sum "
-        return [constructor] + sum
-
-    def p_sum_2(self, (constructor, _, sum)):
-        " sum ::= constructor | sum "
-        return [constructor] + sum
-
-    def p_constructor_0(self, (id,)):
-        " constructor ::= Id "
-        return Constructor(id)
-
-    def p_constructor_1(self, (id, _0, fields, _1)):
-        " constructor ::= Id ( fields ) "
-        # XXX can't I just construct things in the right order?
-        fields.reverse()
-        return Constructor(id, fields)
-
-    def p_fields_0(self, (field,)):
-        " fields ::= field "
-        return [field]
-
-    def p_fields_1(self, (field, _, fields)):
-        " fields ::= field , fields "
-        return fields + [field]
-
-    def p_field_0(self, (type,)):
-        " field ::= Id "
-        return Field(type)
-
-    def p_field_1(self, (type, name)):
-        " field ::= Id Id "
-        return Field(type, name)
-
-    def p_field_2(self, (type, _, name)):
-        " field ::= Id * Id "
-        return Field(type, name, seq=True)
-
-    def p_field_3(self, (type, _, name)):
-        " field ::= Id ? Id "
-        return Field(type, name, opt=True)
-
-    def p_field_4(self, (type, _)):
-        " field ::= Id * "
-        return Field(type, seq=True)
-
-    def p_field_5(self, (type, _)):
-        " field ::= Id ? "
-        return Field(type, opt=True)
-
-builtin_types = ("identifier", "string", "int", "bool", "object")
-
-# below is a collection of classes to capture the AST of an AST :-)
-# not sure if any of the methods are useful yet, but I'm adding them
-# piecemeal as they seem helpful
-
-class AST(object):
-    pass # a marker class
-
-class Module(AST):
-    def __init__(self, name, dfns, version):
-        self.name = name
-        self.dfns = dfns
-        self.version = version
-        self.types = {} # maps type name to value (from dfns)
-        for type in dfns:
-            self.types[type.name.value] = type.value
-
-    def __repr__(self):
-        return "Module(%s, %s)" % (self.name, self.dfns)
+        return 'Module({0.name}, {0.dfns})'.format(self)
 
 class Type(AST):
     def __init__(self, name, value):
@@ -245,7 +55,7 @@
         self.value = value
 
     def __repr__(self):
-        return "Type(%s, %s)" % (self.name, self.value)
+        return 'Type({0.name}, {0.value})'.format(self)
 
 class Constructor(AST):
     def __init__(self, name, fields=None):
@@ -253,7 +63,7 @@
         self.fields = fields or []
 
     def __repr__(self):
-        return "Constructor(%s, %s)" % (self.name, self.fields)
+        return 'Constructor({0.name}, {0.fields})'.format(self)
 
 class Field(AST):
     def __init__(self, type, name=None, seq=False, opt=False):
@@ -270,9 +80,9 @@
         else:
             extra = ""
         if self.name is None:
-            return "Field(%s%s)" % (self.type, extra)
+            return 'Field({0.type}{1})'.format(self, extra)
         else:
-            return "Field(%s, %s%s)" % (self.type, self.name, extra)
+            return 'Field({0.type}, {0.name}{1})'.format(self, extra)
 
 class Sum(AST):
     def __init__(self, types, attributes=None):
@@ -280,47 +90,54 @@
         self.attributes = attributes or []
 
     def __repr__(self):
-        if self.attributes is None:
-            return "Sum(%s)" % self.types
+        if self.attributes:
+            return 'Sum({0.types}, {0.attributes})'.format(self)
         else:
-            return "Sum(%s, %s)" % (self.types, self.attributes)
+            return 'Sum({0.types})'.format(self)
 
 class Product(AST):
-    def __init__(self, fields):
+    def __init__(self, fields, attributes=None):
         self.fields = fields
+        self.attributes = attributes or []
 
     def __repr__(self):
-        return "Product(%s)" % self.fields
+        if self.attributes:
+            return 'Product({0.fields}, {0.attributes})'.format(self)
+        else:
+            return 'Product({0.fields})'.format(self)
+
+# A generic visitor for the meta-AST that describes ASDL. This can be used by
+# emitters. Note that this visitor does not provide a generic visit method, so a
+# subclass needs to define visit methods from visitModule to as deep as the
+# interesting node.
+# We also define a Check visitor that makes sure the parsed ASDL is well-formed.
 
 class VisitorBase(object):
+    """Generic tree visitor for ASTs."""
+    def __init__(self):
+        self.cache = {}
 
-    def __init__(self, skip=False):
-        self.cache = {}
-        self.skip = skip
-
-    def visit(self, object, *args):
-        meth = self._dispatch(object)
-        if meth is None:
-            return
-        meth(object, *args)
-
-    def _dispatch(self, object):
-        assert isinstance(object, AST), repr(object)
-        klass = object.__class__
+    def visit(self, obj, *args):
+        klass = obj.__class__
         meth = self.cache.get(klass)
         if meth is None:
             methname = "visit" + klass.__name__
-            if self.skip:
-                meth = getattr(self, methname, None)
-            else:
-                meth = getattr(self, methname)
+            meth = getattr(self, methname, None)
             self.cache[klass] = meth
-        return meth
+        if meth:
+            try:
+                meth(obj, *args)
+            except Exception as e:
+                print("Error visiting %r: %s" % (obj, e))
+                raise
 
 class Check(VisitorBase):
+    """A visitor that checks a parsed ASDL tree for correctness.
 
+    Errors are printed and accumulated.
+    """
     def __init__(self):
-        super(Check, self).__init__(skip=True)
+        super(Check, self).__init__()
         self.cons = {}
         self.errors = 0
         self.types = {}
@@ -342,8 +159,8 @@
         if conflict is None:
             self.cons[key] = name
         else:
-            print "Redefinition of constructor %s" % key
-            print "Defined in %s and %s" % (conflict, name)
+            print('Redefinition of constructor {}'.format(key))
+            print('Defined in {} and {}'.format(conflict, name))
             self.errors += 1
         for f in cons.fields:
             self.visit(f, key)
@@ -358,6 +175,11 @@
             self.visit(f, name)
 
 def check(mod):
+    """Check the parsed ASDL tree for correctness.
+
+    Return True if success. For failure, the errors are printed out and False
+    is returned.
+    """
     v = Check()
     v.visit(mod)
 
@@ -365,40 +187,190 @@
         if t not in mod.types and not t in builtin_types:
             v.errors += 1
             uses = ", ".join(v.types[t])
-            print "Undefined type %s, used in %s" % (t, uses)
-
+            print('Undefined type {}, used in {}'.format(t, uses))
     return not v.errors
 
-def parse(file):
-    scanner = ASDLScanner()
-    parser = ASDLParser()
+# The ASDL parser itself comes next. The only interesting external interface
+# here is the top-level parse function.
 
-    buf = open(file).read()
-    tokens = scanner.tokenize(buf)
-    try:
-        return parser.parse(tokens)
-    except ASDLSyntaxError as err:
-        print err
-        lines = buf.split("\n")
-        print lines[err.lineno - 1] # lines starts at 0, files at 1
+def parse(filename):
+    """Parse ASDL from the given file and return a Module node describing it."""
+    with open(filename) as f:
+        parser = ASDLParser()
+        return parser.parse(f.read())
 
-if __name__ == "__main__":
-    import glob
-    import sys
+# Types for describing tokens in an ASDL specification.
+class TokenKind:
+    """TokenKind is provides a scope for enumerated token kinds."""
+    (ConstructorId, TypeId, Equals, Comma, Question, Pipe, Asterisk,
+     LParen, RParen, LBrace, RBrace) = range(11)
 
-    if len(sys.argv) > 1:
-        files = sys.argv[1:]
-    else:
-        testdir = "tests"
-        files = glob.glob(testdir + "/*.asdl")
+    operator_table = {
+        '=': Equals, ',': Comma,    '?': Question, '|': Pipe,    '(': LParen,
+        ')': RParen, '*': Asterisk, '{': LBrace,   '}': RBrace}
 
-    for file in files:
-        print file
-        mod = parse(file)
-        print "module", mod.name
-        print len(mod.dfns), "definitions"
-        if not check(mod):
-            print "Check failed"
+Token = namedtuple('Token', 'kind value lineno')
+
+class ASDLSyntaxError(Exception):
+    def __init__(self, msg, lineno=None):
+        self.msg = msg
+        self.lineno = lineno or '<unknown>'
+
+    def __str__(self):
+        return 'Syntax error on line {0.lineno}: {0.msg}'.format(self)
+
+def tokenize_asdl(buf):
+    """Tokenize the given buffer. Yield Token objects."""
+    for lineno, line in enumerate(buf.splitlines(), 1):
+        for m in re.finditer(r'\s*(\w+|--.*|.)', line.strip()):
+            c = m.group(1)
+            if c[0].isalpha():
+                # Some kind of identifier
+                if c[0].isupper():
+                    yield Token(TokenKind.ConstructorId, c, lineno)
+                else:
+                    yield Token(TokenKind.TypeId, c, lineno)
+            elif c[:2] == '--':
+                # Comment
+                break
+            else:
+                # Operators
+                try:
+                    op_kind = TokenKind.operator_table[c]
+                except KeyError:
+                    raise ASDLSyntaxError('Invalid operator %s' % c, lineno)
+                yield Token(op_kind, c, lineno)
+
+class ASDLParser:
+    """Parser for ASDL files.
+
+    Create, then call the parse method on a buffer containing ASDL.
+    This is a simple recursive descent parser that uses tokenize_asdl for the
+    lexing.
+    """
+    def __init__(self):
+        self._tokenizer = None
+        self.cur_token = None
+
+    def parse(self, buf):
+        """Parse the ASDL in the buffer and return an AST with a Module root.
+        """
+        self._tokenizer = tokenize_asdl(buf)
+        self._advance()
+        return self._parse_module()
+
+    def _parse_module(self):
+        if self._at_keyword('module'):
+            self._advance()
         else:
-            for dfn in mod.dfns:
-                print dfn.type
+            raise ASDLSyntaxError(
+                'Expected "module" (found {})'.format(self.cur_token.value),
+                self.cur_token.lineno)
+        name = self._match(self._id_kinds)
+        self._match(TokenKind.LBrace)
+        defs = self._parse_definitions()
+        self._match(TokenKind.RBrace)
+        return Module(name, defs)
+
+    def _parse_definitions(self):
+        defs = []
+        while self.cur_token.kind == TokenKind.TypeId:
+            typename = self._advance()
+            self._match(TokenKind.Equals)
+            type = self._parse_type()
+            defs.append(Type(typename, type))
+        return defs
+
+    def _parse_type(self):
+        if self.cur_token.kind == TokenKind.LParen:
+            # If we see a (, it's a product
+            return self._parse_product()
+        else:
+            # Otherwise it's a sum. Look for ConstructorId
+            sumlist = [Constructor(self._match(TokenKind.ConstructorId),
+                                   self._parse_optional_fields())]
+            while self.cur_token.kind  == TokenKind.Pipe:
+                # More constructors
+                self._advance()
+                sumlist.append(Constructor(
+                                self._match(TokenKind.ConstructorId),
+                                self._parse_optional_fields()))
+            return Sum(sumlist, self._parse_optional_attributes())
+
+    def _parse_product(self):
+        return Product(self._parse_fields(), self._parse_optional_attributes())
+
+    def _parse_fields(self):
+        fields = []
+        self._match(TokenKind.LParen)
+        while self.cur_token.kind == TokenKind.TypeId:
+            typename = self._advance()
+            is_seq, is_opt = self._parse_optional_field_quantifier()
+            id = (self._advance() if self.cur_token.kind in self._id_kinds
+                                  else None)
+            fields.append(Field(typename, id, seq=is_seq, opt=is_opt))
+            if self.cur_token.kind == TokenKind.RParen:
+                break
+            elif self.cur_token.kind == TokenKind.Comma:
+                self._advance()
+        self._match(TokenKind.RParen)
+        return fields
+
+    def _parse_optional_fields(self):
+        if self.cur_token.kind == TokenKind.LParen:
+            return self._parse_fields()
+        else:
+            return None
+
+    def _parse_optional_attributes(self):
+        if self._at_keyword('attributes'):
+            self._advance()
+            return self._parse_fields()
+        else:
+            return None
+
+    def _parse_optional_field_quantifier(self):
+        is_seq, is_opt = False, False
+        if self.cur_token.kind == TokenKind.Asterisk:
+            is_seq = True
+            self._advance()
+        elif self.cur_token.kind == TokenKind.Question:
+            is_opt = True
+            self._advance()
+        return is_seq, is_opt
+
+    def _advance(self):
+        """ Return the value of the current token and read the next one into
+            self.cur_token.
+        """
+        cur_val = None if self.cur_token is None else self.cur_token.value
+        try:
+            self.cur_token = next(self._tokenizer)
+        except StopIteration:
+            self.cur_token = None
+        return cur_val
+
+    _id_kinds = (TokenKind.ConstructorId, TokenKind.TypeId)
+
+    def _match(self, kind):
+        """The 'match' primitive of RD parsers.
+
+        * Verifies that the current token is of the given kind (kind can
+          be a tuple, in which the kind must match one of its members).
+        * Returns the value of the current token
+        * Reads in the next token
+        """
+        if (isinstance(kind, tuple) and self.cur_token.kind in kind or
+            self.cur_token.kind == kind
+            ):
+            value = self.cur_token.value
+            self._advance()
+            return value
+        else:
+            raise ASDLSyntaxError(
+                'Unmatched {} (found {})'.format(kind, self.cur_token.kind),
+                self.cur_token.lineno)
+
+    def _at_keyword(self, keyword):
+        return (self.cur_token.kind == TokenKind.TypeId and
+                self.cur_token.value == keyword)
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -85,7 +85,7 @@
             self.emit("class %s(AST):" % (base,))
             if sum.attributes:
                 self.emit("")
-                args = ", ".join(attr.name.value for attr in sum.attributes)
+                args = ", ".join(attr.name for attr in sum.attributes)
                 self.emit("def __init__(self, %s):" % (args,), 1)
                 for attr in sum.attributes:
                     self.visit(attr)
@@ -101,8 +101,8 @@
                           % (typ.name,), 3)
             self.emit("raise oefmt(space.w_TypeError,", 2)
             self.emit("        \"Expected %s node, got %%T\", w_node)" % (base,), 2)
-            self.emit("State.ast_type('%r', 'AST', None, %s)" %
-                      (base, [repr(attr.name) for attr in sum.attributes]))
+            self.emit("State.ast_type(%r, 'AST', None, %s)" %
+                      (base, [attr.name for attr in sum.attributes]))
             self.emit("")
             for cons in sum.types:
                 self.visit(cons, base, sum.attributes)
@@ -118,37 +118,37 @@
         self.emit("visitor.visit_%s(self)" % (name,), 2)
         self.emit("")
         self.make_converters(product.fields, name)
-        self.emit("State.ast_type('%r', 'AST', %s)" %
-                  (name, [repr(f.name) for f in product.fields]))
+        self.emit("State.ast_type(%r, 'AST', %s)" %
+                  (name, [f.name for f in product.fields]))
         self.emit("")
 
     def get_value_converter(self, field, value):
-        if field.type.value in self.data.simple_types:
+        if field.type in self.data.simple_types:
             return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
-        elif field.type.value in ("object", "string"):
+        elif field.type in ("object", "singleton", "string", "bytes"):
             return value
-        elif field.type.value in ("identifier", "int", "bool"):
+        elif field.type in ("identifier", "int", "bool"):
             return "space.wrap(%s)" % (value,)
         else:
             wrapper = "%s.to_object(space)" % (value,)
             if field.opt:
                 wrapper += " if %s is not None else space.w_None" % (value,)
             return wrapper
-        
+
     def get_value_extractor(self, field, value):
-        if field.type.value in self.data.simple_types:
+        if field.type in self.data.simple_types:
             return "%s.from_object(space, %s)" % (field.type, value)
-        elif field.type.value in ("object",):
+        elif field.type in ("object","singleton"):
             return value
-        elif field.type.value in ("string",):
+        elif field.type in ("string","bytes"):
             return "check_string(space, %s)" % (value,)
-        elif field.type.value in ("identifier",):
+        elif field.type in ("identifier",):
             if field.opt:
                 return "space.str_or_None_w(%s)" % (value,)
             return "space.realstr_w(%s)" % (value,)
-        elif field.type.value in ("int",):
+        elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
-        elif field.type.value in ("bool",):
+        elif field.type in ("bool",):
             return "space.bool_w(%s)" % (value,)
         else:
             return "%s.from_object(space, %s)" % (field.type, value)
@@ -179,6 +179,11 @@
         else:
             value = self.get_value_extractor(field, "w_%s" % (field.name,))
             lines = ["_%s = %s" % (field.name, value)]
+            if not field.opt and field.type not in ("int",):
+                lines.append("if _%s is None:" % (field.name,))
+                lines.append("    raise_required_value(space, w_node, '%s')"
+                             % (field.name,))
+
         return lines
 
     def make_converters(self, fields, name, extras=None):
@@ -216,12 +221,12 @@
             if extras:
                 base_args = ", ".join(str(field.name) for field in extras)
                 self.emit("%s.__init__(self, %s)" % (base, base_args), 2)
-    
+
     def make_mutate_over(self, cons, name):
         self.emit("def mutate_over(self, visitor):", 1)
         for field in cons.fields:
-            if (field.type.value not in asdl.builtin_types and
-                field.type.value not in self.data.simple_types):
+            if (field.type not in asdl.builtin_types and
+                field.type not in self.data.simple_types):
                 if field.opt or field.seq:
                     level = 3
                     self.emit("if self.%s:" % (field.name,), 2)
@@ -247,8 +252,8 @@
         self.emit("")
         self.make_mutate_over(cons, cons.name)
         self.make_converters(cons.fields, cons.name, extra_attributes)
-        self.emit("State.ast_type('%r', '%s', %s)" % 
-                  (cons.name, base, [repr(f.name) for f in cons.fields]))
+        self.emit("State.ast_type(%r, '%s', %s)" %
+                  (cons.name, base, [f.name for f in cons.fields]))
         self.emit("")
 
     def visitField(self, field):
@@ -320,8 +325,8 @@
         self.emit("")
 
     def visitField(self, field):
-        if (field.type.value not in asdl.builtin_types and 
-            field.type.value not in self.data.simple_types):
+        if (field.type not in asdl.builtin_types and
+            field.type not in self.data.simple_types):
             level = 2
             template = "node.%s.walkabout(self)"
             if field.seq:
@@ -362,7 +367,7 @@
             if isinstance(tp.value, asdl.Sum):
                 sum = tp.value
                 if is_simple_sum(sum):
-                    simple_types.add(tp.name.value)
+                    simple_types.add(tp.name)
                 else:
                     attrs = [field for field in sum.attributes]
                     for cons in sum.types:
@@ -370,7 +375,7 @@
                         cons_attributes[cons] = attrs
             else:
                 prod = tp.value
-                prod_simple.add(tp.name.value)
+                prod_simple.add(tp.name)
                 add_masks(prod.fields, prod)
         prod_simple.update(simple_types)
         self.cons_attributes = cons_attributes
@@ -391,10 +396,9 @@
 from pypy.interpreter.gateway import interp2app
 
 
-def raise_attriberr(space, w_obj, name):
-    raise oefmt(space.w_AttributeError,
-                "'%T' object has no attribute '%s'", w_obj, name)
-
+def raise_required_value(space, w_obj, name):
+    raise oefmt(space.w_ValueError,
+                "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
     if not (space.isinstance_w(w_obj, space.w_str) or
@@ -522,7 +526,7 @@
         self.w_AST = space.gettypeobject(W_AST.typedef)
         for (name, base, fields, attributes) in self.AST_TYPES:
             self.make_new_type(space, name, base, fields, attributes)
-        
+
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
@@ -534,7 +538,7 @@
             space.setitem_str(w_dict, "_attributes",
                               space.newtuple([space.wrap(a) for a in attributes]))
         w_type = space.call_function(
-            space.w_type, 
+            space.w_type,
             space.wrap(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
diff --git a/pypy/interpreter/astcompiler/tools/spark.py b/pypy/interpreter/astcompiler/tools/spark.py
deleted file mode 100644
--- a/pypy/interpreter/astcompiler/tools/spark.py
+++ /dev/null
@@ -1,839 +0,0 @@
-#  Copyright (c) 1998-2002 John Aycock
-#
-#  Permission is hereby granted, free of charge, to any person obtaining
-#  a copy of this software and associated documentation files (the
-#  "Software"), to deal in the Software without restriction, including
-#  without limitation the rights to use, copy, modify, merge, publish,
-#  distribute, sublicense, and/or sell copies of the Software, and to
-#  permit persons to whom the Software is furnished to do so, subject to
-#  the following conditions:
-#
-#  The above copyright notice and this permission notice shall be
-#  included in all copies or substantial portions of the Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-#  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-#  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-#  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-__version__ = 'SPARK-0.7 (pre-alpha-5)'
-
-import re
-import string
-
-def _namelist(instance):
-    namelist, namedict, classlist = [], {}, [instance.__class__]
-    for c in classlist:
-        for b in c.__bases__:
-            classlist.append(b)
-        for name in c.__dict__.keys():
-            if not namedict.has_key(name):
-                namelist.append(name)
-                namedict[name] = 1
-    return namelist
-
-class GenericScanner:
-    def __init__(self, flags=0):
-        pattern = self.reflect()
-        self.re = re.compile(pattern, re.VERBOSE|flags)
-
-        self.index2func = {}
-        for name, number in self.re.groupindex.items():
-            self.index2func[number-1] = getattr(self, 't_' + name)
-
-    def makeRE(self, name):
-        doc = getattr(self, name).__doc__
-        rv = '(?P<%s>%s)' % (name[2:], doc)
-        return rv
-
-    def reflect(self):
-        rv = []
-        for name in _namelist(self):
-            if name[:2] == 't_' and name != 't_default':
-                rv.append(self.makeRE(name))
-
-        rv.append(self.makeRE('t_default'))
-        return string.join(rv, '|')
-
-    def error(self, s, pos):
-        print "Lexical error at position %s" % pos
-        raise SystemExit
-
-    def tokenize(self, s):
-        pos = 0
-        n = len(s)
-        while pos < n:
-            m = self.re.match(s, pos)
-            if m is None:
-                self.error(s, pos)
-
-            groups = m.groups()
-            for i in range(len(groups)):
-                if groups[i] and self.index2func.has_key(i):
-                    self.index2func[i](groups[i])
-            pos = m.end()
-
-    def t_default(self, s):
-        r'( . | \n )+'
-        print "Specification error: unmatched input"
-        raise SystemExit
-
-#
-#  Extracted from GenericParser and made global so that [un]picking works.
-#
-class _State:
-    def __init__(self, stateno, items):
-        self.T, self.complete, self.items = [], [], items
-        self.stateno = stateno
-
-class GenericParser:
-    #
-    #  An Earley parser, as per J. Earley, "An Efficient Context-Free
-    #  Parsing Algorithm", CACM 13(2), pp. 94-102.  Also J. C. Earley,
-    #  "An Efficient Context-Free Parsing Algorithm", Ph.D. thesis,
-    #  Carnegie-Mellon University, August 1968.  New formulation of
-    #  the parser according to J. Aycock, "Practical Earley Parsing
-    #  and the SPARK Toolkit", Ph.D. thesis, University of Victoria,
-    #  2001, and J. Aycock and R. N. Horspool, "Practical Earley
-    #  Parsing", unpublished paper, 2001.
-    #
-
-    def __init__(self, start):
-        self.rules = {}
-        self.rule2func = {}
-        self.rule2name = {}
-        self.collectRules()
-        self.augment(start)
-        self.ruleschanged = 1
-
-    _NULLABLE = '\e_'
-    _START = 'START'
-    _BOF = '|-'
-
-    #
-    #  When pickling, take the time to generate the full state machine;
-    #  some information is then extraneous, too.  Unfortunately we
-    #  can't save the rule2func map.
-    #
-    def __getstate__(self):
-        if self.ruleschanged:
-            #
-            #  XXX - duplicated from parse()
-            #
-            self.computeNull()
-            self.newrules = {}
-            self.new2old = {}
-            self.makeNewRules()
-            self.ruleschanged = 0
-            self.edges, self.cores = {}, {}
-            self.states = { 0: self.makeState0() }
-            self.makeState(0, self._BOF)
-        #
-        #  XXX - should find a better way to do this..
-        #
-        changes = 1
-        while changes:
-            changes = 0
-            for k, v in self.edges.items():
-                if v is None:
-                    state, sym = k
-                    if self.states.has_key(state):
-                        self.goto(state, sym)
-                        changes = 1
-        rv = self.__dict__.copy()
-        for s in self.states.values():
-            del s.items
-        del rv['rule2func']
-        del rv['nullable']
-        del rv['cores']
-        return rv
-
-    def __setstate__(self, D):
-        self.rules = {}
-        self.rule2func = {}
-        self.rule2name = {}
-        self.collectRules()
-        start = D['rules'][self._START][0][1][1]        # Blech.
-        self.augment(start)
-        D['rule2func'] = self.rule2func
-        D['makeSet'] = self.makeSet_fast
-        self.__dict__ = D
-
-    #
-    #  A hook for GenericASTBuilder and GenericASTMatcher.  Mess
-    #  thee not with this; nor shall thee toucheth the _preprocess
-    #  argument to addRule.
-    #
-    def preprocess(self, rule, func):       return rule, func
-
-    def addRule(self, doc, func, _preprocess=1):
-        fn = func
-        rules = string.split(doc)
-
-        index = []
-        for i in range(len(rules)):
-            if rules[i] == '::=':
-                index.append(i-1)
-        index.append(len(rules))
-
-        for i in range(len(index)-1):
-            lhs = rules[index[i]]
-            rhs = rules[index[i]+2:index[i+1]]
-            rule = (lhs, tuple(rhs))
-
-            if _preprocess:
-                rule, fn = self.preprocess(rule, func)
-
-            if self.rules.has_key(lhs):
-                self.rules[lhs].append(rule)
-            else:
-                self.rules[lhs] = [ rule ]
-            self.rule2func[rule] = fn
-            self.rule2name[rule] = func.__name__[2:]
-        self.ruleschanged = 1
-
-    def collectRules(self):
-        for name in _namelist(self):
-            if name[:2] == 'p_':
-                func = getattr(self, name)
-                doc = func.__doc__
-                self.addRule(doc, func)
-
-    def augment(self, start):
-        rule = '%s ::= %s %s' % (self._START, self._BOF, start)
-        self.addRule(rule, lambda args: args[1], 0)
-
-    def computeNull(self):
-        self.nullable = {}
-        tbd = []
-
-        for rulelist in self.rules.values():
-            lhs = rulelist[0][0]
-            self.nullable[lhs] = 0
-            for rule in rulelist:
-                rhs = rule[1]
-                if len(rhs) == 0:
-                    self.nullable[lhs] = 1
-                    continue
-                #
-                #  We only need to consider rules which
-                #  consist entirely of nonterminal symbols.
-                #  This should be a savings on typical
-                #  grammars.
-                #
-                for sym in rhs:
-                    if not self.rules.has_key(sym):
-                        break
-                else:
-                    tbd.append(rule)
-        changes = 1
-        while changes:
-            changes = 0
-            for lhs, rhs in tbd:
-                if self.nullable[lhs]:
-                    continue
-                for sym in rhs:
-                    if not self.nullable[sym]:
-                        break
-                else:
-                    self.nullable[lhs] = 1
-                    changes = 1
-
-    def makeState0(self):
-        s0 = _State(0, [])
-        for rule in self.newrules[self._START]:
-            s0.items.append((rule, 0))
-        return s0
-
-    def finalState(self, tokens):
-        #
-        #  Yuck.
-        #
-        if len(self.newrules[self._START]) == 2 and len(tokens) == 0:
-            return 1
-        start = self.rules[self._START][0][1][1]
-        return self.goto(1, start)
-
-    def makeNewRules(self):
-        worklist = []
-        for rulelist in self.rules.values():
-            for rule in rulelist:
-                worklist.append((rule, 0, 1, rule))
-
-        for rule, i, candidate, oldrule in worklist:
-            lhs, rhs = rule
-            n = len(rhs)
-            while i < n:
-                sym = rhs[i]
-                if not self.rules.has_key(sym) or \
-                   not self.nullable[sym]:
-                    candidate = 0
-                    i = i + 1
-                    continue
-
-                newrhs = list(rhs)
-                newrhs[i] = self._NULLABLE+sym
-                newrule = (lhs, tuple(newrhs))
-                worklist.append((newrule, i+1,
-                                 candidate, oldrule))
-                candidate = 0
-                i = i + 1
-            else:
-                if candidate:
-                    lhs = self._NULLABLE+lhs
-                    rule = (lhs, rhs)
-                if self.newrules.has_key(lhs):
-                    self.newrules[lhs].append(rule)
-                else:
-                    self.newrules[lhs] = [ rule ]
-                self.new2old[rule] = oldrule
-
-    def typestring(self, token):
-        return None
-
-    def error(self, token):
-        print "Syntax error at or near `%s' token" % token
-        raise SystemExit
-
-    def parse(self, tokens):
-        sets = [ [(1,0), (2,0)] ]
-        self.links = {}
-
-        if self.ruleschanged:
-            self.computeNull()
-            self.newrules = {}
-            self.new2old = {}
-            self.makeNewRules()
-            self.ruleschanged = 0
-            self.edges, self.cores = {}, {}
-            self.states = { 0: self.makeState0() }
-            self.makeState(0, self._BOF)
-
-        for i in xrange(len(tokens)):
-            sets.append([])
-
-            if sets[i] == []:
-                break
-            self.makeSet(tokens[i], sets, i)
-        else:
-            sets.append([])
-            self.makeSet(None, sets, len(tokens))
-
-        #_dump(tokens, sets, self.states)
-
-        finalitem = (self.finalState(tokens), 0)
-        if finalitem not in sets[-2]:
-            if len(tokens) > 0:
-                self.error(tokens[i-1])
-            else:
-                self.error(None)
-
-        return self.buildTree(self._START, finalitem,
-                              tokens, len(sets)-2)
-
-    def isnullable(self, sym):
-        #
-        #  For symbols in G_e only.  If we weren't supporting 1.5,
-        #  could just use sym.startswith().
-        #
-        return self._NULLABLE == sym[0:len(self._NULLABLE)]
-
-    def skip(self, (lhs, rhs), pos=0):
-        n = len(rhs)
-        while pos < n:
-            if not self.isnullable(rhs[pos]):
-                break
-            pos = pos + 1
-        return pos
-
-    def makeState(self, state, sym):
-        assert sym is not None
-        #
-        #  Compute \epsilon-kernel state's core and see if
-        #  it exists already.
-        #
-        kitems = []
-        for rule, pos in self.states[state].items:
-            lhs, rhs = rule
-            if rhs[pos:pos+1] == (sym,):
-                kitems.append((rule, self.skip(rule, pos+1)))
-        core = kitems
-
-        core.sort()
-        tcore = tuple(core)
-        if self.cores.has_key(tcore):
-            return self.cores[tcore]
-        #
-        #  Nope, doesn't exist.  Compute it and the associated
-        #  \epsilon-nonkernel state together; we'll need it right away.
-        #
-        k = self.cores[tcore] = len(self.states)
-        K, NK = _State(k, kitems), _State(k+1, [])
-        self.states[k] = K
-        predicted = {}
-
-        edges = self.edges
-        rules = self.newrules
-        for X in K, NK:
-            worklist = X.items
-            for item in worklist:
-                rule, pos = item
-                lhs, rhs = rule
-                if pos == len(rhs):
-                    X.complete.append(rule)
-                    continue
-
-                nextSym = rhs[pos]
-                key = (X.stateno, nextSym)
-                if not rules.has_key(nextSym):
-                    if not edges.has_key(key):
-                        edges[key] = None
-                        X.T.append(nextSym)
-                else:
-                    edges[key] = None
-                    if not predicted.has_key(nextSym):
-                        predicted[nextSym] = 1
-                        for prule in rules[nextSym]:
-                            ppos = self.skip(prule)
-                            new = (prule, ppos)
-                            NK.items.append(new)
-            #
-            #  Problem: we know K needs generating, but we
-            #  don't yet know about NK.  Can't commit anything
-            #  regarding NK to self.edges until we're sure.  Should
-            #  we delay committing on both K and NK to avoid this
-            #  hacky code?  This creates other problems..
-            #
-            if X is K:
-                edges = {}
-
-        if NK.items == []:
-            return k
-
-        #
-        #  Check for \epsilon-nonkernel's core.  Unfortunately we
-        #  need to know the entire set of predicted nonterminals
-        #  to do this without accidentally duplicating states.
-        #
-        core = predicted.keys()
-        core.sort()
-        tcore = tuple(core)
-        if self.cores.has_key(tcore):
-            self.edges[(k, None)] = self.cores[tcore]
-            return k
-
-        nk = self.cores[tcore] = self.edges[(k, None)] = NK.stateno
-        self.edges.update(edges)
-        self.states[nk] = NK
-        return k
-
-    def goto(self, state, sym):
-        key = (state, sym)
-        if not self.edges.has_key(key):
-            #
-            #  No transitions from state on sym.
-            #
-            return None
-
-        rv = self.edges[key]
-        if rv is None:
-            #
-            #  Target state isn't generated yet.  Remedy this.
-            #
-            rv = self.makeState(state, sym)
-            self.edges[key] = rv
-        return rv
-
-    def gotoT(self, state, t):
-        return [self.goto(state, t)]
-
-    def gotoST(self, state, st):
-        rv = []
-        for t in self.states[state].T:
-            if st == t:
-                rv.append(self.goto(state, t))
-        return rv
-
-    def add(self, set, item, i=None, predecessor=None, causal=None):
-        if predecessor is None:
-            if item not in set:
-                set.append(item)
-        else:
-            key = (item, i)
-            if item not in set:
-                self.links[key] = []
-                set.append(item)
-            self.links[key].append((predecessor, causal))
-
-    def makeSet(self, token, sets, i):
-        cur, next = sets[i], sets[i+1]
-
-        ttype = token is not None and self.typestring(token) or None
-        if ttype is not None:
-            fn, arg = self.gotoT, ttype
-        else:
-            fn, arg = self.gotoST, token
-
-        for item in cur:
-            ptr = (item, i)
-            state, parent = item
-            add = fn(state, arg)
-            for k in add:
-                if k is not None:
-                    self.add(next, (k, parent), i+1, ptr)
-                    nk = self.goto(k, None)
-                    if nk is not None:
-                        self.add(next, (nk, i+1))
-
-            if parent == i:
-                continue
-
-            for rule in self.states[state].complete:
-                lhs, rhs = rule
-                for pitem in sets[parent]:
-                    pstate, pparent = pitem
-                    k = self.goto(pstate, lhs)
-                    if k is not None:
-                        why = (item, i, rule)
-                        pptr = (pitem, parent)
-                        self.add(cur, (k, pparent),
-                                 i, pptr, why)
-                        nk = self.goto(k, None)
-                        if nk is not None:
-                            self.add(cur, (nk, i))
-
-    def makeSet_fast(self, token, sets, i):
-        #
-        #  Call *only* when the entire state machine has been built!
-        #  It relies on self.edges being filled in completely, and
-        #  then duplicates and inlines code to boost speed at the
-        #  cost of extreme ugliness.
-        #
-        cur, next = sets[i], sets[i+1]
-        ttype = token is not None and self.typestring(token) or None
-
-        for item in cur:
-            ptr = (item, i)
-            state, parent = item
-            if ttype is not None:
-                k = self.edges.get((state, ttype), None)
-                if k is not None:
-                    #self.add(next, (k, parent), i+1, ptr)
-                    #INLINED --v
-                    new = (k, parent)
-                    key = (new, i+1)
-                    if new not in next:
-                        self.links[key] = []
-                        next.append(new)
-                    self.links[key].append((ptr, None))
-                    #INLINED --^
-                    #nk = self.goto(k, None)
-                    nk = self.edges.get((k, None), None)
-                    if nk is not None:
-                        #self.add(next, (nk, i+1))
-                        #INLINED --v
-                        new = (nk, i+1)
-                        if new not in next:
-                            next.append(new)
-                        #INLINED --^
-            else:
-                add = self.gotoST(state, token)
-                for k in add:
-                    if k is not None:
-                        self.add(next, (k, parent), i+1, ptr)
-                        #nk = self.goto(k, None)
-                        nk = self.edges.get((k, None), None)
-                        if nk is not None:
-                            self.add(next, (nk, i+1))
-
-            if parent == i:
-                continue
-
-            for rule in self.states[state].complete:
-                lhs, rhs = rule
-                for pitem in sets[parent]:
-                    pstate, pparent = pitem
-                    #k = self.goto(pstate, lhs)
-                    k = self.edges.get((pstate, lhs), None)
-                    if k is not None:
-                        why = (item, i, rule)
-                        pptr = (pitem, parent)
-                        #self.add(cur, (k, pparent),
-                        #        i, pptr, why)
-                        #INLINED --v
-                        new = (k, pparent)
-                        key = (new, i)
-                        if new not in cur:
-                            self.links[key] = []
-                            cur.append(new)
-                        self.links[key].append((pptr, why))
-                        #INLINED --^
-                        #nk = self.goto(k, None)
-                        nk = self.edges.get((k, None), None)
-                        if nk is not None:
-                            #self.add(cur, (nk, i))
-                            #INLINED --v
-                            new = (nk, i)
-                            if new not in cur:
-                                cur.append(new)
-                            #INLINED --^
-
-    def predecessor(self, key, causal):
-        for p, c in self.links[key]:
-            if c == causal:
-                return p
-        assert 0
-
-    def causal(self, key):
-        links = self.links[key]
-        if len(links) == 1:
-            return links[0][1]
-        choices = []
-        rule2cause = {}
-        for p, c in links:
-            rule = c[2]
-            choices.append(rule)
-            rule2cause[rule] = c
-        return rule2cause[self.ambiguity(choices)]
-
-    def deriveEpsilon(self, nt):
-        if len(self.newrules[nt]) > 1:
-            rule = self.ambiguity(self.newrules[nt])
-        else:
-            rule = self.newrules[nt][0]
-        #print rule
-
-        rhs = rule[1]
-        attr = [None] * len(rhs)
-
-        for i in range(len(rhs)-1, -1, -1):
-            attr[i] = self.deriveEpsilon(rhs[i])
-        return self.rule2func[self.new2old[rule]](attr)
-
-    def buildTree(self, nt, item, tokens, k):
-        state, parent = item
-
-        choices = []
-        for rule in self.states[state].complete:
-            if rule[0] == nt:
-                choices.append(rule)
-        rule = choices[0]
-        if len(choices) > 1:
-            rule = self.ambiguity(choices)
-        #print rule
-
-        rhs = rule[1]
-        attr = [None] * len(rhs)
-
-        for i in range(len(rhs)-1, -1, -1):
-            sym = rhs[i]
-            if not self.newrules.has_key(sym):
-                if sym != self._BOF:
-                    attr[i] = tokens[k-1]
-                    key = (item, k)
-                    item, k = self.predecessor(key, None)
-            #elif self.isnullable(sym):
-            elif self._NULLABLE == sym[0:len(self._NULLABLE)]:
-                attr[i] = self.deriveEpsilon(sym)
-            else:
-                key = (item, k)
-                why = self.causal(key)
-                attr[i] = self.buildTree(sym, why[0],
-                                         tokens, why[1])
-                item, k = self.predecessor(key, why)
-        return self.rule2func[self.new2old[rule]](attr)
-
-    def ambiguity(self, rules):
-        #
-        #  XXX - problem here and in collectRules() if the same rule
-        #        appears in >1 method.  Also undefined results if rules
-        #        causing the ambiguity appear in the same method.
-        #
-        sortlist = []
-        name2index = {}
-        for i in range(len(rules)):
-            lhs, rhs = rule = rules[i]
-            name = self.rule2name[self.new2old[rule]]
-            sortlist.append((len(rhs), name))
-            name2index[name] = i
-        sortlist.sort()
-        list = map(lambda (a,b): b, sortlist)
-        return rules[name2index[self.resolve(list)]]
-
-    def resolve(self, list):
-        #
-        #  Resolve ambiguity in favor of the shortest RHS.
-        #  Since we walk the tree from the top down, this
-        #  should effectively resolve in favor of a "shift".
-        #
-        return list[0]
-
-#
-#  GenericASTBuilder automagically constructs a concrete/abstract syntax tree
-#  for a given input.  The extra argument is a class (not an instance!)
-#  which supports the "__setslice__" and "__len__" methods.
-#
-#  XXX - silently overrides any user code in methods.
-#
-
-class GenericASTBuilder(GenericParser):
-    def __init__(self, AST, start):
-        GenericParser.__init__(self, start)
-        self.AST = AST
-
-    def preprocess(self, rule, func):

From pypy.commits at gmail.com  Mon Jan 23 17:48:32 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 23 Jan 2017 14:48:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58868840.499adf0a.b9ff3.756b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89710:9690cd452a80
Date: 2017-01-23 22:47 +0000
http://bitbucket.org/pypy/pypy/changeset/9690cd452a80/

Log:	hg merge default

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -138,7 +138,7 @@
         self.w_AST = space.gettypeobject(W_AST.typedef)
         for (name, base, fields, attributes) in self.AST_TYPES:
             self.make_new_type(space, name, base, fields, attributes)
-        
+
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
@@ -150,7 +150,7 @@
             space.setitem_str(w_dict, "_attributes",
                               space.newtuple([space.wrap(a) for a in attributes]))
         w_type = space.call_function(
-            space.w_type, 
+            space.w_type,
             space.wrap(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -8,6 +8,7 @@
 # please.
 import struct
 
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.astcompiler import ast, assemble, symtable, consts, misc
 from pypy.interpreter.astcompiler import optimize # For side effects
 from pypy.interpreter.pyparser.error import SyntaxError
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.error import OperationError
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.runicode import MAXUNICODE
+from rpython.rlib.objectmodel import specialize
 
 
 def optimize_ast(space, tree, compile_info):
@@ -177,6 +178,7 @@
         self.space = space
         self.compile_info = compile_info
 
+    @specialize.argtype(1)
     def default_visitor(self, node):
         return node
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl.py b/pypy/interpreter/astcompiler/tools/asdl.py
--- a/pypy/interpreter/astcompiler/tools/asdl.py
+++ b/pypy/interpreter/astcompiler/tools/asdl.py
@@ -33,7 +33,8 @@
 # See the EBNF at the top of the file to understand the logical connection
 # between the various node types.
 
-builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton'}
+builtin_types = {'identifier', 'string', 'bytes', 'int', 'bool', 'object',
+                 'singleton'}
 
 class AST:
     def __repr__(self):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -145,7 +145,7 @@
             if allow_none:
                 wrapper += " if %s is not None else space.w_None" % (value,)
             return wrapper
-        
+
     def get_value_extractor(self, field, value):
         if field.type in self.data.simple_types:
             return "%s.from_object(space, %s)" % (field.type, value)
@@ -207,7 +207,7 @@
                 lines.append("if _%s is None:" % (field.name,))
                 lines.append("    raise_required_value(space, w_node, '%s')"
                              % (field.name,))
-            
+
         return lines
 
     def make_converters(self, fields, name, extras=None):
@@ -245,7 +245,7 @@
             if extras:
                 base_args = ", ".join(str(field.name) for field in extras)
                 self.emit("%s.__init__(self, %s)" % (base, base_args), 2)
-    
+
     def make_mutate_over(self, cons, name):
         self.emit("def mutate_over(self, visitor):", 1)
         for field in cons.fields:
@@ -276,7 +276,7 @@
         self.emit("")
         self.make_mutate_over(cons, cons.name)
         self.make_converters(cons.fields, cons.name, extra_attributes)
-        self.emit("State.ast_type(%r, '%s', %s)" % 
+        self.emit("State.ast_type(%r, '%s', %s)" %
                   (cons.name, base, [f.name for f in cons.fields]))
         self.emit("")
 
@@ -357,7 +357,7 @@
         self.emit("")
 
     def visitField(self, field):
-        if (field.type not in asdl.builtin_types and 
+        if (field.type not in asdl.builtin_types and
             field.type not in self.data.simple_types):
             level = 2
             template = "node.%s.walkabout(self)"
@@ -558,7 +558,7 @@
         self.w_AST = space.gettypeobject(W_AST.typedef)
         for (name, base, fields, attributes) in self.AST_TYPES:
             self.make_new_type(space, name, base, fields, attributes)
-        
+
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
@@ -570,7 +570,7 @@
             space.setitem_str(w_dict, "_attributes",
                               space.newtuple([space.wrap(a) for a in attributes]))
         w_type = space.call_function(
-            space.w_type, 
+            space.w_type,
             space.wrap(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
diff --git a/pypy/interpreter/astcompiler/tools/spark.py b/pypy/interpreter/astcompiler/tools/spark.py
deleted file mode 100644
--- a/pypy/interpreter/astcompiler/tools/spark.py
+++ /dev/null
@@ -1,839 +0,0 @@
-#  Copyright (c) 1998-2002 John Aycock
-#
-#  Permission is hereby granted, free of charge, to any person obtaining
-#  a copy of this software and associated documentation files (the
-#  "Software"), to deal in the Software without restriction, including
-#  without limitation the rights to use, copy, modify, merge, publish,
-#  distribute, sublicense, and/or sell copies of the Software, and to
-#  permit persons to whom the Software is furnished to do so, subject to
-#  the following conditions:
-#
-#  The above copyright notice and this permission notice shall be
-#  included in all copies or substantial portions of the Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-#  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-#  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-#  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-__version__ = 'SPARK-0.7 (pre-alpha-5)'
-
-import re
-import string
-
-def _namelist(instance):
-    namelist, namedict, classlist = [], {}, [instance.__class__]
-    for c in classlist:
-        for b in c.__bases__:
-            classlist.append(b)
-        for name in c.__dict__.keys():
-            if not namedict.has_key(name):
-                namelist.append(name)
-                namedict[name] = 1
-    return namelist
-
-class GenericScanner:
-    def __init__(self, flags=0):
-        pattern = self.reflect()
-        self.re = re.compile(pattern, re.VERBOSE|flags)
-
-        self.index2func = {}
-        for name, number in self.re.groupindex.items():
-            self.index2func[number-1] = getattr(self, 't_' + name)
-
-    def makeRE(self, name):
-        doc = getattr(self, name).__doc__
-        rv = '(?P<%s>%s)' % (name[2:], doc)
-        return rv
-
-    def reflect(self):
-        rv = []
-        for name in _namelist(self):
-            if name[:2] == 't_' and name != 't_default':
-                rv.append(self.makeRE(name))
-
-        rv.append(self.makeRE('t_default'))
-        return string.join(rv, '|')
-
-    def error(self, s, pos):
-        print "Lexical error at position %s" % pos
-        raise SystemExit
-
-    def tokenize(self, s):
-        pos = 0
-        n = len(s)
-        while pos < n:
-            m = self.re.match(s, pos)
-            if m is None:
-                self.error(s, pos)
-
-            groups = m.groups()
-            for i in range(len(groups)):
-                if groups[i] and self.index2func.has_key(i):
-                    self.index2func[i](groups[i])
-            pos = m.end()
-
-    def t_default(self, s):
-        r'( . | \n )+'
-        print "Specification error: unmatched input"
-        raise SystemExit
-
-#
-#  Extracted from GenericParser and made global so that [un]picking works.
-#
-class _State:
-    def __init__(self, stateno, items):
-        self.T, self.complete, self.items = [], [], items
-        self.stateno = stateno
-
-class GenericParser:
-    #
-    #  An Earley parser, as per J. Earley, "An Efficient Context-Free
-    #  Parsing Algorithm", CACM 13(2), pp. 94-102.  Also J. C. Earley,
-    #  "An Efficient Context-Free Parsing Algorithm", Ph.D. thesis,
-    #  Carnegie-Mellon University, August 1968.  New formulation of
-    #  the parser according to J. Aycock, "Practical Earley Parsing
-    #  and the SPARK Toolkit", Ph.D. thesis, University of Victoria,
-    #  2001, and J. Aycock and R. N. Horspool, "Practical Earley
-    #  Parsing", unpublished paper, 2001.
-    #
-
-    def __init__(self, start):
-        self.rules = {}
-        self.rule2func = {}
-        self.rule2name = {}
-        self.collectRules()
-        self.augment(start)
-        self.ruleschanged = 1
-
-    _NULLABLE = '\e_'
-    _START = 'START'
-    _BOF = '|-'
-
-    #
-    #  When pickling, take the time to generate the full state machine;
-    #  some information is then extraneous, too.  Unfortunately we
-    #  can't save the rule2func map.
-    #
-    def __getstate__(self):
-        if self.ruleschanged:
-            #
-            #  XXX - duplicated from parse()
-            #
-            self.computeNull()
-            self.newrules = {}
-            self.new2old = {}
-            self.makeNewRules()
-            self.ruleschanged = 0
-            self.edges, self.cores = {}, {}
-            self.states = { 0: self.makeState0() }
-            self.makeState(0, self._BOF)
-        #
-        #  XXX - should find a better way to do this..
-        #
-        changes = 1
-        while changes:
-            changes = 0
-            for k, v in self.edges.items():
-                if v is None:
-                    state, sym = k
-                    if self.states.has_key(state):
-                        self.goto(state, sym)
-                        changes = 1
-        rv = self.__dict__.copy()
-        for s in self.states.values():
-            del s.items
-        del rv['rule2func']
-        del rv['nullable']
-        del rv['cores']
-        return rv
-
-    def __setstate__(self, D):
-        self.rules = {}
-        self.rule2func = {}
-        self.rule2name = {}
-        self.collectRules()
-        start = D['rules'][self._START][0][1][1]        # Blech.
-        self.augment(start)
-        D['rule2func'] = self.rule2func
-        D['makeSet'] = self.makeSet_fast
-        self.__dict__ = D
-
-    #
-    #  A hook for GenericASTBuilder and GenericASTMatcher.  Mess
-    #  thee not with this; nor shall thee toucheth the _preprocess
-    #  argument to addRule.
-    #
-    def preprocess(self, rule, func):       return rule, func
-
-    def addRule(self, doc, func, _preprocess=1):
-        fn = func
-        rules = string.split(doc)
-
-        index = []
-        for i in range(len(rules)):
-            if rules[i] == '::=':
-                index.append(i-1)
-        index.append(len(rules))
-
-        for i in range(len(index)-1):
-            lhs = rules[index[i]]
-            rhs = rules[index[i]+2:index[i+1]]
-            rule = (lhs, tuple(rhs))
-
-            if _preprocess:
-                rule, fn = self.preprocess(rule, func)
-
-            if self.rules.has_key(lhs):
-                self.rules[lhs].append(rule)
-            else:
-                self.rules[lhs] = [ rule ]
-            self.rule2func[rule] = fn
-            self.rule2name[rule] = func.__name__[2:]
-        self.ruleschanged = 1
-
-    def collectRules(self):
-        for name in _namelist(self):
-            if name[:2] == 'p_':
-                func = getattr(self, name)
-                doc = func.__doc__
-                self.addRule(doc, func)
-
-    def augment(self, start):
-        rule = '%s ::= %s %s' % (self._START, self._BOF, start)
-        self.addRule(rule, lambda args: args[1], 0)
-
-    def computeNull(self):
-        self.nullable = {}
-        tbd = []
-
-        for rulelist in self.rules.values():
-            lhs = rulelist[0][0]
-            self.nullable[lhs] = 0
-            for rule in rulelist:
-                rhs = rule[1]
-                if len(rhs) == 0:
-                    self.nullable[lhs] = 1
-                    continue
-                #
-                #  We only need to consider rules which
-                #  consist entirely of nonterminal symbols.
-                #  This should be a savings on typical
-                #  grammars.
-                #
-                for sym in rhs:
-                    if not self.rules.has_key(sym):
-                        break
-                else:
-                    tbd.append(rule)
-        changes = 1
-        while changes:
-            changes = 0
-            for lhs, rhs in tbd:
-                if self.nullable[lhs]:
-                    continue
-                for sym in rhs:
-                    if not self.nullable[sym]:
-                        break
-                else:
-                    self.nullable[lhs] = 1
-                    changes = 1
-
-    def makeState0(self):
-        s0 = _State(0, [])
-        for rule in self.newrules[self._START]:
-            s0.items.append((rule, 0))
-        return s0
-
-    def finalState(self, tokens):
-        #
-        #  Yuck.
-        #
-        if len(self.newrules[self._START]) == 2 and len(tokens) == 0:
-            return 1
-        start = self.rules[self._START][0][1][1]
-        return self.goto(1, start)
-
-    def makeNewRules(self):
-        worklist = []
-        for rulelist in self.rules.values():
-            for rule in rulelist:
-                worklist.append((rule, 0, 1, rule))
-
-        for rule, i, candidate, oldrule in worklist:
-            lhs, rhs = rule
-            n = len(rhs)
-            while i < n:
-                sym = rhs[i]
-                if not self.rules.has_key(sym) or \
-                   not self.nullable[sym]:
-                    candidate = 0
-                    i = i + 1
-                    continue
-
-                newrhs = list(rhs)
-                newrhs[i] = self._NULLABLE+sym
-                newrule = (lhs, tuple(newrhs))
-                worklist.append((newrule, i+1,
-                                 candidate, oldrule))
-                candidate = 0
-                i = i + 1
-            else:
-                if candidate:
-                    lhs = self._NULLABLE+lhs
-                    rule = (lhs, rhs)
-                if self.newrules.has_key(lhs):
-                    self.newrules[lhs].append(rule)
-                else:
-                    self.newrules[lhs] = [ rule ]
-                self.new2old[rule] = oldrule
-
-    def typestring(self, token):
-        return None
-
-    def error(self, token):
-        print "Syntax error at or near `%s' token" % token
-        raise SystemExit
-
-    def parse(self, tokens):
-        sets = [ [(1,0), (2,0)] ]
-        self.links = {}
-
-        if self.ruleschanged:
-            self.computeNull()
-            self.newrules = {}
-            self.new2old = {}
-            self.makeNewRules()
-            self.ruleschanged = 0
-            self.edges, self.cores = {}, {}
-            self.states = { 0: self.makeState0() }
-            self.makeState(0, self._BOF)
-
-        for i in xrange(len(tokens)):
-            sets.append([])
-
-            if sets[i] == []:
-                break
-            self.makeSet(tokens[i], sets, i)
-        else:
-            sets.append([])
-            self.makeSet(None, sets, len(tokens))
-
-        #_dump(tokens, sets, self.states)
-
-        finalitem = (self.finalState(tokens), 0)
-        if finalitem not in sets[-2]:
-            if len(tokens) > 0:
-                self.error(tokens[i-1])
-            else:
-                self.error(None)
-
-        return self.buildTree(self._START, finalitem,
-                              tokens, len(sets)-2)
-
-    def isnullable(self, sym):
-        #
-        #  For symbols in G_e only.  If we weren't supporting 1.5,
-        #  could just use sym.startswith().
-        #
-        return self._NULLABLE == sym[0:len(self._NULLABLE)]
-
-    def skip(self, (lhs, rhs), pos=0):
-        n = len(rhs)
-        while pos < n:
-            if not self.isnullable(rhs[pos]):
-                break
-            pos = pos + 1
-        return pos
-
-    def makeState(self, state, sym):
-        assert sym is not None
-        #
-        #  Compute \epsilon-kernel state's core and see if
-        #  it exists already.
-        #
-        kitems = []
-        for rule, pos in self.states[state].items:
-            lhs, rhs = rule
-            if rhs[pos:pos+1] == (sym,):
-                kitems.append((rule, self.skip(rule, pos+1)))
-        core = kitems
-
-        core.sort()
-        tcore = tuple(core)
-        if self.cores.has_key(tcore):
-            return self.cores[tcore]
-        #
-        #  Nope, doesn't exist.  Compute it and the associated
-        #  \epsilon-nonkernel state together; we'll need it right away.
-        #
-        k = self.cores[tcore] = len(self.states)
-        K, NK = _State(k, kitems), _State(k+1, [])
-        self.states[k] = K
-        predicted = {}
-
-        edges = self.edges
-        rules = self.newrules
-        for X in K, NK:
-            worklist = X.items
-            for item in worklist:
-                rule, pos = item
-                lhs, rhs = rule
-                if pos == len(rhs):
-                    X.complete.append(rule)
-                    continue
-
-                nextSym = rhs[pos]
-                key = (X.stateno, nextSym)
-                if not rules.has_key(nextSym):
-                    if not edges.has_key(key):
-                        edges[key] = None
-                        X.T.append(nextSym)
-                else:
-                    edges[key] = None
-                    if not predicted.has_key(nextSym):
-                        predicted[nextSym] = 1
-                        for prule in rules[nextSym]:
-                            ppos = self.skip(prule)
-                            new = (prule, ppos)
-                            NK.items.append(new)
-            #
-            #  Problem: we know K needs generating, but we
-            #  don't yet know about NK.  Can't commit anything
-            #  regarding NK to self.edges until we're sure.  Should
-            #  we delay committing on both K and NK to avoid this
-            #  hacky code?  This creates other problems..
-            #
-            if X is K:
-                edges = {}
-
-        if NK.items == []:
-            return k
-
-        #
-        #  Check for \epsilon-nonkernel's core.  Unfortunately we
-        #  need to know the entire set of predicted nonterminals
-        #  to do this without accidentally duplicating states.
-        #
-        core = predicted.keys()
-        core.sort()
-        tcore = tuple(core)
-        if self.cores.has_key(tcore):
-            self.edges[(k, None)] = self.cores[tcore]
-            return k
-
-        nk = self.cores[tcore] = self.edges[(k, None)] = NK.stateno
-        self.edges.update(edges)
-        self.states[nk] = NK
-        return k
-
-    def goto(self, state, sym):
-        key = (state, sym)
-        if not self.edges.has_key(key):
-            #
-            #  No transitions from state on sym.
-            #
-            return None
-
-        rv = self.edges[key]
-        if rv is None:
-            #
-            #  Target state isn't generated yet.  Remedy this.
-            #
-            rv = self.makeState(state, sym)
-            self.edges[key] = rv
-        return rv
-
-    def gotoT(self, state, t):
-        return [self.goto(state, t)]
-
-    def gotoST(self, state, st):
-        rv = []
-        for t in self.states[state].T:
-            if st == t:
-                rv.append(self.goto(state, t))
-        return rv
-
-    def add(self, set, item, i=None, predecessor=None, causal=None):
-        if predecessor is None:
-            if item not in set:
-                set.append(item)
-        else:
-            key = (item, i)
-            if item not in set:
-                self.links[key] = []
-                set.append(item)
-            self.links[key].append((predecessor, causal))
-
-    def makeSet(self, token, sets, i):
-        cur, next = sets[i], sets[i+1]
-
-        ttype = token is not None and self.typestring(token) or None
-        if ttype is not None:
-            fn, arg = self.gotoT, ttype
-        else:
-            fn, arg = self.gotoST, token
-
-        for item in cur:
-            ptr = (item, i)
-            state, parent = item
-            add = fn(state, arg)
-            for k in add:
-                if k is not None:
-                    self.add(next, (k, parent), i+1, ptr)
-                    nk = self.goto(k, None)
-                    if nk is not None:
-                        self.add(next, (nk, i+1))
-
-            if parent == i:
-                continue
-
-            for rule in self.states[state].complete:
-                lhs, rhs = rule
-                for pitem in sets[parent]:
-                    pstate, pparent = pitem
-                    k = self.goto(pstate, lhs)
-                    if k is not None:
-                        why = (item, i, rule)
-                        pptr = (pitem, parent)
-                        self.add(cur, (k, pparent),
-                                 i, pptr, why)
-                        nk = self.goto(k, None)
-                        if nk is not None:
-                            self.add(cur, (nk, i))
-
-    def makeSet_fast(self, token, sets, i):
-        #
-        #  Call *only* when the entire state machine has been built!
-        #  It relies on self.edges being filled in completely, and
-        #  then duplicates and inlines code to boost speed at the
-        #  cost of extreme ugliness.
-        #
-        cur, next = sets[i], sets[i+1]
-        ttype = token is not None and self.typestring(token) or None
-
-        for item in cur:
-            ptr = (item, i)
-            state, parent = item
-            if ttype is not None:
-                k = self.edges.get((state, ttype), None)
-                if k is not None:
-                    #self.add(next, (k, parent), i+1, ptr)
-                    #INLINED --v
-                    new = (k, parent)
-                    key = (new, i+1)
-                    if new not in next:
-                        self.links[key] = []
-                        next.append(new)
-                    self.links[key].append((ptr, None))
-                    #INLINED --^
-                    #nk = self.goto(k, None)
-                    nk = self.edges.get((k, None), None)
-                    if nk is not None:
-                        #self.add(next, (nk, i+1))
-                        #INLINED --v
-                        new = (nk, i+1)
-                        if new not in next:
-                            next.append(new)
-                        #INLINED --^
-            else:
-                add = self.gotoST(state, token)
-                for k in add:
-                    if k is not None:
-                        self.add(next, (k, parent), i+1, ptr)
-                        #nk = self.goto(k, None)
-                        nk = self.edges.get((k, None), None)
-                        if nk is not None:
-                            self.add(next, (nk, i+1))
-
-            if parent == i:
-                continue
-
-            for rule in self.states[state].complete:
-                lhs, rhs = rule
-                for pitem in sets[parent]:
-                    pstate, pparent = pitem
-                    #k = self.goto(pstate, lhs)
-                    k = self.edges.get((pstate, lhs), None)
-                    if k is not None:
-                        why = (item, i, rule)
-                        pptr = (pitem, parent)
-                        #self.add(cur, (k, pparent),
-                        #        i, pptr, why)
-                        #INLINED --v
-                        new = (k, pparent)
-                        key = (new, i)
-                        if new not in cur:
-                            self.links[key] = []
-                            cur.append(new)
-                        self.links[key].append((pptr, why))
-                        #INLINED --^
-                        #nk = self.goto(k, None)
-                        nk = self.edges.get((k, None), None)
-                        if nk is not None:
-                            #self.add(cur, (nk, i))
-                            #INLINED --v
-                            new = (nk, i)
-                            if new not in cur:
-                                cur.append(new)
-                            #INLINED --^
-
-    def predecessor(self, key, causal):
-        for p, c in self.links[key]:
-            if c == causal:
-                return p
-        assert 0
-
-    def causal(self, key):
-        links = self.links[key]
-        if len(links) == 1:
-            return links[0][1]
-        choices = []
-        rule2cause = {}
-        for p, c in links:
-            rule = c[2]
-            choices.append(rule)
-            rule2cause[rule] = c
-        return rule2cause[self.ambiguity(choices)]
-
-    def deriveEpsilon(self, nt):
-        if len(self.newrules[nt]) > 1:
-            rule = self.ambiguity(self.newrules[nt])
-        else:
-            rule = self.newrules[nt][0]
-        #print rule
-
-        rhs = rule[1]
-        attr = [None] * len(rhs)
-
-        for i in range(len(rhs)-1, -1, -1):
-            attr[i] = self.deriveEpsilon(rhs[i])
-        return self.rule2func[self.new2old[rule]](attr)
-
-    def buildTree(self, nt, item, tokens, k):
-        state, parent = item
-
-        choices = []
-        for rule in self.states[state].complete:
-            if rule[0] == nt:
-                choices.append(rule)
-        rule = choices[0]
-        if len(choices) > 1:
-            rule = self.ambiguity(choices)
-        #print rule
-
-        rhs = rule[1]
-        attr = [None] * len(rhs)
-
-        for i in range(len(rhs)-1, -1, -1):
-            sym = rhs[i]
-            if not self.newrules.has_key(sym):
-                if sym != self._BOF:
-                    attr[i] = tokens[k-1]
-                    key = (item, k)
-                    item, k = self.predecessor(key, None)
-            #elif self.isnullable(sym):
-            elif self._NULLABLE == sym[0:len(self._NULLABLE)]:
-                attr[i] = self.deriveEpsilon(sym)
-            else:
-                key = (item, k)
-                why = self.causal(key)
-                attr[i] = self.buildTree(sym, why[0],
-                                         tokens, why[1])
-                item, k = self.predecessor(key, why)
-        return self.rule2func[self.new2old[rule]](attr)
-
-    def ambiguity(self, rules):
-        #
-        #  XXX - problem here and in collectRules() if the same rule
-        #        appears in >1 method.  Also undefined results if rules
-        #        causing the ambiguity appear in the same method.
-        #
-        sortlist = []
-        name2index = {}
-        for i in range(len(rules)):
-            lhs, rhs = rule = rules[i]
-            name = self.rule2name[self.new2old[rule]]
-            sortlist.append((len(rhs), name))
-            name2index[name] = i
-        sortlist.sort()
-        list = map(lambda (a,b): b, sortlist)
-        return rules[name2index[self.resolve(list)]]
-
-    def resolve(self, list):
-        #
-        #  Resolve ambiguity in favor of the shortest RHS.
-        #  Since we walk the tree from the top down, this
-        #  should effectively resolve in favor of a "shift".
-        #
-        return list[0]
-
-#
-#  GenericASTBuilder automagically constructs a concrete/abstract syntax tree
-#  for a given input.  The extra argument is a class (not an instance!)
-#  which supports the "__setslice__" and "__len__" methods.
-#
-#  XXX - silently overrides any user code in methods.
-#
-
-class GenericASTBuilder(GenericParser):
-    def __init__(self, AST, start):
-        GenericParser.__init__(self, start)
-        self.AST = AST
-
-    def preprocess(self, rule, func):
-        rebind = lambda lhs, self=self: \
-                        lambda args, lhs=lhs, self=self: \
-                                self.buildASTNode(args, lhs)
-        lhs, rhs = rule
-        return rule, rebind(lhs)
-
-    def buildASTNode(self, args, lhs):
-        children = []
-        for arg in args:
-            if isinstance(arg, self.AST):
-                children.append(arg)
-            else:
-                children.append(self.terminal(arg))
-        return self.nonterminal(lhs, children)
-
-    def terminal(self, token):      return token
-
-    def nonterminal(self, type, args):
-        rv = self.AST(type)
-        rv[:len(args)] = args
-        return rv
-
-#
-#  GenericASTTraversal is a Visitor pattern according to Design Patterns.  For
-#  each node it attempts to invoke the method n_<node type>, falling
-#  back onto the default() method if the n_* can't be found.  The preorder
-#  traversal also looks for an exit hook named n_<node type>_exit (no default
-#  routine is called if it's not found).  To prematurely halt traversal
-#  of a subtree, call the prune() method -- this only makes sense for a
-#  preorder traversal.  Node type is determined via the typestring() method.
-#
-
-class GenericASTTraversalPruningException:
-    pass
-
-class GenericASTTraversal:
-    def __init__(self, ast):
-        self.ast = ast
-
-    def typestring(self, node):
-        return node.type
-
-    def prune(self):
-        raise GenericASTTraversalPruningException
-
-    def preorder(self, node=None):
-        if node is None:
-            node = self.ast
-
-        try:
-            name = 'n_' + self.typestring(node)
-            if hasattr(self, name):
-                func = getattr(self, name)
-                func(node)
-            else:
-                self.default(node)
-        except GenericASTTraversalPruningException:
-            return
-
-        for kid in node:
-            self.preorder(kid)
-
-        name = name + '_exit'
-        if hasattr(self, name):
-            func = getattr(self, name)
-            func(node)
-
-    def postorder(self, node=None):
-        if node is None:
-            node = self.ast
-
-        for kid in node:
-            self.postorder(kid)
-
-        name = 'n_' + self.typestring(node)
-        if hasattr(self, name):
-            func = getattr(self, name)
-            func(node)
-        else:
-            self.default(node)
-
-
-    def default(self, node):
-        pass
-
-#
-#  GenericASTMatcher.  AST nodes must have "__getitem__" and "__cmp__"
-#  implemented.
-#
-#  XXX - makes assumptions about how GenericParser walks the parse tree.
-#
-
-class GenericASTMatcher(GenericParser):
-    def __init__(self, start, ast):
-        GenericParser.__init__(self, start)
-        self.ast = ast
-
-    def preprocess(self, rule, func):
-        rebind = lambda func, self=self: \
-                        lambda args, func=func, self=self: \
-                                self.foundMatch(args, func)
-        lhs, rhs = rule
-        rhslist = list(rhs)
-        rhslist.reverse()
-
-        return (lhs, tuple(rhslist)), rebind(func)
-
-    def foundMatch(self, args, func):
-        func(args[-1])
-        return args[-1]
-
-    def match_r(self, node):
-        self.input.insert(0, node)
-        children = 0
-
-        for child in node:
-            if children == 0:
-                self.input.insert(0, '(')
-            children = children + 1
-            self.match_r(child)
-
-        if children > 0:
-            self.input.insert(0, ')')
-
-    def match(self, ast=None):
-        if ast is None:
-            ast = self.ast
-        self.input = []
-
-        self.match_r(ast)
-        self.parse(self.input)
-
-    def resolve(self, list):
-        #
-        #  Resolve ambiguity in favor of the longest RHS.
-        #
-        return list[-1]
-
-def _dump(tokens, sets, states):
-    for i in range(len(sets)):
-        print 'set', i
-        for item in sets[i]:
-            print '\t', item
-            for (lhs, rhs), pos in states[item[0]].items:
-                print '\t\t', lhs, '::=',
-                print string.join(rhs[:pos]),
-                print '.',
-                print string.join(rhs[pos:])
-        if i < len(tokens):
-            print
-            print 'token', str(tokens[i])
-            print
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -763,6 +763,15 @@
         finally:
             os.rmdir(name)
 
+    def test_dir_with_only_pyw(self):
+        def imp():
+            import onlypyw
+        raises(ImportError, imp)
+
+    @pytest.mark.skipif(not hasattr(py.path.local, "mksymlinkto"), reason="requires symlinks")
+    def test_dev_null_init_file(self):
+        import devnullpkg
+
 
 class TestAbi:
     def test_abi_tag(self):
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -16,7 +16,11 @@
         if not have_debug_prints():
             return
         inputargs, ops = self._unpack_trace(trace)
-        self.log_loop(inputargs, ops, memo=memo)
+        debug_start("jit-log-noopt")
+        debug_print("# Traced loop or bridge with", len(ops), "ops")
+        logops = self._log_operations(inputargs, ops, None, memo)
+        debug_stop("jit-log-noopt")
+        return logops
 
     def _unpack_trace(self, trace):
         ops = []
@@ -28,6 +32,7 @@
     def log_loop(self, inputargs, operations, number=0, type=None,
                  ops_offset=None, name='', memo=None):
         if type is None:
+            # XXX this case not normally used any more, I think
             debug_start("jit-log-noopt-loop")
             debug_print("# Loop", number, '(%s)' % name, ":", "noopt",
                         "with", len(operations), "ops")
@@ -58,6 +63,7 @@
     def log_bridge(self, inputargs, operations, extra=None,
                    descr=None, ops_offset=None, memo=None):
         if extra == "noopt":
+            # XXX this case no longer used
             debug_start("jit-log-noopt-bridge")
             debug_print("# bridge out of Guard",
                         "0x%x" % compute_unique_id(descr),
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -52,12 +52,19 @@
         jd.jitdriver.is_recursive = True
     else:
         count_recursive = 0
+        invalid = 0
         for jd in warmrunnerdesc.jitdrivers_sd:
             count_recursive += jd.jitdriver.is_recursive
+            invalid += (jd.jitdriver.has_unique_id and
+                           not jd.jitdriver.is_recursive)
         if count_recursive == 0:
             raise Exception("if you have more than one jitdriver, at least"
                 " one of them has to be marked with is_recursive=True,"
                 " none found")
+        if invalid > 0:
+            raise Exception("found %d jitdriver(s) with 'get_unique_id=...' "
+                            "specified but without 'is_recursive=True'" %
+                            (invalid,))
     for jd in warmrunnerdesc.jitdrivers_sd:
         jd.warmstate.set_param_inlining(inline)
         jd.warmstate.set_param_vec(vec)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2226,7 +2226,7 @@
         #
         # 'threshold_objects_made_old', is used inside comparisons
         # with 'size_objects_made_old' to know when we must do
-        # several major GC steps (i.e. several consecurive calls
+        # several major GC steps (i.e. several consecutive calls
         # to the present function).  Here is the target that
         # we try to aim to: either (A1) or (A2)
         #
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -637,8 +637,6 @@
             raise AttributeError("no 'greens' or 'reds' supplied")
         if virtualizables is not None:
             self.virtualizables = virtualizables
-        if get_unique_id is not None:
-            assert is_recursive, "get_unique_id and is_recursive must be specified at the same time"
         for v in self.virtualizables:
             assert v in self.reds
         # if reds are automatic, they won't be passed to jit_merge_point, so
@@ -653,6 +651,7 @@
         assert set_jitcell_at is None, "set_jitcell_at no longer used"
         self.get_printable_location = get_printable_location
         self.get_location = get_location
+        self.has_unique_id = (get_unique_id is not None)
         if get_unique_id is None:
             get_unique_id = lambda *args: 0
         self.get_unique_id = get_unique_id
diff --git a/rpython/rlib/rvmprof/README.txt b/rpython/rlib/rvmprof/README.txt
--- a/rpython/rlib/rvmprof/README.txt
+++ b/rpython/rlib/rvmprof/README.txt
@@ -2,8 +2,6 @@
 VMProf: a profiler for RPython VMs
 ==================================
 
-**Limited to 64-bit Linux.**
-
 
 from rpython.rlib import rvmprof
 
@@ -11,85 +9,36 @@
 Your VM must have an interpreter for "code" objects, which can be
 of any RPython instance.
 
-Use this decorator on the mainloop of the interpreter, to tell vmprof
+Use this as a decorator on the mainloop of the interpreter, to tell vmprof
 when you enter and leave a "code" object:
 
     def vmprof_execute_code(name, get_code_fn, result_class=None):
-        """Decorator to be used on the function that interprets a code object.
 
-        'name' must be a unique name.
-
-        'get_code_fn(*args)' is called to extract the code object from the
-        arguments given to the decorated function.
-
-        The original function can return None, an integer, or an instance.
-        In the latter case (only), 'result_class' must be set.
-
-        NOTE: for now, this assumes that the decorated functions only takes
-        instances or plain integer arguments, and at most 5 of them
-        (including 'self' if applicable).
-        """
+See its docstring in rvmprof.py.
 
 
 The class of code objects needs to be registered by calling the
-following function (once, before translation):
+function ``rpython.rlib.rvmprof.register_code_object_class()``
+(once, before translation).  It is a global function in __init__.py,
+but see the docstring of the method in rvmprof.py.
 
-    def register_code_object_class(CodeClass, full_name_func):
-        """NOT_RPYTHON
-        Register statically the class 'CodeClass' as containing user
-        code objects.
 
-        full_name_func() is a function called at runtime with an
-        instance of CodeClass and it should return a string.  This
-        is the string stored in the vmprof file identifying the code
-        object.  It can be directly an unbound method of CodeClass.
-        IMPORTANT: the name returned must be at most MAX_FUNC_NAME
-        characters long, and with exactly 3 colons, i.e. of the form
+To support adding new code objects at run-time, whenever a code object is
+instantiated, call the function ``rpython.rlib.rvmprof.register_code()``.
 
-            class:func_name:func_line:filename
+If you need JIT support, you also need to add a jitdriver method
+``get_unique_id(*greenkey)``, where you call
+``rpython.rlib.rvmprof.get_unique_code()``.
 
-        where 'class' is 'py' for PyPy.
 
-        Instances of the CodeClass will have a new attribute called
-        '_vmprof_unique_id', but that's managed internally.
-        """
-
-
-To support adding new code objects at run-time, whenever a code
-object is instantiated, call this function:
-
-    @specialize.argtype(1)
-    def register_code(code, name):
-        """Register the code object.  Call when a new code object is made.
-        """
-
-    @specialize.argtype(1)
-    def get_unique_id(code):
-        """Return the internal unique ID of a code object.  Can only be
-        called after register_code().  Call this in the jitdriver's
-        method 'get_unique_id(*greenkey)'.  Always returns 0 if we
-        didn't call register_code_object_class() on the class.
-        """
-
-
-Enable the profiler with:
+Enable/disable the profiler at runtime with:
 
     def enable(fileno, interval):
-        """Enable vmprof.  Writes go to the given 'fileno'.
-        The sampling interval is given by 'interval' as a number of
-        seconds, as a float which must be smaller than 1.0.
-        Raises VMProfError if something goes wrong.
-        """
-
+    def disable():
 
 The file descriptor must remain open until the profiler is disabled.
 The profiler must be disabled before the program exit, otherwise the
-file is incompletely written.  Disable it with:
+file is incompletely written.
 
-    def disable():
-        """Disable vmprof.
-        Raises VMProfError if something goes wrong.
-        """
-
-You should close the file descriptor afterwards; it is not
-automatically closed.
+You should close the file descriptor after disabling the profiler; it is
+not automatically closed.
diff --git a/rpython/rlib/types.py b/rpython/rlib/types.py
--- a/rpython/rlib/types.py
+++ b/rpython/rlib/types.py
@@ -76,8 +76,8 @@
     return model.SomeDict(dictdef)
 
 
-def instance(cls):
-    return lambda bookkeeper: model.SomeInstance(bookkeeper.getuniqueclassdef(cls))
+def instance(cls, can_be_None=False):
+    return lambda bookkeeper: model.SomeInstance(bookkeeper.getuniqueclassdef(cls), can_be_None=can_be_None)
 
 
 class SelfTypeMarker(object):

From pypy.commits at gmail.com  Mon Jan 23 19:03:44 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 23 Jan 2017 16:03:44 -0800 (PST)
Subject: [pypy-commit] pypy default: Kill _mutate_sequence() since it mixes
 random lists of AST nodes together
Message-ID: <588699e0.6f98df0a.d4ac7.8f12@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89711:a69be52268a9
Date: 2017-01-24 00:03 +0000
http://bitbucket.org/pypy/pypy/changeset/a69be52268a9/

Log:	Kill _mutate_sequence() since it mixes random lists of AST nodes
	together

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -31,6 +31,7 @@
 
 class AST(object):
     __metaclass__ = extendabletype
+    _attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")
@@ -184,7 +185,8 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_Module(self)
 
     def to_object(self, space):
@@ -217,7 +219,8 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_Interactive(self)
 
     def to_object(self, space):
@@ -279,7 +282,8 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_Suite(self)
 
     def to_object(self, space):
@@ -377,9 +381,11 @@
     def mutate_over(self, visitor):
         self.args = self.args.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         if self.decorator_list:
-            visitor._mutate_sequence(self.decorator_list)
+            for i in range(len(self.decorator_list)):
+                self.decorator_list[i] = self.decorator_list[i].mutate_over(visitor)
         return visitor.visit_FunctionDef(self)
 
     def to_object(self, space):
@@ -445,11 +451,14 @@
 
     def mutate_over(self, visitor):
         if self.bases:
-            visitor._mutate_sequence(self.bases)
+            for i in range(len(self.bases)):
+                self.bases[i] = self.bases[i].mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         if self.decorator_list:
-            visitor._mutate_sequence(self.decorator_list)
+            for i in range(len(self.decorator_list)):
+                self.decorator_list[i] = self.decorator_list[i].mutate_over(visitor)
         return visitor.visit_ClassDef(self)
 
     def to_object(self, space):
@@ -552,7 +561,8 @@
 
     def mutate_over(self, visitor):
         if self.targets:
-            visitor._mutate_sequence(self.targets)
+            for i in range(len(self.targets)):
+                self.targets[i] = self.targets[i].mutate_over(visitor)
         return visitor.visit_Delete(self)
 
     def to_object(self, space):
@@ -595,7 +605,8 @@
 
     def mutate_over(self, visitor):
         if self.targets:
-            visitor._mutate_sequence(self.targets)
+            for i in range(len(self.targets)):
+                self.targets[i] = self.targets[i].mutate_over(visitor)
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Assign(self)
 
@@ -701,7 +712,8 @@
         if self.dest:
             self.dest = self.dest.mutate_over(visitor)
         if self.values:
-            visitor._mutate_sequence(self.values)
+            for i in range(len(self.values)):
+                self.values[i] = self.values[i].mutate_over(visitor)
         return visitor.visit_Print(self)
 
     def to_object(self, space):
@@ -758,9 +770,11 @@
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_For(self)
 
     def to_object(self, space):
@@ -826,9 +840,11 @@
     def mutate_over(self, visitor):
         self.test = self.test.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_While(self)
 
     def to_object(self, space):
@@ -888,9 +904,11 @@
     def mutate_over(self, visitor):
         self.test = self.test.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_If(self)
 
     def to_object(self, space):
@@ -952,7 +970,8 @@
         if self.optional_vars:
             self.optional_vars = self.optional_vars.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_With(self)
 
     def to_object(self, space):
@@ -1057,11 +1076,14 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         if self.handlers:
-            visitor._mutate_sequence(self.handlers)
+            for i in range(len(self.handlers)):
+                self.handlers[i] = self.handlers[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_TryExcept(self)
 
     def to_object(self, space):
@@ -1122,9 +1144,11 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         if self.finalbody:
-            visitor._mutate_sequence(self.finalbody)
+            for i in range(len(self.finalbody)):
+                self.finalbody[i] = self.finalbody[i].mutate_over(visitor)
         return visitor.visit_TryFinally(self)
 
     def to_object(self, space):
@@ -1220,7 +1244,8 @@
 
     def mutate_over(self, visitor):
         if self.names:
-            visitor._mutate_sequence(self.names)
+            for i in range(len(self.names)):
+                self.names[i] = self.names[i].mutate_over(visitor)
         return visitor.visit_Import(self)
 
     def to_object(self, space):
@@ -1264,7 +1289,8 @@
 
     def mutate_over(self, visitor):
         if self.names:
-            visitor._mutate_sequence(self.names)
+            for i in range(len(self.names)):
+                self.names[i] = self.names[i].mutate_over(visitor)
         return visitor.visit_ImportFrom(self)
 
     def to_object(self, space):
@@ -1595,7 +1621,8 @@
 
     def mutate_over(self, visitor):
         if self.values:
-            visitor._mutate_sequence(self.values)
+            for i in range(len(self.values)):
+                self.values[i] = self.values[i].mutate_over(visitor)
         return visitor.visit_BoolOp(self)
 
     def to_object(self, space):
@@ -1842,9 +1869,11 @@
 
     def mutate_over(self, visitor):
         if self.keys:
-            visitor._mutate_sequence(self.keys)
+            for i in range(len(self.keys)):
+                self.keys[i] = self.keys[i].mutate_over(visitor)
         if self.values:
-            visitor._mutate_sequence(self.values)
+            for i in range(len(self.values)):
+                self.values[i] = self.values[i].mutate_over(visitor)
         return visitor.visit_Dict(self)
 
     def to_object(self, space):
@@ -1895,7 +1924,8 @@
 
     def mutate_over(self, visitor):
         if self.elts:
-            visitor._mutate_sequence(self.elts)
+            for i in range(len(self.elts)):
+                self.elts[i] = self.elts[i].mutate_over(visitor)
         return visitor.visit_Set(self)
 
     def to_object(self, space):
@@ -1939,7 +1969,8 @@
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_ListComp(self)
 
     def to_object(self, space):
@@ -1989,7 +2020,8 @@
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_SetComp(self)
 
     def to_object(self, space):
@@ -2041,7 +2073,8 @@
         self.key = self.key.mutate_over(visitor)
         self.value = self.value.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_DictComp(self)
 
     def to_object(self, space):
@@ -2097,7 +2130,8 @@
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_GeneratorExp(self)
 
     def to_object(self, space):
@@ -2185,7 +2219,8 @@
     def mutate_over(self, visitor):
         self.left = self.left.mutate_over(visitor)
         if self.comparators:
-            visitor._mutate_sequence(self.comparators)
+            for i in range(len(self.comparators)):
+                self.comparators[i] = self.comparators[i].mutate_over(visitor)
         return visitor.visit_Compare(self)
 
     def to_object(self, space):
@@ -2247,9 +2282,11 @@
     def mutate_over(self, visitor):
         self.func = self.func.mutate_over(visitor)
         if self.args:
-            visitor._mutate_sequence(self.args)
+            for i in range(len(self.args)):
+                self.args[i] = self.args[i].mutate_over(visitor)
         if self.keywords:
-            visitor._mutate_sequence(self.keywords)
+            for i in range(len(self.keywords)):
+                self.keywords[i] = self.keywords[i].mutate_over(visitor)
         if self.starargs:
             self.starargs = self.starargs.mutate_over(visitor)
         if self.kwargs:
@@ -2580,7 +2617,8 @@
 
     def mutate_over(self, visitor):
         if self.elts:
-            visitor._mutate_sequence(self.elts)
+            for i in range(len(self.elts)):
+                self.elts[i] = self.elts[i].mutate_over(visitor)
         return visitor.visit_List(self)
 
     def to_object(self, space):
@@ -2629,7 +2667,8 @@
 
     def mutate_over(self, visitor):
         if self.elts:
-            visitor._mutate_sequence(self.elts)
+            for i in range(len(self.elts)):
+                self.elts[i] = self.elts[i].mutate_over(visitor)
         return visitor.visit_Tuple(self)
 
     def to_object(self, space):
@@ -2857,7 +2896,8 @@
 
     def mutate_over(self, visitor):
         if self.dims:
-            visitor._mutate_sequence(self.dims)
+            for i in range(len(self.dims)):
+                self.dims[i] = self.dims[i].mutate_over(visitor)
         return visitor.visit_ExtSlice(self)
 
     def to_object(self, space):
@@ -3216,7 +3256,8 @@
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
         if self.ifs:
-            visitor._mutate_sequence(self.ifs)
+            for i in range(len(self.ifs)):
+                self.ifs[i] = self.ifs[i].mutate_over(visitor)
         return visitor.visit_comprehension(self)
 
     def walkabout(self, visitor):
@@ -3286,7 +3327,8 @@
         if self.name:
             self.name = self.name.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_ExceptHandler(self)
 
     def to_object(self, space):
@@ -3335,9 +3377,11 @@
 
     def mutate_over(self, visitor):
         if self.args:
-            visitor._mutate_sequence(self.args)
+            for i in range(len(self.args)):
+                self.args[i] = self.args[i].mutate_over(visitor)
         if self.defaults:
-            visitor._mutate_sequence(self.defaults)
+            for i in range(len(self.defaults)):
+                self.defaults[i] = self.defaults[i].mutate_over(visitor)
         return visitor.visit_arguments(self)
 
     def walkabout(self, visitor):
@@ -3456,10 +3500,6 @@
     def default_visitor(self, node):
         raise NodeVisitorNotImplemented
 
-    def _mutate_sequence(self, seq):
-        for i in range(len(seq)):
-            seq[i] = seq[i].mutate_over(self)
-
     def visit_Module(self, node):
         return self.default_visitor(node)
     def visit_Interactive(self, node):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -233,12 +233,17 @@
                 else:
                     level = 2
                 if field.seq:
-                    sub = (field.name,)
-                    self.emit("visitor._mutate_sequence(self.%s)" % sub, level)
+                    sub = field.name
+                    self.emit("for i in range(len(self.{})):".format(sub),
+                        level)
+                    self.emit(
+                        "self.{0}[i] = self.{0}[i].mutate_over(visitor)".format(sub),
+                        level + 1)
                 else:
-                    sub = (field.name, field.name)
-                    self.emit("self.%s = self.%s.mutate_over(visitor)" % sub,
-                              level)
+                    sub = field.name
+                    self.emit(
+                        "self.{0} = self.{0}.mutate_over(visitor)".format(sub),
+                        level)
         self.emit("return visitor.visit_%s(self)" % (name,), 2)
         self.emit("")
 
@@ -274,10 +279,6 @@
         self.emit("def default_visitor(self, node):", 1)
         self.emit("raise NodeVisitorNotImplemented", 2)
         self.emit("")
-        self.emit("def _mutate_sequence(self, seq):", 1)
-        self.emit("for i in range(len(seq)):", 2)
-        self.emit("seq[i] = seq[i].mutate_over(self)", 3)
-        self.emit("")
         super(ASTVisitorVisitor, self).visitModule(mod)
         self.emit("")
 
@@ -419,6 +420,7 @@
 
 class AST(object):
     __metaclass__ = extendabletype
+    _attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")

From pypy.commits at gmail.com  Mon Jan 23 19:15:07 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 23 Jan 2017 16:15:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58869c8b.cda0df0a.22ee7.5bc4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89712:686b6385285d
Date: 2017-01-24 00:14 +0000
http://bitbucket.org/pypy/pypy/changeset/686b6385285d/

Log:	hg merge default

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -31,6 +31,7 @@
 
 class AST(object):
     __metaclass__ = extendabletype
+    #_attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")
@@ -184,7 +185,9 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_Module(self)
 
     def to_object(self, space):
@@ -217,7 +220,9 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_Interactive(self)
 
     def to_object(self, space):
@@ -279,7 +284,9 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_Suite(self)
 
     def to_object(self, space):
@@ -380,9 +387,13 @@
     def mutate_over(self, visitor):
         self.args = self.args.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.decorator_list:
-            visitor._mutate_sequence(self.decorator_list)
+            for i in range(len(self.decorator_list)):
+                if self.decorator_list[i] is not None:
+                    self.decorator_list[i] = self.decorator_list[i].mutate_over(visitor)
         if self.returns:
             self.returns = self.returns.mutate_over(visitor)
         return visitor.visit_FunctionDef(self)
@@ -456,9 +467,13 @@
     def mutate_over(self, visitor):
         self.args = self.args.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.decorator_list:
-            visitor._mutate_sequence(self.decorator_list)
+            for i in range(len(self.decorator_list)):
+                if self.decorator_list[i] is not None:
+                    self.decorator_list[i] = self.decorator_list[i].mutate_over(visitor)
         if self.returns:
             self.returns = self.returns.mutate_over(visitor)
         return visitor.visit_AsyncFunctionDef(self)
@@ -531,13 +546,21 @@
 
     def mutate_over(self, visitor):
         if self.bases:
-            visitor._mutate_sequence(self.bases)
+            for i in range(len(self.bases)):
+                if self.bases[i] is not None:
+                    self.bases[i] = self.bases[i].mutate_over(visitor)
         if self.keywords:
-            visitor._mutate_sequence(self.keywords)
+            for i in range(len(self.keywords)):
+                if self.keywords[i] is not None:
+                    self.keywords[i] = self.keywords[i].mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.decorator_list:
-            visitor._mutate_sequence(self.decorator_list)
+            for i in range(len(self.decorator_list)):
+                if self.decorator_list[i] is not None:
+                    self.decorator_list[i] = self.decorator_list[i].mutate_over(visitor)
         return visitor.visit_ClassDef(self)
 
     def to_object(self, space):
@@ -649,7 +672,9 @@
 
     def mutate_over(self, visitor):
         if self.targets:
-            visitor._mutate_sequence(self.targets)
+            for i in range(len(self.targets)):
+                if self.targets[i] is not None:
+                    self.targets[i] = self.targets[i].mutate_over(visitor)
         return visitor.visit_Delete(self)
 
     def to_object(self, space):
@@ -692,7 +717,9 @@
 
     def mutate_over(self, visitor):
         if self.targets:
-            visitor._mutate_sequence(self.targets)
+            for i in range(len(self.targets)):
+                if self.targets[i] is not None:
+                    self.targets[i] = self.targets[i].mutate_over(visitor)
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Assign(self)
 
@@ -799,9 +826,13 @@
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_For(self)
 
     def to_object(self, space):
@@ -869,9 +900,13 @@
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_AsyncFor(self)
 
     def to_object(self, space):
@@ -937,9 +972,13 @@
     def mutate_over(self, visitor):
         self.test = self.test.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_While(self)
 
     def to_object(self, space):
@@ -999,9 +1038,13 @@
     def mutate_over(self, visitor):
         self.test = self.test.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_If(self)
 
     def to_object(self, space):
@@ -1059,9 +1102,13 @@
 
     def mutate_over(self, visitor):
         if self.items:
-            visitor._mutate_sequence(self.items)
+            for i in range(len(self.items)):
+                if self.items[i] is not None:
+                    self.items[i] = self.items[i].mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_With(self)
 
     def to_object(self, space):
@@ -1113,9 +1160,13 @@
 
     def mutate_over(self, visitor):
         if self.items:
-            visitor._mutate_sequence(self.items)
+            for i in range(len(self.items)):
+                if self.items[i] is not None:
+                    self.items[i] = self.items[i].mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_AsyncWith(self)
 
     def to_object(self, space):
@@ -1213,13 +1264,21 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.handlers:
-            visitor._mutate_sequence(self.handlers)
+            for i in range(len(self.handlers)):
+                if self.handlers[i] is not None:
+                    self.handlers[i] = self.handlers[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         if self.finalbody:
-            visitor._mutate_sequence(self.finalbody)
+            for i in range(len(self.finalbody)):
+                if self.finalbody[i] is not None:
+                    self.finalbody[i] = self.finalbody[i].mutate_over(visitor)
         return visitor.visit_Try(self)
 
     def to_object(self, space):
@@ -1333,7 +1392,9 @@
 
     def mutate_over(self, visitor):
         if self.names:
-            visitor._mutate_sequence(self.names)
+            for i in range(len(self.names)):
+                if self.names[i] is not None:
+                    self.names[i] = self.names[i].mutate_over(visitor)
         return visitor.visit_Import(self)
 
     def to_object(self, space):
@@ -1377,7 +1438,9 @@
 
     def mutate_over(self, visitor):
         if self.names:
-            visitor._mutate_sequence(self.names)
+            for i in range(len(self.names)):
+                if self.names[i] is not None:
+                    self.names[i] = self.names[i].mutate_over(visitor)
         return visitor.visit_ImportFrom(self)
 
     def to_object(self, space):
@@ -1706,7 +1769,9 @@
 
     def mutate_over(self, visitor):
         if self.values:
-            visitor._mutate_sequence(self.values)
+            for i in range(len(self.values)):
+                if self.values[i] is not None:
+                    self.values[i] = self.values[i].mutate_over(visitor)
         return visitor.visit_BoolOp(self)
 
     def to_object(self, space):
@@ -1953,9 +2018,13 @@
 
     def mutate_over(self, visitor):
         if self.keys:
-            visitor._mutate_sequence(self.keys)
+            for i in range(len(self.keys)):
+                if self.keys[i] is not None:
+                    self.keys[i] = self.keys[i].mutate_over(visitor)
         if self.values:
-            visitor._mutate_sequence(self.values)
+            for i in range(len(self.values)):
+                if self.values[i] is not None:
+                    self.values[i] = self.values[i].mutate_over(visitor)
         return visitor.visit_Dict(self)
 
     def to_object(self, space):
@@ -2006,7 +2075,9 @@
 
     def mutate_over(self, visitor):
         if self.elts:
-            visitor._mutate_sequence(self.elts)
+            for i in range(len(self.elts)):
+                if self.elts[i] is not None:
+                    self.elts[i] = self.elts[i].mutate_over(visitor)
         return visitor.visit_Set(self)
 
     def to_object(self, space):
@@ -2050,7 +2121,9 @@
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                if self.generators[i] is not None:
+                    self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_ListComp(self)
 
     def to_object(self, space):
@@ -2100,7 +2173,9 @@
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                if self.generators[i] is not None:
+                    self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_SetComp(self)
 
     def to_object(self, space):
@@ -2152,7 +2227,9 @@
         self.key = self.key.mutate_over(visitor)
         self.value = self.value.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                if self.generators[i] is not None:
+                    self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_DictComp(self)
 
     def to_object(self, space):
@@ -2208,7 +2285,9 @@
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                if self.generators[i] is not None:
+                    self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_GeneratorExp(self)
 
     def to_object(self, space):
@@ -2372,7 +2451,9 @@
     def mutate_over(self, visitor):
         self.left = self.left.mutate_over(visitor)
         if self.comparators:
-            visitor._mutate_sequence(self.comparators)
+            for i in range(len(self.comparators)):
+                if self.comparators[i] is not None:
+                    self.comparators[i] = self.comparators[i].mutate_over(visitor)
         return visitor.visit_Compare(self)
 
     def to_object(self, space):
@@ -2432,9 +2513,13 @@
     def mutate_over(self, visitor):
         self.func = self.func.mutate_over(visitor)
         if self.args:
-            visitor._mutate_sequence(self.args)
+            for i in range(len(self.args)):
+                if self.args[i] is not None:
+                    self.args[i] = self.args[i].mutate_over(visitor)
         if self.keywords:
-            visitor._mutate_sequence(self.keywords)
+            for i in range(len(self.keywords)):
+                if self.keywords[i] is not None:
+                    self.keywords[i] = self.keywords[i].mutate_over(visitor)
         return visitor.visit_Call(self)
 
     def to_object(self, space):
@@ -2864,7 +2949,9 @@
 
     def mutate_over(self, visitor):
         if self.elts:
-            visitor._mutate_sequence(self.elts)
+            for i in range(len(self.elts)):
+                if self.elts[i] is not None:
+                    self.elts[i] = self.elts[i].mutate_over(visitor)
         return visitor.visit_List(self)
 
     def to_object(self, space):
@@ -2913,7 +3000,9 @@
 
     def mutate_over(self, visitor):
         if self.elts:
-            visitor._mutate_sequence(self.elts)
+            for i in range(len(self.elts)):
+                if self.elts[i] is not None:
+                    self.elts[i] = self.elts[i].mutate_over(visitor)
         return visitor.visit_Tuple(self)
 
     def to_object(self, space):
@@ -3119,7 +3208,9 @@
 
     def mutate_over(self, visitor):
         if self.dims:
-            visitor._mutate_sequence(self.dims)
+            for i in range(len(self.dims)):
+                if self.dims[i] is not None:
+                    self.dims[i] = self.dims[i].mutate_over(visitor)
         return visitor.visit_ExtSlice(self)
 
     def to_object(self, space):
@@ -3487,7 +3578,9 @@
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
         if self.ifs:
-            visitor._mutate_sequence(self.ifs)
+            for i in range(len(self.ifs)):
+                if self.ifs[i] is not None:
+                    self.ifs[i] = self.ifs[i].mutate_over(visitor)
         return visitor.visit_comprehension(self)
 
     def walkabout(self, visitor):
@@ -3555,7 +3648,9 @@
         if self.type:
             self.type = self.type.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_ExceptHandler(self)
 
     def to_object(self, space):
@@ -3606,17 +3701,25 @@
 
     def mutate_over(self, visitor):
         if self.args:
-            visitor._mutate_sequence(self.args)
+            for i in range(len(self.args)):
+                if self.args[i] is not None:
+                    self.args[i] = self.args[i].mutate_over(visitor)
         if self.vararg:
             self.vararg = self.vararg.mutate_over(visitor)
         if self.kwonlyargs:
-            visitor._mutate_sequence(self.kwonlyargs)
+            for i in range(len(self.kwonlyargs)):
+                if self.kwonlyargs[i] is not None:
+                    self.kwonlyargs[i] = self.kwonlyargs[i].mutate_over(visitor)
         if self.kw_defaults:
-            visitor._mutate_sequence(self.kw_defaults)
+            for i in range(len(self.kw_defaults)):
+                if self.kw_defaults[i] is not None:
+                    self.kw_defaults[i] = self.kw_defaults[i].mutate_over(visitor)
         if self.kwarg:
             self.kwarg = self.kwarg.mutate_over(visitor)
         if self.defaults:
-            visitor._mutate_sequence(self.defaults)
+            for i in range(len(self.defaults)):
+                if self.defaults[i] is not None:
+                    self.defaults[i] = self.defaults[i].mutate_over(visitor)
         return visitor.visit_arguments(self)
 
     def walkabout(self, visitor):
@@ -3827,11 +3930,6 @@
     def default_visitor(self, node):
         raise NodeVisitorNotImplemented
 
-    def _mutate_sequence(self, seq):
-        for i in range(len(seq)):
-            if seq[i] is not None:
-                seq[i] = seq[i].mutate_over(self)
-
     def visit_Module(self, node):
         return self.default_visitor(node)
     def visit_Interactive(self, node):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -257,12 +257,19 @@
                 else:
                     level = 2
                 if field.seq:
-                    sub = (field.name,)
-                    self.emit("visitor._mutate_sequence(self.%s)" % sub, level)
+                    sub = field.name
+                    self.emit("for i in range(len(self.{})):".format(sub),
+                        level)
+                    self.emit("if self.{}[i] is not None:".format(sub),
+                        level + 1)
+                    self.emit(
+                        "self.{0}[i] = self.{0}[i].mutate_over(visitor)".format(sub),
+                        level + 2)
                 else:
-                    sub = (field.name, field.name)
-                    self.emit("self.%s = self.%s.mutate_over(visitor)" % sub,
-                              level)
+                    sub = field.name
+                    self.emit(
+                        "self.{0} = self.{0}.mutate_over(visitor)".format(sub),
+                        level)
         self.emit("return visitor.visit_%s(self)" % (name,), 2)
         self.emit("")
 
@@ -305,11 +312,6 @@
         self.emit("def default_visitor(self, node):", 1)
         self.emit("raise NodeVisitorNotImplemented", 2)
         self.emit("")
-        self.emit("def _mutate_sequence(self, seq):", 1)
-        self.emit("for i in range(len(seq)):", 2)
-        self.emit("if seq[i] is not None:", 3)
-        self.emit("seq[i] = seq[i].mutate_over(self)", 4)
-        self.emit("")
         super(ASTVisitorVisitor, self).visitModule(mod)
         self.emit("")
 
@@ -451,6 +453,7 @@
 
 class AST(object):
     __metaclass__ = extendabletype
+    #_attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")

From pypy.commits at gmail.com  Mon Jan 23 19:24:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 23 Jan 2017 16:24:30 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Prevent unwanted attributes from being
 merged to the ast.AST base class
Message-ID: <58869ebe.07addf0a.2e292.8b34@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89713:515869df8814
Date: 2017-01-23 19:11 +0000
http://bitbucket.org/pypy/pypy/changeset/515869df8814/

Log:	Prevent unwanted attributes from being merged to the ast.AST base
	class

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -31,7 +31,7 @@
 
 class AST(object):
     __metaclass__ = extendabletype
-    #_attrs_ = ['lineno', 'col_offset']
+    _attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -338,6 +338,7 @@
         for i, default in enumerate(args.kw_defaults):
             if default:
                 kwonly = args.kwonlyargs[i]
+                assert isinstance(kwonly, ast.arg)
                 mangled = self.scope.mangle(kwonly.arg).decode('utf-8')
                 self.load_const(self.space.wrap(mangled))
                 default.walkabout(self)
@@ -352,16 +353,20 @@
     def _visit_arg_annotations(self, args, names):
         if args:
             for arg in args:
+                assert isinstance(arg, ast.arg)
                 self._visit_arg_annotation(arg.arg, arg.annotation, names)
 
+    @specialize.argtype(1)
     def _visit_annotations(self, func, args, returns):
         space = self.space
         names = []
         self._visit_arg_annotations(args.args, names)
-        if args.vararg:
-            self._visit_arg_annotation(args.vararg.arg, args.vararg.annotation,
+        vararg = args.vararg
+        if vararg:
+            self._visit_arg_annotation(vararg.arg, vararg.annotation,
                                        names)
         self._visit_arg_annotations(args.kwonlyargs, names)
+        kwarg = args.kwarg
         if args.kwarg:
             self._visit_arg_annotation(args.kwarg.arg, args.kwarg.annotation,
                                        names)
@@ -376,6 +381,7 @@
             l += 1
         return l
 
+    @specialize.arg(2)
     def _visit_function(self, func, function_code_generator):
         self.update_position(func.lineno, True)
         # Load decorators first, but apply them after the function is created.
@@ -924,10 +930,12 @@
         self.update_position(wih.lineno, True)
         self.handle_withitem(wih, 0, is_async=False)
 
+    @specialize.argtype(1)
     def handle_withitem(self, wih, pos, is_async):
         body_block = self.new_block()
         cleanup = self.new_block()
         witem = wih.items[pos]
+        assert isinstance(witem, ast.withitem)
         witem.context_expr.walkabout(self)
         if not is_async:
             self.emit_jump(ops.SETUP_WITH, cleanup)
@@ -1289,6 +1297,7 @@
         nseen = 0 # the number of keyword arguments on the stack following
         if keywords is not None:
             for kw in keywords:
+                assert isinstance(kw, ast.keyword)
                 if kw.arg is None:
                     # A keyword argument unpacking.
                     if nseen:
@@ -1346,6 +1355,7 @@
                     return False
         if call.keywords is not None:
             for kw in call.keywords:
+                assert isinstance(kw, ast.keyword)
                 if kw.arg is None:
                     return False
         return True
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -89,12 +89,12 @@
         """Called when a yield is found."""
         raise SyntaxError("'yield' outside function", yield_node.lineno,
                           yield_node.col_offset)
-    
+
     def note_yieldFrom(self, yieldFrom_node):
         """Called when a yield from is found."""
         raise SyntaxError("'yield' outside function", yieldFrom_node.lineno,
                           yieldFrom_node.col_offset)
-    
+
     def note_await(self, await_node):
         """Called when await is found."""
         raise SyntaxError("'await' outside function", await_node.lineno,
@@ -260,12 +260,12 @@
         self.is_generator = True
         if self._in_try_body_depth > 0:
             self.has_yield_inside_try = True
-    
+
     def note_yieldFrom(self, yield_node):
         self.is_generator = True
         if self._in_try_body_depth > 0:
             self.has_yield_inside_try = True
-            
+
     def note_await(self, await_node):
         if self.name == '<genexpr>':
             msg = "'await' expressions in comprehensions are not supported"
@@ -315,7 +315,7 @@
     def note_yieldFrom(self, yield_node):
         raise SyntaxError("'yield from' inside async function", yield_node.lineno,
                           yield_node.col_offset)
-        
+
     def note_await(self, await_node):
         # Compatibility with CPython 3.5: set the CO_GENERATOR flag in
         # addition to the CO_COROUTINE flag if the function uses the
@@ -414,7 +414,7 @@
         func.args.walkabout(self)
         self.visit_sequence(func.body)
         self.pop_scope()
-    
+
     def visit_AsyncFunctionDef(self, func):
         self.note_symbol(func.name, SYM_ASSIGNED)
         # Function defaults and decorators happen in the outer scope.
@@ -429,7 +429,7 @@
         func.args.walkabout(self)
         self.visit_sequence(func.body)
         self.pop_scope()
-    
+
     def visit_Await(self, aw):
         self.scope.note_await(aw)
         ast.GenericASTVisitor.visit_Await(self, aw)
@@ -572,7 +572,7 @@
         witem.context_expr.walkabout(self)
         if witem.optional_vars:
             witem.optional_vars.walkabout(self)
-    
+
     def visit_AsyncWith(self, aw):
         self.scope.new_temporary_name()
         self.visit_sequence(aw.items)
@@ -595,8 +595,9 @@
             scope.note_keywords_arg(arguments.kwarg)
 
     def _handle_params(self, params, is_toplevel):
-        for i in range(len(params)):
-            arg = params[i].arg
+        for param in params:
+            assert isinstance(param, ast.arg)
+            arg = param.arg
             self.note_symbol(arg, SYM_PARAM)
 
     def _visit_annotations(self, func):
@@ -611,6 +612,7 @@
 
     def _visit_arg_annotations(self, args):
         for arg in args:
+            assert isinstance(arg, ast.arg)
             if arg.annotation:
                 arg.annotation.walkabout(self)
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -453,7 +453,7 @@
 
 class AST(object):
     __metaclass__ = extendabletype
-    #_attrs_ = ['lineno', 'col_offset']
+    _attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -46,7 +46,7 @@
             raise ValidationError(
                 "expression which can't be assigned to in %s context" %
                 expr_context_name(ctx))
-        
+
 
 class __extend__(ast.Name):
 
@@ -286,12 +286,12 @@
 
     def visit_Import(self, node):
         self._validate_nonempty_seq(node.names, "names", "Import")
-        
+
     def visit_ImportFrom(self, node):
         if node.level < -1:
             raise ValidationError("ImportFrom level less than -1")
         self._validate_nonempty_seq(node.names, "names", "ImportFrom")
-        
+
     def visit_Global(self, node):
         self._validate_nonempty_seq_s(node.names, "names", "Global")
 
@@ -353,6 +353,7 @@
         if not generators:
             raise ValidationError("comprehension with no generators")
         for comp in generators:
+            assert isinstance(comp, ast.comprehension)
             self._validate_expr(comp.target, ast.Store)
             self._validate_expr(comp.iter)
             self._validate_exprs(comp.ifs)

From pypy.commits at gmail.com  Mon Jan 23 19:30:07 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 23 Jan 2017 16:30:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: The attribute of NameConstant must be
 named 'value'
Message-ID: <5886a00f.12ad1c0a.fcc73.0de0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89714:f8438a882d93
Date: 2017-01-24 00:29 +0000
http://bitbucket.org/pypy/pypy/changeset/f8438a882d93/

Log:	The attribute of NameConstant must be named 'value'

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2678,8 +2678,8 @@
 
 class NameConstant(expr):
 
-    def __init__(self, single, lineno, col_offset):
-        self.single = single
+    def __init__(self, value, lineno, col_offset):
+        self.value = value
         expr.__init__(self, lineno, col_offset)
 
     def walkabout(self, visitor):
@@ -2690,8 +2690,8 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_NameConstant)
-        w_single = self.single  # singleton
-        space.setattr(w_node, space.wrap('single'), w_single)
+        w_value = self.value  # singleton
+        space.setattr(w_node, space.wrap('value'), w_value)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
         w_col_offset = space.wrap(self.col_offset)  # int
@@ -2700,17 +2700,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        w_single = get_field(space, w_node, 'single', False)
+        w_value = get_field(space, w_node, 'value', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _single = w_single
-        if _single is None:
-            raise_required_value(space, w_node, 'single')
+        _value = w_value
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
-        return NameConstant(_single, _lineno, _col_offset)
-
-State.ast_type('NameConstant', 'expr', ['single'])
+        return NameConstant(_value, _lineno, _col_offset)
+
+State.ast_type('NameConstant', 'expr', ['value'])
 
 
 class Ellipsis(expr):
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1246,7 +1246,7 @@
 
     def visit_NameConstant(self, node):
         self.update_position(node.lineno)
-        self.load_const(node.single)
+        self.load_const(node.value)
 
     def visit_keyword(self, keyword):
         if keyword.arg is not None:
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -71,7 +71,7 @@
 class __extend__(ast.NameConstant):
 
     def as_constant(self):
-        return self.single
+        return self.value
 
 class __extend__(ast.Index):
     def as_constant(self):
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -71,8 +71,7 @@
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
          | Bytes(bytes s)
-         -- PyPy mod. first argument name must not be value
-         | NameConstant(singleton single)
+         | NameConstant(singleton value)
          | Ellipsis
 
          -- the following expression can appear in assignment context
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -444,7 +444,7 @@
 
     def visit_NameConstant(self, node):
         space = self.space
-        if (node.single is not space.w_None and
-            node.single is not space.w_True and
-            node.single is not space.w_False):
+        if (node.value is not space.w_None and
+            node.value is not space.w_True and
+            node.value is not space.w_False):
             raise ValidationError("singleton must be True, False, or None")

From pypy.commits at gmail.com  Mon Jan 23 20:56:05 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Mon, 23 Jan 2017 17:56:05 -0800 (PST)
Subject: [pypy-commit] pypy taskengine-sorted-optionals: Comments for
 SimpleTaskEngine._plan
Message-ID: <5886b435.499adf0a.b9ff3.9c9f@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: taskengine-sorted-optionals
Changeset: r89715:0ab64f36afea
Date: 2017-01-24 12:54 +1100
http://bitbucket.org/pypy/pypy/changeset/0ab64f36afea/

Log:	Comments for SimpleTaskEngine._plan

diff --git a/rpython/translator/tool/taskengine.py b/rpython/translator/tool/taskengine.py
--- a/rpython/translator/tool/taskengine.py
+++ b/rpython/translator/tool/taskengine.py
@@ -22,8 +22,11 @@
         except KeyError:
             pass
 
+        # a map from {task : inferred priority}.
         optionality = dict((goal.lstrip('?'), goal.count('?'))
                            for goal in goals)
+
+        # a map from a task to its depedencies.
         task_deps = {}
 
         def will_do(task):
@@ -32,6 +35,17 @@
                 return True
             return priority == 1 and task not in skip
 
+        # we're going to first consider all tasks that are reachable,
+        # regardless of how many ? are found while searching.  We will
+        # record two things about the task - what dependencies it has,
+        # for easy searching later, and how many ? should appear
+        # before the task.  So if A depends on ?C, and C depends on D,
+        # D has one ?.
+        #
+        # some tasks may be considered more than once here - if a
+        # later dependency specifies that a task is not optional,
+        # we'll not only update its optionality but also reconsider
+        # its own dependencies.
         goal_walker = list(goals[::-1])
         while goal_walker:
             goal = goal_walker.pop()
@@ -48,6 +62,8 @@
                     optionality[depname] = dep_qs
                     goal_walker.append(depname)
 
+        # remove any tasks with too-low priority, simple cycles, and
+        # deps with too-low priority.
         for task, deps in list(task_deps.iteritems()):
             if not will_do(task):
                 del task_deps[task]
@@ -58,6 +74,11 @@
                     if not will_do(dep):
                         deps.remove(dep)
 
+        # now it's a matter of toposorting the tasks over their deps.
+        #
+        # we could consider using a sort which is stable in the order
+        # that deps are declared, at least unless that order isn't
+        # consistent.
         plan = []
         seen = set()
         tasks = list(task_deps)
@@ -65,11 +86,12 @@
             remaining = []
             for task in tasks:
                 if task_deps[task] - seen:
+                    # this task has unsatisfied dependencies
                     remaining.append(task)
                 else:
                     plan.append(task)
                     seen.add(task)
-            if len(remaining) == len(tasks):
+            if len(remaining) == len(tasks): # no progress
                 raise RuntimeException('circular dependency')
             tasks = remaining
 

From pypy.commits at gmail.com  Tue Jan 24 01:48:54 2017
From: pypy.commits at gmail.com (gutworth)
Date: Mon, 23 Jan 2017 22:48:54 -0800 (PST)
Subject: [pypy-commit] pypy default: fix "to" vs. "do" typos
Message-ID: <5886f8d6.04abdf0a.cc1f8.f5bc@mx.google.com>

Author: Benjamin Peterson <benjamin at python.org>
Branch: 
Changeset: r89716:c88adb030f33
Date: 2017-01-23 22:48 -0800
http://bitbucket.org/pypy/pypy/changeset/c88adb030f33/

Log:	fix "to" vs. "do" typos

diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -255,7 +255,7 @@
                 pass
             except RZlibError as e:
                 # in this case, CPython raises the direct exception coming
-                # from the zlib module: let's to the same
+                # from the zlib module: let's do the same
                 raise zlib_error(space, e.msg)
             else:
                 if is_package:
@@ -289,7 +289,7 @@
             raise oefmt(space.w_IOError, "Error reading file")
         except RZlibError as e:
             # in this case, CPython raises the direct exception coming
-            # from the zlib module: let's to the same
+            # from the zlib module: let's do the same
             raise zlib_error(space, e.msg)
 
     @unwrap_spec(fullname=str)
@@ -390,7 +390,7 @@
         raise oefmt(get_error(space), "%s seems not to be a zipfile", filename)
     except RZlibError as e:
         # in this case, CPython raises the direct exception coming
-        # from the zlib module: let's to the same
+        # from the zlib module: let's do the same
         raise zlib_error(space, e.msg)
 
     prefix = name[len(filename):]

From pypy.commits at gmail.com  Tue Jan 24 02:44:50 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Mon, 23 Jan 2017 23:44:50 -0800 (PST)
Subject: [pypy-commit] pypy inline-taskengine: Move goal search into
 translator driver
Message-ID: <588705f2.12ad1c0a.fcc73.7d9c@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: inline-taskengine
Changeset: r89717:a174df0f5b63
Date: 2017-01-24 18:34 +1100
http://bitbucket.org/pypy/pypy/changeset/a174df0f5b63/

Log:	Move goal search into translator driver

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -3,7 +3,6 @@
 import shutil
 
 from rpython.translator.translator import TranslationContext
-from rpython.translator.tool.taskengine import SimpleTaskEngine
 from rpython.translator.goal import query
 from rpython.translator.goal.timing import Timer
 from rpython.annotator.listdef import s_list_of_strings
@@ -18,10 +17,12 @@
 
 log = AnsiLogger("translation")
 
-
 def taskdef(deps, title, new_state=None, expected_states=[],
             idemp=False, earlycheck=None):
     def decorator(taskfunc):
+        name = taskfunc.__name__
+        assert name.startswith('task_')
+        taskfunc.task_name = name[len('task_'):]
         taskfunc.task_deps = deps
         taskfunc.task_title = title
         taskfunc.task_newstate = None
@@ -60,7 +61,7 @@
         os._exit(0)
 
 
-class TranslationDriver(SimpleTaskEngine):
+class TranslationDriver(object):
     _backend_extra_options = {}
 
     def __init__(self, setopts=None, default_goal=None,
@@ -69,7 +70,15 @@
                  config=None, overrides=None):
         from rpython.config import translationoption
         self.timer = Timer()
-        SimpleTaskEngine.__init__(self)
+        self.tasks = tasks = {}
+
+        for name in dir(self):
+            if name.startswith('task_'):
+                task = getattr(self, name)
+                assert callable(task)
+                task_deps = getattr(task, 'task_deps', [])
+
+                tasks[task.task_name] = task, task_deps
 
         self.log = log
 
@@ -259,39 +268,37 @@
         KCacheGrind(prof).output(open(goal + ".out", "w"))
         return d['res']
 
-    def _do(self, goal, func, *args, **kwds):
-        title = func.task_title
-        if goal in self.done:
-            self.log.info("already done: %s" % title)
+    def _do(self, func):
+        if func.task_name in self.done:
+            self.log.info("already done: %s" % func.task_title)
             return
         else:
-            self.log.info("%s..." % title)
+            self.log.info("%s..." % func.task_title)
         debug_start('translation-task')
-        debug_print('starting', goal)
-        self.timer.start_event(goal)
+        debug_print('starting', func.task_name)
+        self.timer.start_event(func.task_name)
         try:
             instrument = False
             try:
-                if goal in PROFILE:
-                    res = self._profile(goal, func)
+                if func.task_name in PROFILE:
+                    res = self._profile(func.task_name, func)
                 else:
                     res = func()
             except Instrument:
                 instrument = True
-            if not func.task_idempotent:
-                self.done[goal] = True
+            self.done[func.task_name] = True
             if instrument:
                 self.proceed('compile')
                 assert False, 'we should not get here'
         finally:
             try:
                 debug_stop('translation-task')
-                self.timer.end_event(goal)
+                self.timer.end_event(func.task_name)
             except (KeyboardInterrupt, SystemExit):
                 raise
             except:
-                pass
-        #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal)
+                pass # don't clobber any exception in progress
+        #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % func.task_name)
         return res
 
     @taskdef([], "Annotating&simplifying")
@@ -537,6 +544,38 @@
 
         log.llinterpret("result -> %s" % v)
 
+    def _gather(self, task_name, result, skip=(), also=()):
+        task_name, = self.backend_select_goals([task_name])
+        if task_name in self.done:
+            return
+        _, deps = self.tasks[task_name]
+        for dep in deps:
+            depname = dep.lstrip('?')
+            if dep.startswith('??'):
+                if depname in also:
+                    self._gather(depname, result, skip)
+            elif dep.startswith('?'):
+                if depname not in skip:
+                    self._gather(depname, result, skip)
+            else:
+                self._gather(depname, result, skip)
+        result.append(task_name)
+
+    def _plan(self, goals, skip=None, also=()):
+        if skip is None:
+            skip = self._disabled
+        # gather once to determine the tasks to run, another time to
+        # determine their order, which may move a task forward if it
+        # is an optional dependency of something that appears earlier
+        # in the first plan.
+        selected_goals = []
+        for goal in goals:
+            self._gather(goal, selected_goals, skip, goals)
+        plan = []
+        for goal in selected_goals:
+            self._gather(goal, plan, skip, selected_goals)
+        return plan
+
     def proceed(self, goals):
         if not goals:
             if self.default_goal:
@@ -546,9 +585,15 @@
                 return
         elif isinstance(goals, str):
             goals = [goals]
-        goals.extend(self.extra_goals)
-        goals = self.backend_select_goals(goals)
-        result = self._execute(goals, task_skip = self._maybe_skip())
+        goals = self._plan(goals + self.extra_goals)
+        print goals
+        tasks = [self.tasks[goal] for goal in goals]
+        for task, _ in tasks:
+            if task.task_earlycheck:
+                task.task_earlycheck(self)
+        for task, _ in tasks:
+            self.fork_before(task.task_name)
+            result = self._do(task)
         self.log.info('usession directory: %s' % (udir,))
         return result
 
@@ -588,19 +633,17 @@
     prereq_checkpt_rtype_lltype = prereq_checkpt_rtype
 
     # checkpointing support
-    def _event(self, kind, goal, func):
-        if kind == 'planned' and func.task_earlycheck:
-            func.task_earlycheck(self)
-        if kind == 'pre':
-            fork_before = self.config.translation.fork_before
-            if fork_before:
-                fork_before, = self.backend_select_goals([fork_before])
-                if not fork_before in self.done and fork_before == goal:
-                    prereq = getattr(self, 'prereq_checkpt_%s' % goal, None)
-                    if prereq:
-                        prereq()
-                    from rpython.translator.goal import unixcheckpoint
-                    unixcheckpoint.restartable_point(auto='run')
+    def fork_before(self, goal):
+        fork_before = self.config.translation.fork_before
+        if fork_before:
+            fork_before, = self.backend_select_goals([fork_before])
+            if not fork_before in self.done and fork_before == goal:
+                prereq = getattr(self, 'prereq_checkpt_%s' % goal, None)
+                if prereq:
+                    prereq()
+                from rpython.translator.goal import unixcheckpoint
+                unixcheckpoint.restartable_point(auto='run')
+
 
 def mkexename(name):
     if sys.platform == 'win32':

From pypy.commits at gmail.com  Tue Jan 24 02:44:52 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Mon, 23 Jan 2017 23:44:52 -0800 (PST)
Subject: [pypy-commit] pypy inline-taskengine: Remove the taskengine
Message-ID: <588705f4.9989df0a.fde4d.05c9@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: inline-taskengine
Changeset: r89718:e6538126874a
Date: 2017-01-24 18:36 +1100
http://bitbucket.org/pypy/pypy/changeset/e6538126874a/

Log:	Remove the taskengine

diff --git a/rpython/translator/tool/taskengine.py b/rpython/translator/tool/taskengine.py
deleted file mode 100644
--- a/rpython/translator/tool/taskengine.py
+++ /dev/null
@@ -1,130 +0,0 @@
-class SimpleTaskEngine(object):
-    def __init__(self):
-        self._plan_cache = {}
-
-        self.tasks = tasks = {}
-
-        for name in dir(self):
-            if name.startswith('task_'):
-                task_name = name[len('task_'):]
-                task = getattr(self, name)
-                assert callable(task)
-                task_deps = getattr(task, 'task_deps', [])
-
-                tasks[task_name] = task, task_deps
-
-    def _plan(self, goals, skip=[]):
-        skip = [toskip for toskip in skip if toskip not in goals]
-
-        key = (tuple(goals), tuple(skip))
-        try:
-            return self._plan_cache[key]
-        except KeyError:
-            pass
-        constraints = []
-
-        def subgoals(task_name):
-            taskcallable, deps = self.tasks[task_name]
-            for dep in deps:
-                if dep.startswith('??'): # optional
-                    dep = dep[2:]
-                    if dep not in goals:
-                        continue
-                if dep.startswith('?'): # suggested
-                    dep = dep[1:]
-                    if dep in skip:
-                        continue
-                yield dep
-
-        seen = {}
-
-        def consider(subgoal):
-            if subgoal in seen:
-                return
-            else:
-                seen[subgoal] = True
-            constraints.append([subgoal])
-            deps = subgoals(subgoal)
-            for dep in deps:
-                constraints.append([subgoal, dep])
-                consider(dep)
-
-        for goal in goals:
-            consider(goal)
-
-        #sort
-
-        plan = []
-
-        while True:
-            cands = dict.fromkeys([constr[0] for constr in constraints if constr])
-            if not cands:
-                break
-
-            for cand in cands:
-                for constr in constraints:
-                    if cand in constr[1:]:
-                        break
-                else:
-                    break
-            else:
-                raise RuntimeError("circular dependecy")
-
-            plan.append(cand)
-            for constr in constraints:
-                if constr and constr[0] == cand:
-                    del constr[0]
-
-        plan.reverse()
-
-        self._plan_cache[key] = plan
-
-        return plan
-
-    def _depending_on(self, goal):
-        l = []
-        for task_name, (task, task_deps) in self.tasks.iteritems():
-            if goal in task_deps:
-                l.append(task_name)
-        return l
-
-    def _depending_on_closure(self, goal):
-        d = {}
-
-        def track(goal):
-            if goal in d:
-                return
-            d[goal] = True
-            for depending in self._depending_on(goal):
-                track(depending)
-        track(goal)
-        return d.keys()
-
-    def _execute(self, goals, *args, **kwds):
-        task_skip = kwds.get('task_skip', [])
-        res = None
-        goals = self._plan(goals, skip=task_skip)
-        for goal in goals:
-            taskcallable, _ = self.tasks[goal]
-            self._event('planned', goal, taskcallable)
-        for goal in goals:
-            taskcallable, _ = self.tasks[goal]
-            self._event('pre', goal, taskcallable)
-            try:
-                res = self._do(goal, taskcallable, *args, **kwds)
-            except (SystemExit, KeyboardInterrupt):
-                raise
-            except:
-                self._error(goal)
-                raise
-            self._event('post', goal, taskcallable)
-        return res
-
-    def _do(self, goal, func, *args, **kwds):
-        return func()
-
-    def _event(self, kind, goal, func):
-        pass
-
-    def _error(self, goal):
-        pass
diff --git a/rpython/translator/tool/test/test_taskengine.py b/rpython/translator/tool/test/test_taskengine.py
deleted file mode 100644
--- a/rpython/translator/tool/test/test_taskengine.py
+++ /dev/null
@@ -1,150 +0,0 @@
-from rpython.translator.tool.taskengine import SimpleTaskEngine
-
-def test_simple():
-
-    class ABC(SimpleTaskEngine):
-
-        def task_A(self):
-            pass
-
-        task_A.task_deps = ['B', '?C']
-
-        def task_B(self):
-            pass
-
-        def task_C(self):
-            pass
-
-        task_C.task_deps = ['B']
-
-        def task_D(self):
-            pass
-        task_D.task_deps = ['E']
-
-        def task_E(self):
-            pass
-        task_E.task_deps = ['F']
-
-        def task_F(self):
-            pass
-
-    abc = ABC()
-
-    assert abc._plan('B') == ['B']
-    assert abc._plan('C') == ['B', 'C']
-    assert abc._plan('A') == ['B', 'C', 'A']
-    assert abc._plan('A', skip=['C']) == ['B', 'A']
-
-    assert abc._depending_on('C') == []
-    assert dict.fromkeys(abc._depending_on('B'), True) == {'A':True, 'C':True}
-    assert abc._depending_on('A') == []
-   
-    assert abc._depending_on('F') == ['E']
-    assert abc._depending_on('E') == ['D']
-    assert abc._depending_on('D') == []
-
-    assert abc._depending_on_closure('C') == ['C']
-    assert dict.fromkeys(abc._depending_on_closure('B'), True) == {'A':True, 'C':True, 'B': True}
-    assert abc._depending_on_closure('A') == ['A']
-   
-    assert dict.fromkeys(abc._depending_on_closure('F'), True) == {'D':True, 'E':True, 'F': True}
-    assert dict.fromkeys(abc._depending_on_closure('E'), True) == {'D':True, 'E':True}
-    assert abc._depending_on_closure('D') == ['D']
-
-
-def test_execute():
-
-    class ABC(SimpleTaskEngine):
-
-        def __init__(self):
-            SimpleTaskEngine.__init__(self)
-            self.done = []
-
-        def task_A(self):
-            self.done.append('A')
-
-        task_A.task_deps = ['B', '?C']
-
-        def task_B(self):
-            self.done.append('B')
-
-        def task_C(self):
-            self.done.append('C')
-
-        task_C.task_deps = ['B']
-
-        def _event(self, kind, goal, taskcallable):
-            self.done.append((kind, goal))
-
-    def test(goals, task_skip=[]):
-        if isinstance(goals, str):
-            goals = [goals]
-        abc = ABC()
-        abc._execute(goals, task_skip=task_skip)
-        return abc.done
-
-    def trace(goals):
-        t = []
-        for goal in goals:
-            t.append(('planned', goal))
-        for goal in goals:
-            t.extend([('pre', goal), goal, ('post', goal)])
-        return t
-
-    assert test('B') == trace('B')
-    assert test('C') == trace(['B', 'C'])
-    assert test('A') == trace(['B', 'C', 'A'])
-    assert test('A', ['C']) == trace(['B', 'A'])
-    assert test(['B', 'C']) == trace(['B', 'C'])
-    assert test(['C', 'B']) == trace(['B', 'C'])
-    assert test(['B', 'A']) == trace(['B', 'C', 'A'])
-    assert test(['B', 'A'], ['C']) == trace(['B', 'A'])
-    assert test(['B', 'A', 'C']) == trace(['B', 'C', 'A'])
-    assert test(['B', 'A', 'C'], ['C']) == trace(['B', 'C', 'A'])
-
-def test_driver():
-    class Drv(SimpleTaskEngine):
-
-        def task_A():
-            pass
-        task_A.task_deps = []
-
-        def task_R():
-            pass
-        task_R.task_deps = ['A']
-
-        def task_b():
-            pass
-        task_b.task_deps = ['R']
-
-        def task_H():
-            pass
-        task_H.task_deps = ['b']
-
-        def task_T():
-            pass
-        task_T.task_deps = ['H']
-
-        def task_B():
-            pass
-        task_B.task_deps = ['R', '??T']
-
-        def task_D():
-            pass
-        task_D.task_deps = ['R', '?B', '?A', '??T']
-
-    drv = Drv()
-    assert drv._plan(['R']) == ['A', 'R']
-    assert drv._plan(['B']) == ['A', 'R', 'B']
-    assert drv._plan(['D']) == ['A', 'R', 'B', 'D']
-    assert drv._plan(['D'], skip=['B']) == ['A', 'R', 'D']
-    assert drv._plan(['D', 'R']) == ['A', 'R', 'B', 'D']
-
-
-    assert drv._plan(['H', 'R']) == ['A', 'R', 'b', 'H']
-    assert drv._plan(['H']) == ['A', 'R', 'b', 'H']
-    assert drv._plan(['T', 'B']) == ['A', 'R', 'b', 'H', 'T', 'B']
-    assert drv._plan(['D', 'T']) == ['A', 'R', 'b', 'H', 'T', 'B', 'D']
-    assert drv._plan(['D', 'T', 'R']) == ['A', 'R', 'b', 'H', 'T', 'B', 'D']
-    assert drv._plan(['D', 'T']) == ['A', 'R', 'b', 'H', 'T', 'B', 'D']
-    assert drv._plan(['D', 'T'], skip=['B']) == ['A', 'R', 'b', 'H', 'T', 'D']

From pypy.commits at gmail.com  Tue Jan 24 02:56:46 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Mon, 23 Jan 2017 23:56:46 -0800 (PST)
Subject: [pypy-commit] pypy inline-taskengine: Remove unused attributes
Message-ID: <588708be.ad8ddf0a.8c974.0c44@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: inline-taskengine
Changeset: r89719:072b29f6ce76
Date: 2017-01-24 18:50 +1100
http://bitbucket.org/pypy/pypy/changeset/072b29f6ce76/

Log:	Remove unused attributes

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -17,17 +17,13 @@
 
 log = AnsiLogger("translation")
 
-def taskdef(deps, title, new_state=None, expected_states=[],
-            idemp=False, earlycheck=None):
+def taskdef(deps, title, earlycheck=None):
     def decorator(taskfunc):
         name = taskfunc.__name__
         assert name.startswith('task_')
         taskfunc.task_name = name[len('task_'):]
         taskfunc.task_deps = deps
         taskfunc.task_title = title
-        taskfunc.task_newstate = None
-        taskfunc.task_expected_states = expected_states
-        taskfunc.task_idempotent = idemp
         taskfunc.task_earlycheck = earlycheck
         return taskfunc
     return decorator

From pypy.commits at gmail.com  Tue Jan 24 02:56:48 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Mon, 23 Jan 2017 23:56:48 -0800 (PST)
Subject: [pypy-commit] pypy inline-taskengine: More descriptive method name
Message-ID: <588708c0.5190df0a.16278.09eb@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: inline-taskengine
Changeset: r89720:b9fc36716d25
Date: 2017-01-24 18:55 +1100
http://bitbucket.org/pypy/pypy/changeset/b9fc36716d25/

Log:	More descriptive method name

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -264,7 +264,7 @@
         KCacheGrind(prof).output(open(goal + ".out", "w"))
         return d['res']
 
-    def _do(self, func):
+    def _run_task(self, func):
         if func.task_name in self.done:
             self.log.info("already done: %s" % func.task_title)
             return
@@ -589,7 +589,7 @@
                 task.task_earlycheck(self)
         for task, _ in tasks:
             self.fork_before(task.task_name)
-            result = self._do(task)
+            result = self._run_task(task)
         self.log.info('usession directory: %s' % (udir,))
         return result
 

From pypy.commits at gmail.com  Tue Jan 24 04:42:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 01:42:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: Split the logic into its
 own file
Message-ID: <58872185.85e11c0a.1b84b.76bd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89721:9f329ca0b48f
Date: 2017-01-24 08:50 +0100
http://bitbucket.org/pypy/pypy/changeset/9f329ca0b48f/

Log:	Split the logic into its own file

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1,9 +1,9 @@
 from pypy.interpreter.astcompiler import ast, consts, misc
 from pypy.interpreter.astcompiler import asthelpers # Side effects
+from pypy.interpreter.astcompiler import fstring
 from pypy.interpreter import error
 from pypy.interpreter.pyparser.pygram import syms, tokens
 from pypy.interpreter.pyparser.error import SyntaxError
-from pypy.interpreter.pyparser import parsestring
 from rpython.rlib.objectmodel import always_inline, we_are_translated
 
 
@@ -1191,150 +1191,6 @@
             i += 3
         return (i,key,value)
 
-    def _add_constant_string(self, joined_pieces, w_string, atom_node):
-        space = self.space
-        is_unicode = space.isinstance_w(w_string, space.w_unicode)
-        # Implement implicit string concatenation.
-        if joined_pieces:
-            prev = joined_pieces[-1]
-            if is_unicode and isinstance(prev, ast.Str):
-                w_string = space.add(prev.s, w_string)
-                del joined_pieces[-1]
-            elif not is_unicode and isinstance(prev, ast.Bytes):
-                w_string = space.add(prev.s, w_string)
-                del joined_pieces[-1]
-        node = ast.Str if is_unicode else ast.Bytes
-        joined_pieces.append(node(w_string, atom_node.get_lineno(),
-                                            atom_node.get_column()))
-
-    def _f_constant_string(self, joined_pieces, u, atom_node):
-        self._add_constant_string(joined_pieces, self.space.newunicode(u),
-                                  atom_node)
-
-    def _f_string_compile(self, source, atom_node):
-        # Note: a f-string is kept as a single literal up to here.
-        # At this point only, we recursively call the AST compiler
-        # on all the '{expr}' parts.  The 'expr' part is not parsed
-        # or even tokenized together with the rest of the source code!
-        from pypy.interpreter.pyparser import pyparse
-
-        # complain if 'source' is only whitespace or an empty string
-        for c in source:
-            if c not in ' \t\n\r\v\f':
-                break
-        else:
-            self.error("f-string: empty expression not allowed", atom_node)
-
-        if self.recursive_parser is None:
-            self.error("internal error: parser not available for parsing "
-                       "the expressions inside the f-string", atom_node)
-        source = '(%s)' % source.encode('utf-8')
-
-        info = pyparse.CompileInfo("<fstring>", "eval",
-                                   consts.PyCF_SOURCE_IS_UTF8 |
-                                   consts.PyCF_IGNORE_COOKIE |
-                                   consts.PyCF_REFUSE_COMMENTS,
-                                   optimize=self.compile_info.optimize)
-        parse_tree = self.recursive_parser.parse_source(source, info)
-        return ast_from_node(self.space, parse_tree, info)
-
-    def _f_string_expr(self, joined_pieces, u, start, atom_node, rec=0):
-        conversion = -1     # the conversion char.  -1 if not specified.
-        format_spec = None
-        nested_depth = 0    # nesting level for braces/parens/brackets in exprs
-        p = start
-        while p < len(u):
-            ch = u[p]
-            p += 1
-            if ch in u'[{(':
-                nested_depth += 1
-            elif nested_depth > 0 and ch in u']})':
-                nested_depth -= 1
-            elif nested_depth == 0 and ch in u'!:}':
-                # special-case '!='
-                if ch == u'!' and p < len(u) and u[p] == u'=':
-                    continue
-                break     # normal way out of this loop
-        else:
-            ch = u'\x00'
-        #
-        if nested_depth > 0:
-            self.error("f-string: mismatched '(', '{' or '['", atom_node)
-        end_expression = p - 1
-        if ch == u'!':
-            if p + 1 < len(u):
-                conversion = ord(u[p])
-                ch = u[p + 1]
-                p += 2
-            if conversion not in (ord('s'), ord('r'), ord('a')):
-                self.error("f-string: invalid conversion character: "
-                           "expected 's', 'r', or 'a'", atom_node)
-        if ch == u':':
-            if rec >= 2:
-                self.error("f-string: expressions nested too deeply", atom_node)
-            subpieces = []
-            p = self._parse_f_string(subpieces, u, p, atom_node, rec + 1)
-            format_spec = self._f_string_to_ast_node(subpieces, atom_node)
-            ch = u[p] if p >= 0 else u'\x00'
-            p += 1
-
-        if ch != u'}':
-            self.error("f-string: expecting '}'", atom_node)
-        end_f_string = p
-        assert end_expression >= start
-        expr = self._f_string_compile(u[start:end_expression], atom_node)
-        assert isinstance(expr, ast.Expression)
-        fval = ast.FormattedValue(expr.body, conversion, format_spec,
-                                  atom_node.get_lineno(),
-                                  atom_node.get_column())
-        joined_pieces.append(fval)
-        return end_f_string
-
-    def _parse_f_string(self, joined_pieces, u, start, atom_node, rec=0):
-        space = self.space
-        p1 = u.find(u'{', start)
-        prestart = start
-        while True:
-            if p1 < 0:
-                p1 = len(u)
-            p2 = u.find(u'}', start, p1)
-            if p2 >= 0:
-                self._f_constant_string(joined_pieces, u[prestart:p2],
-                                        atom_node)
-                pn = p2 + 1
-                if pn < len(u) and u[pn] == u'}':    # '}}' => single '}'
-                    start = pn + 1
-                    prestart = pn
-                    continue
-                return p2     # found a single '}', stop here
-            self._f_constant_string(joined_pieces, u[prestart:p1], atom_node)
-            if p1 == len(u):
-                return -1     # no more '{' or '}' left
-            pn = p1 + 1
-            if pn < len(u) and u[pn] == u'{':    # '{{' => single '{'
-                start = pn + 1
-                prestart = pn
-            else:
-                assert u[p1] == u'{'
-                start = self._f_string_expr(joined_pieces, u, pn,
-                                            atom_node, rec)
-                assert u[start - 1] == u'}'
-                prestart = start
-            p1 = u.find(u'{', start)
-
-    def _f_string_to_ast_node(self, joined_pieces, atom_node):
-        # remove empty Strs
-        values = [node for node in joined_pieces
-                       if not (isinstance(node, ast.Str) and not node.s)]
-        if len(values) > 1:
-            return ast.JoinedStr(values, atom_node.get_lineno(),
-                                         atom_node.get_column())
-        elif len(values) == 1:
-            return values[0]
-        else:
-            assert len(joined_pieces) > 0    # they are all empty strings
-            return joined_pieces[0]
-
     def handle_atom(self, atom_node):
         first_child = atom_node.get_child(0)
         first_child_type = first_child.type
@@ -1354,38 +1210,7 @@
                                 first_child.get_column())
         #
         elif first_child_type == tokens.STRING:
-            space = self.space
-            encoding = self.compile_info.encoding
-            joined_pieces = []
-            for i in range(atom_node.num_children()):
-                try:
-                    w_next, saw_f = parsestring.parsestr(
-                            space, encoding, atom_node.get_child(i).get_value())
-                except error.OperationError as e:
-                    if not (e.match(space, space.w_UnicodeError) or
-                            e.match(space, space.w_ValueError)):
-                        raise
-                    # Unicode/ValueError in literal: turn into SyntaxError
-                    raise self.error(e.errorstr(space), atom_node)
-                if not saw_f:
-                    self._add_constant_string(joined_pieces, w_next, atom_node)
-                else:
-                    p = self._parse_f_string(joined_pieces,
-                                             space.unicode_w(w_next), 0,
-                                             atom_node)
-                    if p != -1:
-                        self.error("f-string: single '}' is not allowed",
-                                   atom_node)
-            if len(joined_pieces) == 1:   # <= the common path
-                return joined_pieces[0]   # ast.Str, Bytes or FormattedValue
-            # with more than one piece, it is a combination of Str and
-            # FormattedValue pieces---if there is a Bytes, then we got
-            # an invalid mixture of bytes and unicode literals
-            for node in joined_pieces:
-                if isinstance(node, ast.Bytes):
-                    self.error("cannot mix bytes and nonbytes literals",
-                               atom_node)
-            return self._f_string_to_ast_node(joined_pieces, atom_node)
+            return fstring.string_parse_literal(self, atom_node)
         #
         elif first_child_type == tokens.NUMBER:
             num_value = self.parse_number(first_child.get_value())
diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -0,0 +1,185 @@
+from pypy.interpreter.astcompiler import ast, consts
+from pypy.interpreter.pyparser import parsestring
+from pypy.interpreter import error
+
+
+def add_constant_string(astbuilder, joined_pieces, w_string, atom_node):
+    space = astbuilder.space
+    is_unicode = space.isinstance_w(w_string, space.w_unicode)
+    # Implement implicit string concatenation.
+    if joined_pieces:
+        prev = joined_pieces[-1]
+        if is_unicode and isinstance(prev, ast.Str):
+            w_string = space.add(prev.s, w_string)
+            del joined_pieces[-1]
+        elif not is_unicode and isinstance(prev, ast.Bytes):
+            w_string = space.add(prev.s, w_string)
+            del joined_pieces[-1]
+    node = ast.Str if is_unicode else ast.Bytes
+    joined_pieces.append(node(w_string, atom_node.get_lineno(),
+                                        atom_node.get_column()))
+
+def f_constant_string(astbuilder, joined_pieces, u, atom_node):
+    space = astbuilder.space
+    add_constant_string(astbuilder, joined_pieces, space.newunicode(u),
+                        atom_node)
+
+def f_string_compile(astbuilder, source, atom_node):
+    # Note: a f-string is kept as a single literal up to here.
+    # At this point only, we recursively call the AST compiler
+    # on all the '{expr}' parts.  The 'expr' part is not parsed
+    # or even tokenized together with the rest of the source code!
+    from pypy.interpreter.pyparser import pyparse
+    from pypy.interpreter.astcompiler.astbuilder import ast_from_node
+
+    # complain if 'source' is only whitespace or an empty string
+    for c in source:
+        if c not in ' \t\n\r\v\f':
+            break
+    else:
+        astbuilder.error("f-string: empty expression not allowed", atom_node)
+
+    if astbuilder.recursive_parser is None:
+        astbuilder.error("internal error: parser not available for parsing "
+                   "the expressions inside the f-string", atom_node)
+    source = '(%s)' % source.encode('utf-8')
+
+    info = pyparse.CompileInfo("<fstring>", "eval",
+                               consts.PyCF_SOURCE_IS_UTF8 |
+                               consts.PyCF_IGNORE_COOKIE |
+                               consts.PyCF_REFUSE_COMMENTS,
+                               optimize=astbuilder.compile_info.optimize)
+    parse_tree = astbuilder.recursive_parser.parse_source(source, info)
+    return ast_from_node(astbuilder.space, parse_tree, info)
+
+def f_string_expr(astbuilder, joined_pieces, u, start, atom_node, rec=0):
+    conversion = -1     # the conversion char.  -1 if not specified.
+    format_spec = None
+    nested_depth = 0    # nesting level for braces/parens/brackets in exprs
+    p = start
+    while p < len(u):
+        ch = u[p]
+        p += 1
+        if ch in u'[{(':
+            nested_depth += 1
+        elif nested_depth > 0 and ch in u']})':
+            nested_depth -= 1
+        elif nested_depth == 0 and ch in u'!:}':
+            # special-case '!='
+            if ch == u'!' and p < len(u) and u[p] == u'=':
+                continue
+            break     # normal way out of this loop
+    else:
+        ch = u'\x00'
+    #
+    if nested_depth > 0:
+        astbuilder.error("f-string: mismatched '(', '{' or '['", atom_node)
+    end_expression = p - 1
+    if ch == u'!':
+        if p + 1 < len(u):
+            conversion = ord(u[p])
+            ch = u[p + 1]
+            p += 2
+        if conversion not in (ord('s'), ord('r'), ord('a')):
+            astbuilder.error("f-string: invalid conversion character: "
+                             "expected 's', 'r', or 'a'", atom_node)
+    if ch == u':':
+        if rec >= 2:
+            astbuilder.error("f-string: expressions nested too deeply",
+                             atom_node)
+        subpieces = []
+        p = parse_f_string(astbuilder, subpieces, u, p, atom_node, rec + 1)
+        format_spec = f_string_to_ast_node(astbuilder, subpieces, atom_node)
+        ch = u[p] if p >= 0 else u'\x00'
+        p += 1
+
+    if ch != u'}':
+        astbuilder.error("f-string: expecting '}'", atom_node)
+    end_f_string = p
+    assert end_expression >= start
+    expr = f_string_compile(astbuilder, u[start:end_expression], atom_node)
+    assert isinstance(expr, ast.Expression)
+    fval = ast.FormattedValue(expr.body, conversion, format_spec,
+                              atom_node.get_lineno(),
+                              atom_node.get_column())
+    joined_pieces.append(fval)
+    return end_f_string
+
+def parse_f_string(astbuilder, joined_pieces, u, start, atom_node, rec=0):
+    space = astbuilder.space
+    p1 = u.find(u'{', start)
+    prestart = start
+    while True:
+        if p1 < 0:
+            p1 = len(u)
+        p2 = u.find(u'}', start, p1)
+        if p2 >= 0:
+            f_constant_string(astbuilder, joined_pieces, u[prestart:p2],
+                              atom_node)
+            pn = p2 + 1
+            if pn < len(u) and u[pn] == u'}':    # '}}' => single '}'
+                start = pn + 1
+                prestart = pn
+                continue
+            return p2     # found a single '}', stop here
+        f_constant_string(astbuilder, joined_pieces, u[prestart:p1], atom_node)
+        if p1 == len(u):
+            return -1     # no more '{' or '}' left
+        pn = p1 + 1
+        if pn < len(u) and u[pn] == u'{':    # '{{' => single '{'
+            start = pn + 1
+            prestart = pn
+        else:
+            assert u[p1] == u'{'
+            start = f_string_expr(astbuilder, joined_pieces, u, pn,
+                                  atom_node, rec)
+            assert u[start - 1] == u'}'
+            prestart = start
+        p1 = u.find(u'{', start)
+
+def f_string_to_ast_node(astbuilder, joined_pieces, atom_node):
+    # remove empty Strs
+    values = [node for node in joined_pieces
+                   if not (isinstance(node, ast.Str) and not node.s)]
+    if len(values) > 1:
+        return ast.JoinedStr(values, atom_node.get_lineno(),
+                                     atom_node.get_column())
+    elif len(values) == 1:
+        return values[0]
+    else:
+        assert len(joined_pieces) > 0    # they are all empty strings
+        return joined_pieces[0]
+
+def string_parse_literal(astbuilder, atom_node):
+    space = astbuilder.space
+    encoding = astbuilder.compile_info.encoding
+    joined_pieces = []
+    for i in range(atom_node.num_children()):
+        try:
+            w_next, saw_f = parsestring.parsestr(
+                    space, encoding, atom_node.get_child(i).get_value())
+        except error.OperationError as e:
+            if not (e.match(space, space.w_UnicodeError) or
+                    e.match(space, space.w_ValueError)):
+                raise
+            # Unicode/ValueError in literal: turn into SyntaxError
+            raise astbuilder.error(e.errorstr(space), atom_node)
+        if not saw_f:
+            add_constant_string(astbuilder, joined_pieces, w_next, atom_node)
+        else:
+            p = parse_f_string(astbuilder, joined_pieces,
+                                     space.unicode_w(w_next), 0,
+                                     atom_node)
+            if p != -1:
+                astbuilder.error("f-string: single '}' is not allowed",
+                                 atom_node)
+    if len(joined_pieces) == 1:   # <= the common path
+        return joined_pieces[0]   # ast.Str, Bytes or FormattedValue
+    # with more than one piece, it is a combination of Str and
+    # FormattedValue pieces---if there is a Bytes, then we got
+    # an invalid mixture of bytes and unicode literals
+    for node in joined_pieces:
+        if isinstance(node, ast.Bytes):
+            astbuilder.error("cannot mix bytes and nonbytes literals",
+                             atom_node)
+    return f_string_to_ast_node(astbuilder, joined_pieces, atom_node)

From pypy.commits at gmail.com  Tue Jan 24 04:42:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 01:42:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: This version sticks to
 CPython more closely
Message-ID: <58872187.54b31c0a.8e3a4.680d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89722:d4206d15e05e
Date: 2017-01-24 10:41 +0100
http://bitbucket.org/pypy/pypy/changeset/d4206d15e05e/

Log:	This version sticks to CPython more closely

diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -33,7 +33,6 @@
 PyCF_IGNORE_COOKIE = 0x0800
 PyCF_ACCEPT_NULL_BYTES = 0x10000000   # PyPy only, for compile()
 PyCF_FOUND_ENCODING = 0x20000000      # PyPy only, for pytokenizer
-PyCF_REFUSE_COMMENTS = 0x40000000     # PyPy only, for f-strings
 
 # Masks and values used by FORMAT_VALUE opcode
 FVC_MASK      = 0x3
diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -1,6 +1,8 @@
 from pypy.interpreter.astcompiler import ast, consts
 from pypy.interpreter.pyparser import parsestring
 from pypy.interpreter import error
+from pypy.interpreter import unicodehelper
+from rpython.rlib.rstring import UnicodeBuilder
 
 
 def add_constant_string(astbuilder, joined_pieces, w_string, atom_node):
@@ -46,96 +48,261 @@
 
     info = pyparse.CompileInfo("<fstring>", "eval",
                                consts.PyCF_SOURCE_IS_UTF8 |
-                               consts.PyCF_IGNORE_COOKIE |
-                               consts.PyCF_REFUSE_COMMENTS,
+                               consts.PyCF_IGNORE_COOKIE,
                                optimize=astbuilder.compile_info.optimize)
     parse_tree = astbuilder.recursive_parser.parse_source(source, info)
     return ast_from_node(astbuilder.space, parse_tree, info)
 
-def f_string_expr(astbuilder, joined_pieces, u, start, atom_node, rec=0):
-    conversion = -1     # the conversion char.  -1 if not specified.
+
+def unexpected_end_of_string(astbuilder, atom_node):
+    astbuilder.error("f-string: expecting '}'", atom_node)
+
+
+def fstring_find_expr(astbuilder, fstr, atom_node, rec):
+    # Parse the f-string at fstr.current_index.  We know it starts an
+    # expression (so it must be at '{'). Returns the FormattedValue node,
+    # which includes the expression, conversion character, and
+    # format_spec expression.
+    conversion = -1      # the conversion char.  -1 if not specified.
     format_spec = None
-    nested_depth = 0    # nesting level for braces/parens/brackets in exprs
-    p = start
-    while p < len(u):
-        ch = u[p]
-        p += 1
-        if ch in u'[{(':
+
+    # 0 if we're not in a string, else the quote char we're trying to
+    # match (single or double quote).
+    quote_char = 0
+
+    # If we're inside a string, 1=normal, 3=triple-quoted.
+    string_type = 0
+
+    # Keep track of nesting level for braces/parens/brackets in
+    # expressions.
+    nested_depth = 0
+
+    # Can only nest one level deep.
+    if rec >= 2:
+        astbuilder.error("f-string: expressions nested too deeply", atom_node)
+
+    # The first char must be a left brace, or we wouldn't have gotten
+    # here. Skip over it.
+    u = fstr.unparsed
+    i = fstr.current_index
+    assert u[i] == u'{'
+    i += 1
+
+    expr_start = i
+    while i < len(u):
+
+        # Loop invariants.
+        assert nested_depth >= 0
+        if quote_char:
+            assert string_type == 1 or string_type == 3
+        else:
+            assert string_type == 0
+
+        ch = u[i]
+        # Nowhere inside an expression is a backslash allowed.
+        if ch == u'\\':
+            # Error: can't include a backslash character, inside
+            # parens or strings or not.
+            astbuilder.error("f-string expression part "
+                             "cannot include a backslash", atom_node)
+
+        if quote_char:
+            # We're inside a string. See if we're at the end.
+            # <a long comment goes here about how we're duplicating
+            # some existing logic>
+            if ord(ch) == quote_char:
+                # Does this match the string_type (single or triple
+                # quoted)?
+                if string_type == 3:
+                    if i + 2 < len(u) and u[i + 1] == u[i + 2] == ch:
+                        # We're at the end of a triple quoted string.
+                        i += 3
+                        string_type = 0
+                        quote_char = 0
+                        continue
+                else:
+                    # We're at the end of a normal string.
+                    i += 1
+                    string_type = 0
+                    quote_char = 0
+                    continue
+        elif ch == u"'" or ch == u'"':
+            # Is this a triple quoted string?
+            if i + 2 < len(u) and u[i + 1] == u[i + 2] == ch:
+                string_type = 3
+                i += 2
+            else:
+                # Start of a normal string.
+                string_type = 1
+            # Start looking for the end of the string.
+            quote_char = ord(ch)
+        elif ch in u"[{(":
             nested_depth += 1
-        elif nested_depth > 0 and ch in u']})':
+        elif nested_depth != 0 and ch in u"]})":
             nested_depth -= 1
-        elif nested_depth == 0 and ch in u'!:}':
-            # special-case '!='
-            if ch == u'!' and p < len(u) and u[p] == u'=':
+        elif ch == u'#':
+            # Error: can't include a comment character, inside parens
+            # or not.
+            astbuilder.error("f-string expression part cannot include '#'",
+                             atom_node)
+        elif nested_depth == 0 and ch in u"!:}":
+            # First, test for the special case of "!=". Since '=' is
+            # not an allowed conversion character, nothing is lost in
+            # this test.
+            if ch == '!' and i + 1 < len(u) and u[i+1] == u'=':
+                # This isn't a conversion character, just continue.
+                i += 1
                 continue
-            break     # normal way out of this loop
-    else:
-        ch = u'\x00'
-    #
-    if nested_depth > 0:
+            # Normal way out of this loop.
+            break
+        #else:
+        #   This isn't a conversion character, just continue.
+        i += 1
+
+    # If we leave this loop in a string or with mismatched parens, we
+    # don't care. We'll get a syntax error when compiling the
+    # expression. But, we can produce a better error message, so
+    # let's just do that.
+    if quote_char:
+        astbuilder.error("f-string: unterminated string", atom_node)
+
+    if nested_depth:
         astbuilder.error("f-string: mismatched '(', '{' or '['", atom_node)
-    end_expression = p - 1
-    if ch == u'!':
-        if p + 1 < len(u):
-            conversion = ord(u[p])
-            ch = u[p + 1]
-            p += 2
+
+    if i >= len(u):
+        unexpected_end_of_string(astbuilder, atom_node)
+
+    # Compile the expression as soon as possible, so we show errors
+    # related to the expression before errors related to the
+    # conversion or format_spec.
+    expr = f_string_compile(astbuilder, u[expr_start:i], atom_node)
+    assert isinstance(expr, ast.Expression)
+
+    # Check for a conversion char, if present.
+    if u[i] == u'!':
+        i += 1
+        if i >= len(u):
+            unexpected_end_of_string(astbuilder, atom_node)
+
+        conversion = ord(u[i])
+        i += 1
         if conversion not in (ord('s'), ord('r'), ord('a')):
             astbuilder.error("f-string: invalid conversion character: "
                              "expected 's', 'r', or 'a'", atom_node)
-    if ch == u':':
-        if rec >= 2:
-            astbuilder.error("f-string: expressions nested too deeply",
-                             atom_node)
+
+    # Check for the format spec, if present.
+    if i >= len(u):
+        unexpected_end_of_string(astbuilder, atom_node)
+    if u[i] == u':':
+        i += 1
+        if i >= len(u):
+            unexpected_end_of_string(astbuilder, atom_node)
+        fstr.current_index = i
         subpieces = []
-        p = parse_f_string(astbuilder, subpieces, u, p, atom_node, rec + 1)
+        parse_f_string(astbuilder, subpieces, fstr, atom_node, rec + 1)
         format_spec = f_string_to_ast_node(astbuilder, subpieces, atom_node)
-        ch = u[p] if p >= 0 else u'\x00'
-        p += 1
+        i = fstr.current_index
 
-    if ch != u'}':
-        astbuilder.error("f-string: expecting '}'", atom_node)
-    end_f_string = p
-    assert end_expression >= start
-    expr = f_string_compile(astbuilder, u[start:end_expression], atom_node)
-    assert isinstance(expr, ast.Expression)
-    fval = ast.FormattedValue(expr.body, conversion, format_spec,
+    if i >= len(u) or u[i] != u'}':
+        unexpected_end_of_string(astbuilder, atom_node)
+
+    # We're at a right brace. Consume it.
+    i += 1
+    fstr.current_index = i
+
+    # And now create the FormattedValue node that represents this
+    # entire expression with the conversion and format spec.
+    return ast.FormattedValue(expr.body, conversion, format_spec,
                               atom_node.get_lineno(),
                               atom_node.get_column())
-    joined_pieces.append(fval)
-    return end_f_string
 
-def parse_f_string(astbuilder, joined_pieces, u, start, atom_node, rec=0):
+
+def fstring_find_literal(astbuilder, fstr, atom_node, rec):
+    # Return the next literal part.  Updates the current index inside 'fstr'.
+    # Differs from CPython: this version handles double-braces on its own.
+    u = fstr.unparsed
+    literal_start = fstr.current_index
+    in_named_escape = False
+
+    # Get any literal string. It ends when we hit an un-doubled left
+    # brace (which isn't part of a unicode name escape such as
+    # "\N{EULER CONSTANT}"), or the end of the string.
+    i = literal_start
+    builder = UnicodeBuilder()
+    while i < len(u):
+        ch = u[i]
+        if (not in_named_escape and ch == u'{' and i - literal_start >= 2
+                and u[i - 2] == u'\\' and u[i - 1] == u'N'):
+            in_named_escape = True
+        elif in_named_escape and ch == u'}':
+            in_named_escape = False
+        elif ch == u'{' or ch == u'}':
+            # Check for doubled braces, but only at the top level. If
+            # we checked at every level, then f'{0:{3}}' would fail
+            # with the two closing braces.
+            if rec == 0 and i + 1 < len(u) and u[i + 1] == ch:
+                i += 1   # skip over the second brace
+            elif rec == 0 and ch == u'}':
+                # Where a single '{' is the start of a new expression, a
+                # single '}' is not allowed.
+                astbuilder.error("f-string: single '}' is not allowed",
+                                 atom_node)
+            else:
+                # We're either at a '{', which means we're starting another
+                # expression; or a '}', which means we're at the end of this
+                # f-string (for a nested format_spec).
+                break
+        builder.append(ch)
+        i += 1
+
+    fstr.current_index = i
+    literal = builder.build()
+    if not fstr.raw_mode:
+        literal = unicodehelper.decode_unicode_escape(astbuilder.space, literal)
+    return literal
+
+
+def fstring_find_literal_and_expr(astbuilder, fstr, atom_node, rec):
+    # Return a tuple with the next literal part, and optionally the
+    # following expression node.  Updates the current index inside 'fstr'.
+    literal = fstring_find_literal(astbuilder, fstr, atom_node, rec)
+
+    u = fstr.unparsed
+    i = fstr.current_index
+    if i >= len(u) or u[i] == u'}':
+        # We're at the end of the string or the end of a nested
+        # f-string: no expression.
+        expr = None
+    else:
+        # We must now be the start of an expression, on a '{'.
+        assert u[i] == u'{'
+        expr = fstring_find_expr(astbuilder, fstr, atom_node, rec)
+    return literal, expr
+
+
+def parse_f_string(astbuilder, joined_pieces, fstr, atom_node, rec=0):
     space = astbuilder.space
-    p1 = u.find(u'{', start)
-    prestart = start
     while True:
-        if p1 < 0:
-            p1 = len(u)
-        p2 = u.find(u'}', start, p1)
-        if p2 >= 0:
-            f_constant_string(astbuilder, joined_pieces, u[prestart:p2],
-                              atom_node)
-            pn = p2 + 1
-            if pn < len(u) and u[pn] == u'}':    # '}}' => single '}'
-                start = pn + 1
-                prestart = pn
-                continue
-            return p2     # found a single '}', stop here
-        f_constant_string(astbuilder, joined_pieces, u[prestart:p1], atom_node)
-        if p1 == len(u):
-            return -1     # no more '{' or '}' left
-        pn = p1 + 1
-        if pn < len(u) and u[pn] == u'{':    # '{{' => single '{'
-            start = pn + 1
-            prestart = pn
-        else:
-            assert u[p1] == u'{'
-            start = f_string_expr(astbuilder, joined_pieces, u, pn,
-                                  atom_node, rec)
-            assert u[start - 1] == u'}'
-            prestart = start
-        p1 = u.find(u'{', start)
+        literal, expr = fstring_find_literal_and_expr(astbuilder, fstr,
+                                                      atom_node, rec)
+
+        # add the literal part
+        f_constant_string(astbuilder, joined_pieces, literal, atom_node)
+
+        if expr is None:
+            break         # We're done with this f-string.
+
+        joined_pieces.append(expr)
+
+    # If recurse_lvl is zero, then we must be at the end of the
+    # string. Otherwise, we must be at a right brace.
+    if rec == 0 and fstr.current_index < len(fstr.unparsed) - 1:
+        astbuilder.error("f-string: unexpected end of string", atom_node)
+
+    if rec != 0 and (fstr.current_index >= len(fstr.unparsed) or
+                     fstr.unparsed[fstr.current_index] != u'}'):
+        astbuilder.error("f-string: expecting '}'", atom_node)
+
 
 def f_string_to_ast_node(astbuilder, joined_pieces, atom_node):
     # remove empty Strs
@@ -150,13 +317,14 @@
         assert len(joined_pieces) > 0    # they are all empty strings
         return joined_pieces[0]
 
+
 def string_parse_literal(astbuilder, atom_node):
     space = astbuilder.space
     encoding = astbuilder.compile_info.encoding
     joined_pieces = []
     for i in range(atom_node.num_children()):
         try:
-            w_next, saw_f = parsestring.parsestr(
+            w_next = parsestring.parsestr(
                     space, encoding, atom_node.get_child(i).get_value())
         except error.OperationError as e:
             if not (e.match(space, space.w_UnicodeError) or
@@ -164,15 +332,10 @@
                 raise
             # Unicode/ValueError in literal: turn into SyntaxError
             raise astbuilder.error(e.errorstr(space), atom_node)
-        if not saw_f:
+        if not isinstance(w_next, parsestring.W_FString):
             add_constant_string(astbuilder, joined_pieces, w_next, atom_node)
         else:
-            p = parse_f_string(astbuilder, joined_pieces,
-                                     space.unicode_w(w_next), 0,
-                                     atom_node)
-            if p != -1:
-                astbuilder.error("f-string: single '}' is not allowed",
-                                 atom_node)
+            parse_f_string(astbuilder, joined_pieces, w_next, atom_node)
     if len(joined_pieces) == 1:   # <= the common path
         return joined_pieces[0]   # ast.Str, Bytes or FormattedValue
     # with more than one piece, it is a combination of Str and
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1182,6 +1182,8 @@
         yield self.st, """x = 42;   z = f'{x:5}'""", 'z', '   42'
         yield self.st, """x = 2; z = f'{5:{x:+1}0}'""", 'z', (' ' * 18 + '+5')
 
+        yield self.st, """z=f'{"}"}'""", 'z', '}'
+
     def test_fstring_error(self):
         raises(SyntaxError, self.run, "f'{}'")
         raises(SyntaxError, self.run, "f'{   \t   }'")
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -1,12 +1,22 @@
 # coding: utf-8
+from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import unicodehelper
 from rpython.rlib.rstring import StringBuilder
 
 
+class W_FString(W_Root):
+    def __init__(self, unparsed, raw_mode):
+        assert isinstance(unparsed, unicode)
+        self.unparsed = unparsed     # but the quotes are removed
+        self.raw_mode = raw_mode
+        self.current_index = 0       # for astcompiler.fstring
+
+
 def parsestr(space, encoding, s):
-    """Parses a string or unicode literal, and return a pair
-    (wrapped value, f_string_flag).
+    """Parses a string or unicode literal, and return usually
+    a wrapped value.  If we get an f-string, then instead return
+    an unparsed but unquoted W_FString instance.
 
     If encoding=None, the source string is ascii only.
     In other cases, the source string is in utf-8 encoding.
@@ -74,18 +84,17 @@
                                         'unmatched triple quotes in literal')
         q -= 2
 
-    if saw_f:
-        # forbid any '\' inside '{' and '}' pairs
-        pass # XXX DO IT
-
     if unicode_literal and not rawmode: # XXX Py_UnicodeFlag is ignored for now
         if encoding is None:
             assert 0 <= ps <= q
             substr = s[ps:q]
         else:
             substr = decode_unicode_utf8(space, s, ps, q)
+        if saw_f:
+            v = unicodehelper.decode_utf8(space, substr)
+            return W_FString(v, rawmode)
         v = unicodehelper.decode_unicode_escape(space, substr)
-        return space.wrap(v), saw_f
+        return space.wrap(v)
 
     assert 0 <= ps <= q
     substr = s[ps : q]
@@ -99,13 +108,15 @@
 
     if rawmode or '\\' not in substr:
         if not unicode_literal:
-            return space.newbytes(substr), saw_f
+            return space.newbytes(substr)
         else:
             v = unicodehelper.decode_utf8(space, substr)
-            return space.wrap(v), saw_f
+            if saw_f:
+                return W_FString(v, rawmode)
+            return space.wrap(v)
 
     v = PyString_DecodeEscape(space, substr, 'strict', encoding)
-    return space.newbytes(v), saw_f
+    return space.newbytes(v)
 
 def decode_unicode_utf8(space, s, ps, q):
     # ****The Python 2.7 version, producing UTF-32 escapes****
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -187,9 +187,6 @@
                 continue
             if line[pos] == '#':
                 # skip full-line comment, but still check that it is valid utf-8
-                if flags & consts.PyCF_REFUSE_COMMENTS:
-                    raise TokenError("comments not allowed here",
-                                     line, lnum, pos, token_list)
                 if not verify_utf8(line):
                     raise bad_utf8("comment",
                                    line, lnum, pos, token_list, flags)
@@ -260,9 +257,6 @@
                     last_comment = ''
                 elif initial == '#':
                     # skip comment, but still check that it is valid utf-8
-                    if flags & consts.PyCF_REFUSE_COMMENTS:
-                        raise TokenError("comments not allowed here",
-                                         line, lnum, start, token_list)
                     if not verify_utf8(token):
                         raise bad_utf8("comment",
                                        line, lnum, start, token_list, flags)

From pypy.commits at gmail.com  Tue Jan 24 05:20:02 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Tue, 24 Jan 2017 02:20:02 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: More explicit
 arrangement of tasks
Message-ID: <58872a52.828bdf0a.2e8d7.9c9f@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89723:b36e251c762f
Date: 2017-01-24 21:19 +1100
http://bitbucket.org/pypy/pypy/changeset/b36e251c762f/

Log:	More explicit arrangement of tasks

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -3,7 +3,6 @@
 import shutil
 
 from rpython.translator.translator import TranslationContext
-from rpython.translator.tool.taskengine import SimpleTaskEngine
 from rpython.translator.goal import query
 from rpython.translator.goal.timing import Timer
 from rpython.annotator.listdef import s_list_of_strings
@@ -18,18 +17,7 @@
 
 log = AnsiLogger("translation")
 
-
-def taskdef(deps, title, new_state=None, expected_states=[],
-            idemp=False, earlycheck=None):
-    def decorator(taskfunc):
-        taskfunc.task_deps = deps
-        taskfunc.task_title = title
-        taskfunc.task_newstate = None
-        taskfunc.task_expected_states = expected_states
-        taskfunc.task_idempotent = idemp
-        taskfunc.task_earlycheck = earlycheck
-        return taskfunc
-    return decorator
+class Done(Exception): pass
 
 # TODO:
 # sanity-checks using states
@@ -60,16 +48,13 @@
         os._exit(0)
 
 
-class TranslationDriver(SimpleTaskEngine):
+class TranslationDriver(object):
     _backend_extra_options = {}
 
-    def __init__(self, setopts=None, default_goal=None,
-                 disable=[],
-                 exe_name=None, extmod_name=None,
-                 config=None, overrides=None):
+    def __init__(self, setopts=None, default_goal=None, disable=(),
+                 exe_name=None, config=None, overrides=None):
         from rpython.config import translationoption
         self.timer = Timer()
-        SimpleTaskEngine.__init__(self)
 
         self.log = log
 
@@ -85,9 +70,8 @@
             self.config.set(**setopts)
 
         self.exe_name = exe_name
-        self.extmod_name = extmod_name
 
-        self.done = {}
+        self.done = set()
 
         self.disable(disable)
 
@@ -98,40 +82,51 @@
 
         self.default_goal = default_goal
         self.extra_goals = []
-        self.exposed = []
 
-        # expose tasks
-        def expose_task(task, backend_goal=None):
-            if backend_goal is None:
-                backend_goal = task
-            def proc():
-                return self.proceed(backend_goal)
-            self.exposed.append(task)
-            setattr(self, task, proc)
+    def annotate(self):
+        return self.proceed(['annotate'])
 
-        backend, ts = self.get_backend_and_type_system()
-        for task in self.tasks:
-            explicit_task = task
-            if task == 'annotate':
-                expose_task(task)
-            else:
-                task, postfix = task.split('_')
-                if task in ('rtype', 'backendopt', 'llinterpret',
-                            'pyjitpl'):
-                    if ts:
-                        if ts == postfix:
-                            expose_task(task, explicit_task)
-                    else:
-                        expose_task(explicit_task)
-                elif task in ('source', 'compile', 'run'):
-                    if backend:
-                        if backend == postfix:
-                            expose_task(task, explicit_task)
-                    elif ts:
-                        if ts == 'lltype':
-                            expose_task(explicit_task)
-                    else:
-                        expose_task(explicit_task)
+    def rtype(self):
+        return self.proceed(['rtype'])
+
+    def backendopt(self):
+        return self.proceed(['backendopt'])
+
+    def llinterpret(self):
+        return self.proceed(['llinterpret'])
+
+    def pyjitpl(self):
+        return self.proceed(['pyjitpl'])
+
+    def rtype_lltype(self):
+        return self.proceed(['rtype_lltype'])
+
+    def backendopt_lltype(self):
+        return self.proceed(['backendopt_lltype'])
+
+    def llinterpret_lltype(self):
+        return self.proceed(['llinterpret_lltype'])
+
+    def pyjitpl_lltype(self):
+        return self.proceed(['pyjitpl_lltype'])
+
+    def source(self):
+        return self.proceed(['source'])
+
+    def compile(self):
+        return self.proceed(['compile'])
+
+    def run(self):
+        return self.proceed(['run'])
+
+    def source_c(self):
+        return self.proceed(['source_c'])
+
+    def compile_c(self):
+        return self.proceed(['compile_c'])
+
+    def run_c(self):
+        return self.proceed(['run_c'])
 
     def set_extra_goals(self, goals):
         self.extra_goals = goals
@@ -148,31 +143,101 @@
         backend = self.config.translation.backend
         return backend, type_system
 
+    def run_task(self, name, goals, *args, **kwargs):
+        if name in self.done or name in self._disabled:
+            return
+        task = getattr(self, 'task_%s' % name)
+
+        self.fork_before(name)
+
+        debug_start('translation-task')
+        debug_print('starting', name)
+        self.timer.start_event(name)
+        try:
+            instrument = False
+            try:
+                if name in PROFILE:
+                    res = self._profile(name, func)
+                else:
+                    res = task(*args, **kwargs)
+            except Instrument:
+                instrument = True
+            if instrument:
+                self.proceed(['compile_c'])
+                assert False, 'we should not get here'
+        finally:
+            try:
+                debug_stop('translation-task')
+                self.timer.end_event(name)
+            except (KeyboardInterrupt, SystemExit):
+                raise
+            except:
+                pass
+        #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal)
+
+        self.log.info('usession directory: %s' % (udir,))
+
+        self.done.add(name)
+        goals.discard(name)
+        if not goals:
+            raise Done(res)
+        return res
+
+    def proceed(self, goals):
+        try:
+            self._proceed_inner(goals)
+        except Done as d:
+            return d.args[0]
+
+    def _proceed_inner(self, goals):
+        backend, ts = self.get_backend_and_type_system()
+        goals = set(self.backend_select_goals(goals + self.extra_goals))
+
+        if any(cgoal in goals
+               for bakgoal in ['database', 'source', 'compile']
+               for cgoal in [bakgoal, bakgoal + '_c']):
+            if 'check_for_boehm' not in self.done:
+                self.possibly_check_for_boehm()
+                self.done.add('check_for_boehm')
+
+        self.run_task('annotate', goals)
+        self.run_task('rtype_lltype', goals)
+        if 'pyjitpl_lltype' in goals or 'jittest_lltype' in goals:
+            self.run_task('pyjitpl_lltype', goals)
+        if 'jittest_lltype' in goals:
+            self.run_task('jittest_lltype', goals)
+        self.run_task('backendopt_lltype', goals)
+        self.run_task('stackcheckinsertion_lltype', goals)
+        if 'llinterpret_lltype' in goals:
+            self.run_task('llinterpret_lltype', goals)
+        self.run_task('backend_%s' % backend, goals, goals)
+
+    def task_backend_c(self, goals):
+        self.run_task('database_c', goals)
+        self.run_task('source_c', goals)
+        self.run_task('compile_c', goals)
+
     def backend_select_goals(self, goals):
         backend, ts = self.get_backend_and_type_system()
-        postfixes = [''] + ['_'+p for p in (backend, ts) if p]
-        l = []
+        result = []
         for goal in goals:
-            for postfix in postfixes:
-                cand = "%s%s" % (goal, postfix)
-                if cand in self.tasks:
-                    new_goal = cand
+            names = ['task_%s_%s' % (goal, backend),
+                     'task_%s_%s' % (goal, ts),
+                     'task_%s' % (goal,)]
+            if set(names).intersection(self.done):
+                continue
+            for name in names:
+                task = getattr(self, name, None)
+                if task is not None:
+                    result.append(name[len('task_'):])
                     break
             else:
                 raise Exception("cannot infer complete goal from: %r" % goal)
-            l.append(new_goal)
-        return l
-
+        return result
+            
     def disable(self, to_disable):
         self._disabled = to_disable
 
-    def _maybe_skip(self):
-        maybe_skip = []
-        if self._disabled:
-            for goal in self.backend_select_goals(self._disabled):
-                maybe_skip.extend(self._depending_on_closure(goal))
-        return dict.fromkeys(maybe_skip).keys()
-
     def setup(self, entry_point, inputtypes, policy=None, extra={}, empty_translator=None):
         standalone = inputtypes is None
         self.standalone = standalone
@@ -259,42 +324,6 @@
         KCacheGrind(prof).output(open(goal + ".out", "w"))
         return d['res']
 
-    def _do(self, goal, func, *args, **kwds):
-        title = func.task_title
-        if goal in self.done:
-            self.log.info("already done: %s" % title)
-            return
-        else:
-            self.log.info("%s..." % title)
-        debug_start('translation-task')
-        debug_print('starting', goal)
-        self.timer.start_event(goal)
-        try:
-            instrument = False
-            try:
-                if goal in PROFILE:
-                    res = self._profile(goal, func)
-                else:
-                    res = func()
-            except Instrument:
-                instrument = True
-            if not func.task_idempotent:
-                self.done[goal] = True
-            if instrument:
-                self.proceed('compile')
-                assert False, 'we should not get here'
-        finally:
-            try:
-                debug_stop('translation-task')
-                self.timer.end_event(goal)
-            except (KeyboardInterrupt, SystemExit):
-                raise
-            except:
-                pass
-        #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal)
-        return res
-
-    @taskdef([], "Annotating&simplifying")
     def task_annotate(self):
         """ Annotate
         """
@@ -336,15 +365,12 @@
         lost = query.qoutput(query.check_methods_qgen(translator))
         assert not lost, "lost methods, something gone wrong with the annotation of method defs"
 
-    RTYPE = 'rtype_lltype'
-    @taskdef(['annotate'], "RTyping")
     def task_rtype_lltype(self):
         """ RTyping - lltype version
         """
         rtyper = self.translator.buildrtyper()
         rtyper.specialize(dont_simplify_again=True)
 
-    @taskdef([RTYPE], "JIT compiler generation")
     def task_pyjitpl_lltype(self):
         """ Generate bytecodes for JIT and flow the JIT helper functions
         lltype version
@@ -362,7 +388,6 @@
         #
         self.log.info("the JIT compiler was generated")
 
-    @taskdef([RTYPE], "test of the JIT on the llgraph backend")
     def task_jittest_lltype(self):
         """ Run with the JIT on top of the llgraph backend
         """
@@ -375,8 +400,6 @@
         from rpython.jit.tl import jittest
         jittest.jittest(self)
 
-    BACKENDOPT = 'backendopt_lltype'
-    @taskdef([RTYPE, '??pyjitpl_lltype', '??jittest_lltype'], "lltype back-end optimisations")
     def task_backendopt_lltype(self):
         """ Run all backend optimizations - lltype version
         """
@@ -384,8 +407,6 @@
         backend_optimizations(self.translator, replace_we_are_jitted=True)
 
 
-    STACKCHECKINSERTION = 'stackcheckinsertion_lltype'
-    @taskdef(['?'+BACKENDOPT, RTYPE, 'annotate'], "inserting stack checks")
     def task_stackcheckinsertion_lltype(self):
         from rpython.translator.transform import insert_ll_stackcheck
         count = insert_ll_stackcheck(self.translator)
@@ -402,9 +423,6 @@
                 i = 'Boehm GC not installed.  Try e.g. "translate.py --gc=minimark"'
                 raise Exception(str(e) + '\n' + i)
 
-    @taskdef([STACKCHECKINSERTION, '?'+BACKENDOPT, RTYPE, '?annotate'],
-        "Creating database for generating c source",
-        earlycheck = possibly_check_for_boehm)
     def task_database_c(self):
         """ Create a database for further backend generation
         """
@@ -427,14 +445,11 @@
                                        functions=functions,
                                        name='libtesting',
                                        config=self.config)
-        if not standalone:     # xxx more messy
-            cbuilder.modulename = self.extmod_name
         database = cbuilder.build_database()
         self.log.info("database for generating C source was created")
         self.cbuilder = cbuilder
         self.database = database
 
-    @taskdef(['database_c'], "Generating c source")
     def task_source_c(self):
         """ Create C source files from the generated database
         """
@@ -506,7 +521,6 @@
             self.c_entryp = newexename
         self.log.info("created: %s" % (self.c_entryp,))
 
-    @taskdef(['source_c'], "Compiling c source")
     def task_compile_c(self):
         """ Compile the generated C code using either makefile or
         translator/platform
@@ -523,7 +537,6 @@
         else:
             self.c_entryp = cbuilder.get_entry_point()
 
-    @taskdef([STACKCHECKINSERTION, '?'+BACKENDOPT, RTYPE], "LLInterpreting")
     def task_llinterpret_lltype(self):
         from rpython.rtyper.llinterp import LLInterpreter
 
@@ -537,21 +550,6 @@
 
         log.llinterpret("result -> %s" % v)
 
-    def proceed(self, goals):
-        if not goals:
-            if self.default_goal:
-                goals = [self.default_goal]
-            else:
-                self.log.info("nothing to do")
-                return
-        elif isinstance(goals, str):
-            goals = [goals]
-        goals.extend(self.extra_goals)
-        goals = self.backend_select_goals(goals)
-        result = self._execute(goals, task_skip = self._maybe_skip())
-        self.log.info('usession directory: %s' % (udir,))
-        return result
-
     @classmethod
     def from_targetspec(cls, targetspec_dic, config=None, args=None,
                         empty_translator=None,
@@ -588,19 +586,16 @@
     prereq_checkpt_rtype_lltype = prereq_checkpt_rtype
 
     # checkpointing support
-    def _event(self, kind, goal, func):
-        if kind == 'planned' and func.task_earlycheck:
-            func.task_earlycheck(self)
-        if kind == 'pre':
-            fork_before = self.config.translation.fork_before
-            if fork_before:
-                fork_before, = self.backend_select_goals([fork_before])
-                if not fork_before in self.done and fork_before == goal:
-                    prereq = getattr(self, 'prereq_checkpt_%s' % goal, None)
-                    if prereq:
-                        prereq()
-                    from rpython.translator.goal import unixcheckpoint
-                    unixcheckpoint.restartable_point(auto='run')
+    def fork_before(self, goal):
+        fork_before = self.config.translation.fork_before
+        if fork_before:
+            fork_before, = self.backend_select_goals([fork_before])
+            if not fork_before in self.done and fork_before == goal:
+                prereq = getattr(self, 'prereq_checkpt_%s' % goal, None)
+                if prereq:
+                    prereq()
+                from rpython.translator.goal import unixcheckpoint
+                unixcheckpoint.restartable_point(auto='run')
 
 def mkexename(name):
     if sys.platform == 'win32':
diff --git a/rpython/translator/test/test_driver.py b/rpython/translator/test/test_driver.py
--- a/rpython/translator/test/test_driver.py
+++ b/rpython/translator/test/test_driver.py
@@ -5,9 +5,6 @@
 
 def test_ctr():
     td = TranslationDriver()
-    expected = ['annotate', 'backendopt', 'llinterpret', 'rtype', 'source',
-                'compile', 'pyjitpl']
-    assert set(td.exposed) == set(expected)
 
     assert td.backend_select_goals(['compile_c']) == ['compile_c']
     assert td.backend_select_goals(['compile']) == ['compile_c']
@@ -27,10 +24,6 @@
     assert td.backend_select_goals(['backendopt_lltype']) == [
         'backendopt_lltype']
 
-    expected = ['annotate', 'backendopt_lltype', 'llinterpret_lltype',
-                'rtype_lltype', 'source_c', 'compile_c', 'pyjitpl_lltype', ]
-    assert set(td.exposed) == set(expected)
-
     td = TranslationDriver({'backend': None, 'type_system': 'lltype'})
 
     assert td.backend_select_goals(['compile_c']) == ['compile_c']
@@ -41,11 +34,6 @@
     assert td.backend_select_goals(['backendopt_lltype']) == [
         'backendopt_lltype']
 
-    expected = ['annotate', 'backendopt', 'llinterpret', 'rtype', 'source_c',
-                'compile_c', 'pyjitpl']
-
-    assert set(td.exposed) == set(expected)
-
 
 def test_create_exe():
     if not os.name == 'nt':

From pypy.commits at gmail.com  Tue Jan 24 05:31:46 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Tue, 24 Jan 2017 02:31:46 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: Remove taskengine
Message-ID: <58872d12.8c7e1c0a.a8214.7cda@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89724:aa48dbd8e0e2
Date: 2017-01-24 21:30 +1100
http://bitbucket.org/pypy/pypy/changeset/aa48dbd8e0e2/

Log:	Remove taskengine

diff --git a/rpython/translator/tool/taskengine.py b/rpython/translator/tool/taskengine.py
deleted file mode 100644
--- a/rpython/translator/tool/taskengine.py
+++ /dev/null
@@ -1,130 +0,0 @@
-class SimpleTaskEngine(object):
-    def __init__(self):
-        self._plan_cache = {}
-
-        self.tasks = tasks = {}
-
-        for name in dir(self):
-            if name.startswith('task_'):
-                task_name = name[len('task_'):]
-                task = getattr(self, name)
-                assert callable(task)
-                task_deps = getattr(task, 'task_deps', [])
-
-                tasks[task_name] = task, task_deps
-
-    def _plan(self, goals, skip=[]):
-        skip = [toskip for toskip in skip if toskip not in goals]
-
-        key = (tuple(goals), tuple(skip))
-        try:
-            return self._plan_cache[key]
-        except KeyError:
-            pass
-        constraints = []
-
-        def subgoals(task_name):
-            taskcallable, deps = self.tasks[task_name]
-            for dep in deps:
-                if dep.startswith('??'): # optional
-                    dep = dep[2:]
-                    if dep not in goals:
-                        continue
-                if dep.startswith('?'): # suggested
-                    dep = dep[1:]
-                    if dep in skip:
-                        continue
-                yield dep
-
-        seen = {}
-
-        def consider(subgoal):
-            if subgoal in seen:
-                return
-            else:
-                seen[subgoal] = True
-            constraints.append([subgoal])
-            deps = subgoals(subgoal)
-            for dep in deps:
-                constraints.append([subgoal, dep])
-                consider(dep)
-
-        for goal in goals:
-            consider(goal)
-
-        #sort
-
-        plan = []
-
-        while True:
-            cands = dict.fromkeys([constr[0] for constr in constraints if constr])
-            if not cands:
-                break
-
-            for cand in cands:
-                for constr in constraints:
-                    if cand in constr[1:]:
-                        break
-                else:
-                    break
-            else:
-                raise RuntimeError("circular dependecy")
-
-            plan.append(cand)
-            for constr in constraints:
-                if constr and constr[0] == cand:
-                    del constr[0]
-
-        plan.reverse()
-
-        self._plan_cache[key] = plan
-
-        return plan
-
-    def _depending_on(self, goal):
-        l = []
-        for task_name, (task, task_deps) in self.tasks.iteritems():
-            if goal in task_deps:
-                l.append(task_name)
-        return l
-
-    def _depending_on_closure(self, goal):
-        d = {}
-
-        def track(goal):
-            if goal in d:
-                return
-            d[goal] = True
-            for depending in self._depending_on(goal):
-                track(depending)
-        track(goal)
-        return d.keys()
-
-    def _execute(self, goals, *args, **kwds):
-        task_skip = kwds.get('task_skip', [])
-        res = None
-        goals = self._plan(goals, skip=task_skip)
-        for goal in goals:
-            taskcallable, _ = self.tasks[goal]
-            self._event('planned', goal, taskcallable)
-        for goal in goals:
-            taskcallable, _ = self.tasks[goal]
-            self._event('pre', goal, taskcallable)
-            try:
-                res = self._do(goal, taskcallable, *args, **kwds)
-            except (SystemExit, KeyboardInterrupt):
-                raise
-            except:
-                self._error(goal)
-                raise
-            self._event('post', goal, taskcallable)
-        return res
-
-    def _do(self, goal, func, *args, **kwds):
-        return func()
-
-    def _event(self, kind, goal, func):
-        pass
-
-    def _error(self, goal):
-        pass
diff --git a/rpython/translator/tool/test/test_taskengine.py b/rpython/translator/tool/test/test_taskengine.py
deleted file mode 100644
--- a/rpython/translator/tool/test/test_taskengine.py
+++ /dev/null
@@ -1,150 +0,0 @@
-from rpython.translator.tool.taskengine import SimpleTaskEngine
-
-def test_simple():
-
-    class ABC(SimpleTaskEngine):
-
-        def task_A(self):
-            pass
-
-        task_A.task_deps = ['B', '?C']
-
-        def task_B(self):
-            pass
-
-        def task_C(self):
-            pass
-
-        task_C.task_deps = ['B']
-
-        def task_D(self):
-            pass
-        task_D.task_deps = ['E']
-
-        def task_E(self):
-            pass
-        task_E.task_deps = ['F']
-
-        def task_F(self):
-            pass
-
-    abc = ABC()
-
-    assert abc._plan('B') == ['B']
-    assert abc._plan('C') == ['B', 'C']
-    assert abc._plan('A') == ['B', 'C', 'A']
-    assert abc._plan('A', skip=['C']) == ['B', 'A']
-
-    assert abc._depending_on('C') == []
-    assert dict.fromkeys(abc._depending_on('B'), True) == {'A':True, 'C':True}
-    assert abc._depending_on('A') == []
-   
-    assert abc._depending_on('F') == ['E']
-    assert abc._depending_on('E') == ['D']
-    assert abc._depending_on('D') == []
-
-    assert abc._depending_on_closure('C') == ['C']
-    assert dict.fromkeys(abc._depending_on_closure('B'), True) == {'A':True, 'C':True, 'B': True}
-    assert abc._depending_on_closure('A') == ['A']
-   
-    assert dict.fromkeys(abc._depending_on_closure('F'), True) == {'D':True, 'E':True, 'F': True}
-    assert dict.fromkeys(abc._depending_on_closure('E'), True) == {'D':True, 'E':True}
-    assert abc._depending_on_closure('D') == ['D']
-
-
-def test_execute():
-
-    class ABC(SimpleTaskEngine):
-
-        def __init__(self):
-            SimpleTaskEngine.__init__(self)
-            self.done = []
-
-        def task_A(self):
-            self.done.append('A')
-
-        task_A.task_deps = ['B', '?C']
-
-        def task_B(self):
-            self.done.append('B')
-
-        def task_C(self):
-            self.done.append('C')
-
-        task_C.task_deps = ['B']
-
-        def _event(self, kind, goal, taskcallable):
-            self.done.append((kind, goal))
-
-    def test(goals, task_skip=[]):
-        if isinstance(goals, str):
-            goals = [goals]
-        abc = ABC()
-        abc._execute(goals, task_skip=task_skip)
-        return abc.done
-
-    def trace(goals):
-        t = []
-        for goal in goals:
-            t.append(('planned', goal))
-        for goal in goals:
-            t.extend([('pre', goal), goal, ('post', goal)])
-        return t
-
-    assert test('B') == trace('B')
-    assert test('C') == trace(['B', 'C'])
-    assert test('A') == trace(['B', 'C', 'A'])
-    assert test('A', ['C']) == trace(['B', 'A'])
-    assert test(['B', 'C']) == trace(['B', 'C'])
-    assert test(['C', 'B']) == trace(['B', 'C'])
-    assert test(['B', 'A']) == trace(['B', 'C', 'A'])
-    assert test(['B', 'A'], ['C']) == trace(['B', 'A'])
-    assert test(['B', 'A', 'C']) == trace(['B', 'C', 'A'])
-    assert test(['B', 'A', 'C'], ['C']) == trace(['B', 'C', 'A'])
-
-def test_driver():
-    class Drv(SimpleTaskEngine):
-
-        def task_A():
-            pass
-        task_A.task_deps = []
-
-        def task_R():
-            pass
-        task_R.task_deps = ['A']
-
-        def task_b():
-            pass
-        task_b.task_deps = ['R']
-
-        def task_H():
-            pass
-        task_H.task_deps = ['b']
-
-        def task_T():
-            pass
-        task_T.task_deps = ['H']
-
-        def task_B():
-            pass
-        task_B.task_deps = ['R', '??T']
-
-        def task_D():
-            pass
-        task_D.task_deps = ['R', '?B', '?A', '??T']
-
-    drv = Drv()
-    assert drv._plan(['R']) == ['A', 'R']
-    assert drv._plan(['B']) == ['A', 'R', 'B']
-    assert drv._plan(['D']) == ['A', 'R', 'B', 'D']
-    assert drv._plan(['D'], skip=['B']) == ['A', 'R', 'D']
-    assert drv._plan(['D', 'R']) == ['A', 'R', 'B', 'D']
-
-
-    assert drv._plan(['H', 'R']) == ['A', 'R', 'b', 'H']
-    assert drv._plan(['H']) == ['A', 'R', 'b', 'H']
-    assert drv._plan(['T', 'B']) == ['A', 'R', 'b', 'H', 'T', 'B']
-    assert drv._plan(['D', 'T']) == ['A', 'R', 'b', 'H', 'T', 'B', 'D']
-    assert drv._plan(['D', 'T', 'R']) == ['A', 'R', 'b', 'H', 'T', 'B', 'D']
-    assert drv._plan(['D', 'T']) == ['A', 'R', 'b', 'H', 'T', 'B', 'D']
-    assert drv._plan(['D', 'T'], skip=['B']) == ['A', 'R', 'b', 'H', 'T', 'D']

From pypy.commits at gmail.com  Tue Jan 24 05:46:23 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Tue, 24 Jan 2017 02:46:23 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: Quit early if there
 is nothing to do
Message-ID: <5887307f.2ea6df0a.2d470.48ea@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89725:6429b253dfd2
Date: 2017-01-24 21:45 +1100
http://bitbucket.org/pypy/pypy/changeset/6429b253dfd2/

Log:	Quit early if there is nothing to do

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -192,6 +192,9 @@
     def _proceed_inner(self, goals):
         backend, ts = self.get_backend_and_type_system()
         goals = set(self.backend_select_goals(goals + self.extra_goals))
+        if not goals:
+            self.log('Nothing to do.')
+            raise Done(None)
 
         if any(cgoal in goals
                for bakgoal in ['database', 'source', 'compile']

From pypy.commits at gmail.com  Tue Jan 24 05:54:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 02:54:56 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: Leysin 2017
Message-ID: <58873280.42061c0a.84b10.acf5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5764:7a0473301e83
Date: 2017-01-24 11:54 +0100
http://bitbucket.org/pypy/extradoc/changeset/7a0473301e83/

Log:	Leysin 2017

diff --git a/sprintinfo/leysin-winter-2017/announcement.txt b/sprintinfo/leysin-winter-2017/announcement.txt
new file mode 100644
--- /dev/null
+++ b/sprintinfo/leysin-winter-2017/announcement.txt
@@ -0,0 +1,74 @@
+=====================================================================
+      PyPy Leysin Winter Sprint (25/26th Feb. - 4th March 2016)
+=====================================================================
+
+The next PyPy sprint will be in Leysin, Switzerland, for the twelveth time.
+This is a fully public sprint: newcomers and topics other than those
+proposed below are welcome.
+
+------------------------------
+Goals and topics of the sprint
+------------------------------
+
+The list of topics is very open.
+
+* The main topic is Python 3.5 support in PyPy, as most py3.5
+  contributors should be present.  It is also a good topic if you have
+  no or limited experience with PyPy contribution: we can easily find
+  something semi-independent that is not done in py3.5 so far, and
+  do pair-programming with you.
+
+* Any other topic is fine too: JIT compiler optimizations, CFFI,
+  the RevDB reverse debugger, improving to speed of your program on
+  PyPy, etc.
+
+* And as usual, the main side goal is to have fun in winter sports :-)
+  We can take a day off (for ski or anything else).
+
+-----------
+Exact times
+-----------
+
+Work days: starting 26th Feb (~noon), ending March 4th (~noon).
+
+I have pre-booked the week from Saturday Feb 25th to Saturday March 4th.
+If it is possible for you to arrive Sunday before mid-afternoon, then
+you should get a booking from Sunday only.  The break day should be
+around Wednesday.
+
+It is fine to stay a few more days on either side, or conversely to book
+for a part of that time only.
+
+-----------------------
+Location & Accomodation
+-----------------------
+
+Leysin, Switzerland, "same place as before".  Let me refresh your
+memory: both the sprint venue and the lodging will be in a
+pair of chalets built specifically for bed & breakfast:
+http://www.ermina.ch/.  The place has a good ADSL Internet connection
+with wireless installed.  You can also arrange your own lodging
+elsewhere (as long as you are in Leysin, you cannot be more than a 15
+minutes walk away from the sprint venue).
+
+Please *confirm* that you are coming so that we can adjust the
+reservations as appropriate.
+
+The options of rooms are a bit more limited than on previous years
+because the place for bed-and-breakfast is shrinking; but we should
+still have enough room for us.  The price is around 60 CHF, breakfast
+included, in shared rooms (3 or 4 people).  If there are people that
+would prefer a double or single room, please contact me and we'll see
+what choices you have.  There are also a choice of hotels in Leysin.
+
+Please register by Mercurial::
+
+  https://bitbucket.org/pypy/extradoc/
+  https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/leysin-winter-2017
+
+or on the pypy-dev mailing list if you do not yet have check-in rights:
+
+  http://mail.python.org/mailman/listinfo/pypy-dev
+
+You need a Swiss-to-(insert country here) power adapter.  There will be
+some Swiss-to-EU adapters around, and at least one EU-format power strip.
diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
new file mode 100644
--- /dev/null
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -0,0 +1,25 @@
+
+People coming to the Leysin sprint Winter 2016
+==================================================
+
+People who have a ``?`` in their arrive/depart or accomodation
+column are known to be coming but there are no details 
+available yet from them.
+
+==================== ============== =======================
+    Name              Arrive/Depart     Accomodation 
+==================== ============== =======================
+Armin Rigo                          private
+==================== ============== =======================
+
+**NOTE:** lodging is by default in Ermina.  There are two ~4 people
+rooms as well as smaller rooms on demand.  (Individual rooms should be
+available if needed, but ask Armin Rigo.)
+
+Of course you're free to book a different hotel yourself if you prefer.
+
+The standard booking is for the nights from Saturday to Saturday, but it
+is possible to extend that.  If it is possible for you to arrive Sunday
+not too late, then you should get a booking from Sunday only.  We'll
+work from Sunday noon-ish to the next Saturday (with a break day in the
+middle of the week).

From pypy.commits at gmail.com  Tue Jan 24 05:56:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 02:56:27 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: typo
Message-ID: <588732db.c3a4df0a.a0d6.53dc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5765:fcf6157938b6
Date: 2017-01-24 11:56 +0100
http://bitbucket.org/pypy/extradoc/changeset/fcf6157938b6/

Log:	typo

diff --git a/sprintinfo/leysin-winter-2017/announcement.txt b/sprintinfo/leysin-winter-2017/announcement.txt
--- a/sprintinfo/leysin-winter-2017/announcement.txt
+++ b/sprintinfo/leysin-winter-2017/announcement.txt
@@ -1,5 +1,5 @@
 =====================================================================
-      PyPy Leysin Winter Sprint (25/26th Feb. - 4th March 2016)
+      PyPy Leysin Winter Sprint (25/26th Feb. - 4th March 2017)
 =====================================================================
 
 The next PyPy sprint will be in Leysin, Switzerland, for the twelveth time.
diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -1,5 +1,5 @@
 
-People coming to the Leysin sprint Winter 2016
+People coming to the Leysin sprint Winter 2017
 ==================================================
 
 People who have a ``?`` in their arrive/depart or accomodation

From pypy.commits at gmail.com  Tue Jan 24 06:03:28 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 03:03:28 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: update link
Message-ID: <58873480.4b371c0a.1c8fb.2e92@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5766:37959844c889
Date: 2017-01-24 12:03 +0100
http://bitbucket.org/pypy/extradoc/changeset/37959844c889/

Log:	update link

diff --git a/sprintinfo/leysin-winter-2017/announcement.txt b/sprintinfo/leysin-winter-2017/announcement.txt
--- a/sprintinfo/leysin-winter-2017/announcement.txt
+++ b/sprintinfo/leysin-winter-2017/announcement.txt
@@ -64,7 +64,7 @@
 Please register by Mercurial::
 
   https://bitbucket.org/pypy/extradoc/
-  https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/leysin-winter-2017
+  https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/leysin-winter-2017/
 
 or on the pypy-dev mailing list if you do not yet have check-in rights:
 

From pypy.commits at gmail.com  Tue Jan 24 06:24:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 03:24:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: test and fix + tweaks
Message-ID: <5887395d.ccaddf0a.320ea.5d51@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89726:086cef12dead
Date: 2017-01-24 12:19 +0100
http://bitbucket.org/pypy/pypy/changeset/086cef12dead/

Log:	test and fix + tweaks

diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -50,8 +50,10 @@
                                consts.PyCF_SOURCE_IS_UTF8 |
                                consts.PyCF_IGNORE_COOKIE,
                                optimize=astbuilder.compile_info.optimize)
-    parse_tree = astbuilder.recursive_parser.parse_source(source, info)
-    return ast_from_node(astbuilder.space, parse_tree, info)
+    parser = astbuilder.recursive_parser
+    parse_tree = parser.parse_source(source, info)
+    return ast_from_node(astbuilder.space, parse_tree, info,
+                         recursive_parser=parser)
 
 
 def unexpected_end_of_string(astbuilder, atom_node):
@@ -257,7 +259,8 @@
 
     fstr.current_index = i
     literal = builder.build()
-    if not fstr.raw_mode:
+    if not fstr.raw_mode and u'\\' in literal:
+        literal = literal.encode('utf-8')
         literal = unicodehelper.decode_unicode_escape(astbuilder.space, literal)
     return literal
 
@@ -281,7 +284,10 @@
 
 
 def parse_f_string(astbuilder, joined_pieces, fstr, atom_node, rec=0):
-    space = astbuilder.space
+    # In our case, parse_f_string() and fstring_find_literal_and_expr()
+    # could be merged into a single function with a clearer logic.  It's
+    # done this way to follow CPython's source code more closely.
+
     while True:
         literal, expr = fstring_find_literal_and_expr(astbuilder, fstr,
                                                       atom_node, rec)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1184,6 +1184,8 @@
 
         yield self.st, """z=f'{"}"}'""", 'z', '}'
 
+        yield self.st, """z=f'{f"{0}"*3}'""", 'z', '000'
+
     def test_fstring_error(self):
         raises(SyntaxError, self.run, "f'{}'")
         raises(SyntaxError, self.run, "f'{   \t   }'")

From pypy.commits at gmail.com  Tue Jan 24 06:51:11 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Tue, 24 Jan 2017 03:51:11 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: Remove unused
 methods
Message-ID: <58873faf.54b31c0a.8e3a4.9b56@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89727:7cff1328be02
Date: 2017-01-24 21:59 +1100
http://bitbucket.org/pypy/pypy/changeset/7cff1328be02/

Log:	Remove unused methods

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -86,18 +86,6 @@
     def annotate(self):
         return self.proceed(['annotate'])
 
-    def rtype(self):
-        return self.proceed(['rtype'])
-
-    def backendopt(self):
-        return self.proceed(['backendopt'])
-
-    def llinterpret(self):
-        return self.proceed(['llinterpret'])
-
-    def pyjitpl(self):
-        return self.proceed(['pyjitpl'])
-
     def rtype_lltype(self):
         return self.proceed(['rtype_lltype'])
 
@@ -110,15 +98,6 @@
     def pyjitpl_lltype(self):
         return self.proceed(['pyjitpl_lltype'])
 
-    def source(self):
-        return self.proceed(['source'])
-
-    def compile(self):
-        return self.proceed(['compile'])
-
-    def run(self):
-        return self.proceed(['run'])
-
     def source_c(self):
         return self.proceed(['source_c'])
 

From pypy.commits at gmail.com  Tue Jan 24 06:51:13 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Tue, 24 Jan 2017 03:51:13 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: Call the dependent
 function before running your task. Straightforward.
Message-ID: <58873fb1.04ba1c0a.40ed2.9de9@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89728:8ffe9002f13b
Date: 2017-01-24 22:50 +1100
http://bitbucket.org/pypy/pypy/changeset/8ffe9002f13b/

Log:	Call the dependent function before running your task.
	Straightforward.

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -17,8 +17,6 @@
 
 log = AnsiLogger("translation")
 
-class Done(Exception): pass
-
 # TODO:
 # sanity-checks using states
 
@@ -84,28 +82,57 @@
         self.extra_goals = []
 
     def annotate(self):
-        return self.proceed(['annotate'])
+        return self.run_task(self.task_annotate, 'annotate')
 
     def rtype_lltype(self):
-        return self.proceed(['rtype_lltype'])
+        self.annotate()
+        return self.run_task(self.task_rtype_lltype, 'rtype_lltype')
+
+    def pyjitpl_lltype(self):
+        self.rtype_lltype()
+        return self.run_task(self.task_pyjitpl_lltype, 'pyjitpl_lltype')
+
+    def jittest_lltype(self):
+        self.rtype_lltype()
+        return self.run_task(self.task_jittest_lltype, 'jittest_lltype')
 
     def backendopt_lltype(self):
-        return self.proceed(['backendopt_lltype'])
+        self.rtype_lltype()
+        if 'pyjitpl' in self.extra_goals:
+            self.pyjitpl_lltype()
+        if 'jittest' in self.extra_goals:
+            self.jittest_lltype()
+        
+        return self.run_task(self.task_backendopt_lltype,
+                             'backendopt_lltype')
+
+    def stackcheckinsertion_lltype(self):
+        self.rtype_lltype()
+        if 'backendopt' in self.extra_goals:
+            self.backendopt_lltype()
+        return self.run_task(self.task_stackcheckinsertion_lltype,
+                             'stackcheckinsertion_lltype')
 
     def llinterpret_lltype(self):
-        return self.proceed(['llinterpret_lltype'])
-
-    def pyjitpl_lltype(self):
-        return self.proceed(['pyjitpl_lltype'])
+        self.stackcheckinsertion_lltype()
+        return self.run_task(self.task_llinterpret_lltype,
+                             'llinterpret_lltype')
 
     def source_c(self):
-        return self.proceed(['source_c'])
+        if 'check_for_boehm' not in self.done:
+            self.possibly_check_for_boehm()
+            self.done.add('check_for_boehm')
+        self.stackcheckinsertion_lltype()
+        self.run_task(self.task_database_c, 'database_c')
+        return self.run_task(self.task_source_c, 'source_c')
 
     def compile_c(self):
-        return self.proceed(['compile_c'])
+        self.source_c()
+        return self.run_task(self.task_compile_c, 'compile_c')
 
     def run_c(self):
-        return self.proceed(['run_c'])
+        self.compile_c()
+        return self.run_task(self.task_run_c, 'run_c')
 
     def set_extra_goals(self, goals):
         self.extra_goals = goals
@@ -122,7 +149,7 @@
         backend = self.config.translation.backend
         return backend, type_system
 
-    def run_task(self, name, goals, *args, **kwargs):
+    def run_task(self, task, name, *args, **kwargs):
         if name in self.done or name in self._disabled:
             return
         task = getattr(self, 'task_%s' % name)
@@ -157,47 +184,21 @@
         self.log.info('usession directory: %s' % (udir,))
 
         self.done.add(name)
-        goals.discard(name)
-        if not goals:
-            raise Done(res)
         return res
 
     def proceed(self, goals):
-        try:
-            self._proceed_inner(goals)
-        except Done as d:
-            return d.args[0]
-
-    def _proceed_inner(self, goals):
         backend, ts = self.get_backend_and_type_system()
         goals = set(self.backend_select_goals(goals + self.extra_goals))
         if not goals:
             self.log('Nothing to do.')
-            raise Done(None)
+        self.extra_goals += goals
 
-        if any(cgoal in goals
-               for bakgoal in ['database', 'source', 'compile']
-               for cgoal in [bakgoal, bakgoal + '_c']):
-            if 'check_for_boehm' not in self.done:
-                self.possibly_check_for_boehm()
-                self.done.add('check_for_boehm')
-
-        self.run_task('annotate', goals)
-        self.run_task('rtype_lltype', goals)
-        if 'pyjitpl_lltype' in goals or 'jittest_lltype' in goals:
-            self.run_task('pyjitpl_lltype', goals)
-        if 'jittest_lltype' in goals:
-            self.run_task('jittest_lltype', goals)
-        self.run_task('backendopt_lltype', goals)
-        self.run_task('stackcheckinsertion_lltype', goals)
-        if 'llinterpret_lltype' in goals:
-            self.run_task('llinterpret_lltype', goals)
-        self.run_task('backend_%s' % backend, goals, goals)
-
-    def task_backend_c(self, goals):
-        self.run_task('database_c', goals)
-        self.run_task('source_c', goals)
-        self.run_task('compile_c', goals)
+        # run C goals first to catch missing boehm.
+        for goal in goals:
+            if goal.endswith('_c'):
+                getattr(self, goal)()
+        for goal in goals:
+            getattr(self, goal)()
 
     def backend_select_goals(self, goals):
         backend, ts = self.get_backend_and_type_system()

From pypy.commits at gmail.com  Tue Jan 24 06:57:08 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Tue, 24 Jan 2017 03:57:08 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: Move
 backend_select_goals up near get_backend_and_type_system to be less
 confusing
Message-ID: <58874114.d5091c0a.30ed4.ad72@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89729:fa9749741225
Date: 2017-01-24 22:56 +1100
http://bitbucket.org/pypy/pypy/changeset/fa9749741225/

Log:	Move backend_select_goals up near get_backend_and_type_system to be
	less confusing

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -149,6 +149,27 @@
         backend = self.config.translation.backend
         return backend, type_system
 
+    def backend_select_goals(self, goals):
+        backend, ts = self.get_backend_and_type_system()
+        result = []
+        for goal in goals:
+            names = ['task_%s_%s' % (goal, backend),
+                     'task_%s_%s' % (goal, ts),
+                     'task_%s' % (goal,)]
+            if set(names).intersection(self.done):
+                continue
+            for name in names:
+                task = getattr(self, name, None)
+                if task is not None:
+                    result.append(name[len('task_'):])
+                    break
+            else:
+                raise Exception("cannot infer complete goal from: %r" % goal)
+        return result
+            
+    def disable(self, to_disable):
+        self._disabled = to_disable
+
     def run_task(self, task, name, *args, **kwargs):
         if name in self.done or name in self._disabled:
             return
@@ -200,27 +221,6 @@
         for goal in goals:
             getattr(self, goal)()
 
-    def backend_select_goals(self, goals):
-        backend, ts = self.get_backend_and_type_system()
-        result = []
-        for goal in goals:
-            names = ['task_%s_%s' % (goal, backend),
-                     'task_%s_%s' % (goal, ts),
-                     'task_%s' % (goal,)]
-            if set(names).intersection(self.done):
-                continue
-            for name in names:
-                task = getattr(self, name, None)
-                if task is not None:
-                    result.append(name[len('task_'):])
-                    break
-            else:
-                raise Exception("cannot infer complete goal from: %r" % goal)
-        return result
-            
-    def disable(self, to_disable):
-        self._disabled = to_disable
-
     def setup(self, entry_point, inputtypes, policy=None, extra={}, empty_translator=None):
         standalone = inputtypes is None
         self.standalone = standalone

From pypy.commits at gmail.com  Tue Jan 24 07:32:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 04:32:57 -0800 (PST)
Subject: [pypy-commit] pypy default: Update to cffi/b0204d9d8b96
Message-ID: <58874979.ad8ddf0a.8c974.7760@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89730:c30204eb6247
Date: 2017-01-24 13:04 +0100
http://bitbucket.org/pypy/pypy/changeset/c30204eb6247/

Log:	Update to cffi/b0204d9d8b96

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.9.2
+Version: 1.10.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -1,11 +1,11 @@
 __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError',
            'FFIError']
 
-from .api import FFI, CDefError, FFIError
-from .ffiplatform import VerificationError, VerificationMissing
+from .api import FFI
+from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.9.2"
-__version_info__ = (1, 9, 2)
+__version__ = "1.10.0"
+__version_info__ = (1, 10, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.9.2"
+                               "\ncompiled with cffi version: 1.10.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -1,5 +1,7 @@
 import sys, types
 from .lock import allocate_lock
+from .error import CDefError
+from . import model
 
 try:
     callable
@@ -15,17 +17,6 @@
     basestring = str
 
 
-class FFIError(Exception):
-    pass
-
-class CDefError(Exception):
-    def __str__(self):
-        try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
-        except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
-
 
 class FFI(object):
     r'''
@@ -49,18 +40,27 @@
         """Create an FFI instance.  The 'backend' argument is used to
         select a non-default backend, mostly for tests.
         """
-        from . import cparser, model
         if backend is None:
             # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with
             # _cffi_backend.so compiled.
             import _cffi_backend as backend
             from . import __version__
-            assert backend.__version__ == __version__, \
-               "version mismatch, %s != %s" % (backend.__version__, __version__)
+            if backend.__version__ != __version__:
+                # bad version!  Try to be as explicit as possible.
+                if hasattr(backend, '__file__'):
+                    # CPython
+                    raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r.  When we import the top-level '_cffi_backend' extension module, we get version %s, located in %r.  The two versions should be equal; check your installation." % (
+                        __version__, __file__,
+                        backend.__version__, backend.__file__))
+                else:
+                    # PyPy
+                    raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r.  This interpreter comes with a built-in '_cffi_backend' module, which is version %s.  The two versions should be equal; check your installation." % (
+                        __version__, __file__, backend.__version__))
             # (If you insist you can also try to pass the option
             # 'backend=backend_ctypes.CTypesBackend()', but don't
             # rely on it!  It's probably not going to work well.)
 
+        from . import cparser
         self._backend = backend
         self._lock = allocate_lock()
         self._parser = cparser.Parser()
@@ -212,7 +212,7 @@
 
     def offsetof(self, cdecl, *fields_or_indexes):
         """Return the offset of the named field inside the given
-        structure or array, which must be given as a C type name.  
+        structure or array, which must be given as a C type name.
         You can give several field names in case of nested structures.
         You can also give numeric values which correspond to array
         items, in case of an array type.
@@ -300,7 +300,7 @@
         return self._backend.string(cdata, maxlen)
 
     def unpack(self, cdata, length):
-        """Unpack an array of C data of the given length, 
+        """Unpack an array of C data of the given length,
         returning a Python string/unicode/list.
 
         If 'cdata' is a pointer to 'char', returns a byte string.
@@ -452,7 +452,6 @@
         return self._backend.getwinerror(code)
 
     def _pointer_to(self, ctype):
-        from . import model
         with self._lock:
             return model.pointer_cache(self, ctype)
 
@@ -764,7 +763,6 @@
         return backend.load_library(path, flags)
 
 def _make_ffi_library(ffi, libname, flags):
-    import os
     backend = ffi._backend
     backendlib = _load_backend_lib(backend, libname, flags)
     #
@@ -802,7 +800,6 @@
         if accessors_version[0] is ffi._cdef_version:
             return
         #
-        from . import model
         for key, (tp, _) in ffi._parser._declarations.items():
             if not isinstance(tp, model.EnumType):
                 tag, name = key.split(' ', 1)
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -1,3 +1,4 @@
+from .error import VerificationError
 
 class CffiOp(object):
     def __init__(self, op, arg):
@@ -19,7 +20,6 @@
                                     % (self.arg,))
             return format_four_bytes(value)
         if isinstance(self.arg, str):
-            from .ffiplatform import VerificationError
             raise VerificationError("cannot emit to Python: %r" % (self.arg,))
         return format_four_bytes((self.arg << 8) | self.op)
 
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -1,5 +1,6 @@
 import sys
-from . import api, model
+from . import model
+from .error import FFIError
 
 
 COMMON_TYPES = {}
@@ -31,11 +32,11 @@
         elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result, quals = model.PrimitiveType(cdecl), 0
         elif cdecl == 'set-unicode-needed':
-            raise api.FFIError("The Windows type %r is only available after "
-                               "you call ffi.set_unicode()" % (commontype,))
+            raise FFIError("The Windows type %r is only available after "
+                           "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == cdecl:
-                raise api.FFIError(
+                raise FFIError(
                     "Unsupported type: %r.  Please look at "
         "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
                     "and file an issue if you think this type should really "
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -1,5 +1,6 @@
-from . import api, model
+from . import model
 from .commontypes import COMMON_TYPES, resolve_common_type
+from .error import FFIError, CDefError
 try:
     from . import _pycparser as pycparser
 except ImportError:
@@ -113,7 +114,7 @@
             # grouping variant
             closing = csource.find('}', endpos)
             if closing < 0:
-                raise api.CDefError("'extern \"Python\" {': no '}' found")
+                raise CDefError("'extern \"Python\" {': no '}' found")
             if csource.find('{', endpos + 1, closing) >= 0:
                 raise NotImplementedError("cannot use { } inside a block "
                                           "'extern \"Python\" { ... }'")
@@ -123,7 +124,7 @@
             # non-grouping variant
             semicolon = csource.find(';', endpos)
             if semicolon < 0:
-                raise api.CDefError("'extern \"Python\": no ';' found")
+                raise CDefError("'extern \"Python\": no ';' found")
             parts.append(csource[endpos:semicolon+1])
             csource = csource[semicolon+1:]
         parts.append(' void __cffi_extern_python_stop;')
@@ -288,7 +289,7 @@
             msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
-        raise api.CDefError(msg)
+        raise CDefError(msg)
 
     def parse(self, csource, override=False, packed=False, dllexport=False):
         prev_options = self._options
@@ -318,8 +319,8 @@
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
                     if not decl.name:
-                        raise api.CDefError("typedef does not declare any name",
-                                            decl)
+                        raise CDefError("typedef does not declare any name",
+                                        decl)
                     quals = 0
                     if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
                             and decl.type.type.names[-1] == '__dotdotdot__'):
@@ -337,8 +338,8 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise api.CDefError("unrecognized construct", decl)
-        except api.FFIError as e:
+                    raise CDefError("unrecognized construct", decl)
+        except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
                 e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
@@ -348,7 +349,7 @@
         if key in self._int_constants:
             if self._int_constants[key] == val:
                 return     # ignore identical double declarations
-            raise api.FFIError(
+            raise FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
 
@@ -375,7 +376,7 @@
             elif value == '...':
                 self._declare('macro ' + key, value)
             else:
-                raise api.CDefError(
+                raise CDefError(
                     'only supports one of the following syntax:\n'
                     '  #define %s ...     (literally dot-dot-dot)\n'
                     '  #define %s NUMBER  (with NUMBER an integer'
@@ -410,8 +411,8 @@
             elif isinstance(node, pycparser.c_ast.Enum):
                 self._get_struct_union_enum_type('enum', node)
             elif not decl.name:
-                raise api.CDefError("construct does not declare any variable",
-                                    decl)
+                raise CDefError("construct does not declare any variable",
+                                decl)
             #
             if decl.name:
                 tp, quals = self._get_type_and_quals(node,
@@ -438,7 +439,7 @@
                     self._inside_extern_python = decl.name
                 else:
                     if self._inside_extern_python !='__cffi_extern_python_stop':
-                        raise api.CDefError(
+                        raise CDefError(
                             "cannot declare constants or "
                             "variables with 'extern \"Python\"'")
                     if (quals & model.Q_CONST) and not tp.is_array_type:
@@ -454,7 +455,7 @@
         assert not macros
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
-            raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
+            raise CDefError("unknown identifier '%s'" % (exprnode.name,))
         return self._get_type_and_quals(exprnode.type)
 
     def _declare(self, name, obj, included=False, quals=0):
@@ -463,7 +464,7 @@
             if prevobj is obj and prevquals == quals:
                 return
             if not self._options.get('override'):
-                raise api.FFIError(
+                raise FFIError(
                     "multiple declarations of %s (for interactive usage, "
                     "try cdef(xx, override=True))" % (name,))
         assert '__dotdotdot__' not in name.split()
@@ -551,7 +552,7 @@
                 if ident == 'void':
                     return model.void_type, quals
                 if ident == '__dotdotdot__':
-                    raise api.FFIError(':%d: bad usage of "..."' %
+                    raise FFIError(':%d: bad usage of "..."' %
                             typenode.coord.line)
                 tp0, quals0 = resolve_common_type(self, ident)
                 return tp0, (quals | quals0)
@@ -583,14 +584,14 @@
             return self._get_struct_union_enum_type('union', typenode, name,
                                                     nested=True), 0
         #
-        raise api.FFIError(":%d: bad or unsupported type declaration" %
+        raise FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
     def _parse_function_type(self, typenode, funcname=None):
         params = list(getattr(typenode.args, 'params', []))
         for i, arg in enumerate(params):
             if not hasattr(arg, 'type'):
-                raise api.CDefError("%s arg %d: unknown type '%s'"
+                raise CDefError("%s arg %d: unknown type '%s'"
                     " (if you meant to use the old C syntax of giving"
                     " untyped arguments, it is not supported)"
                     % (funcname or 'in expression', i + 1,
@@ -604,7 +605,7 @@
         if ellipsis:
             params.pop()
             if not params:
-                raise api.CDefError(
+                raise CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
         args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type))
@@ -705,7 +706,7 @@
             return tp
         #
         if tp.fldnames is not None:
-            raise api.CDefError("duplicate declaration of struct %s" % name)
+            raise CDefError("duplicate declaration of struct %s" % name)
         fldnames = []
         fldtypes = []
         fldbitsize = []
@@ -749,7 +750,7 @@
 
     def _make_partial(self, tp, nested):
         if not isinstance(tp, model.StructOrUnion):
-            raise api.CDefError("%s cannot be partial" % (tp,))
+            raise CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
         tp.partial = True
@@ -769,7 +770,7 @@
                     len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
                 return ord(s[-2])
             else:
-                raise api.CDefError("invalid constant %r" % (s,))
+                raise CDefError("invalid constant %r" % (s,))
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '+'):
@@ -788,12 +789,12 @@
             if partial_length_ok:
                 self._partial_length = True
                 return '...'
-            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
-                               "the actual array length in this context"
-                               % exprnode.coord.line)
+            raise FFIError(":%d: unsupported '[...]' here, cannot derive "
+                           "the actual array length in this context"
+                           % exprnode.coord.line)
         #
-        raise api.FFIError(":%d: unsupported expression: expected a "
-                           "simple numeric constant" % exprnode.coord.line)
+        raise FFIError(":%d: unsupported expression: expected a "
+                       "simple numeric constant" % exprnode.coord.line)
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
@@ -843,8 +844,8 @@
             for t in typenames[:-1]:
                 if t not in ['int', 'short', 'long', 'signed',
                              'unsigned', 'char']:
-                    raise api.FFIError(':%d: bad usage of "..."' %
-                                       decl.coord.line)
+                    raise FFIError(':%d: bad usage of "..."' %
+                                   decl.coord.line)
             result = model.UnknownIntegerType(decl.name)
 
         if self._uses_new_feature is None:
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -1,14 +1,5 @@
 import sys, os
-
-
-class VerificationError(Exception):
-    """ An error raised when verification fails
-    """
-
-class VerificationMissing(Exception):
-    """ An error raised when incomplete structures are passed into
-    cdef, but no verification has been done
-    """
+from .error import VerificationError
 
 
 LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -1,8 +1,8 @@
-import types, sys
+import types
 import weakref
 
 from .lock import allocate_lock
-
+from .error import CDefError, VerificationError, VerificationMissing
 
 # type qualifiers
 Q_CONST    = 0x01
@@ -39,7 +39,6 @@
         replace_with = qualify(quals, replace_with)
         result = result.replace('&', replace_with)
         if '$' in result:
-            from .ffiplatform import VerificationError
             raise VerificationError(
                 "cannot generate '%s' in %s: unknown type name"
                 % (self._get_c_name(), context))
@@ -223,9 +222,8 @@
     is_raw_function = True
 
     def build_backend_type(self, ffi, finishlist):
-        from . import api
-        raise api.CDefError("cannot render the type %r: it is a function "
-                            "type, not a pointer-to-function type" % (self,))
+        raise CDefError("cannot render the type %r: it is a function "
+                        "type, not a pointer-to-function type" % (self,))
 
     def as_function_pointer(self):
         return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi)
@@ -307,9 +305,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         if self.length == '...':
-            from . import api
-            raise api.CDefError("cannot render the type %r: unknown length" %
-                                (self,))
+            raise CDefError("cannot render the type %r: unknown length" %
+                            (self,))
         self.item.get_cached_btype(ffi, finishlist)   # force the item BType
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
@@ -455,13 +452,11 @@
         self.completed = 2
 
     def _verification_error(self, msg):
-        from .ffiplatform import VerificationError
         raise VerificationError(msg)
 
     def check_not_partial(self):
         if self.partial and self.fixedlayout is None:
-            from . import ffiplatform
-            raise ffiplatform.VerificationMissing(self._get_c_name())
+            raise VerificationMissing(self._get_c_name())
 
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
@@ -499,8 +494,7 @@
 
     def check_not_partial(self):
         if self.partial and not self.partial_resolved:
-            from . import ffiplatform
-            raise ffiplatform.VerificationMissing(self._get_c_name())
+            raise VerificationMissing(self._get_c_name())
 
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
@@ -514,7 +508,6 @@
         if self.baseinttype is not None:
             return self.baseinttype.get_cached_btype(ffi, finishlist)
         #
-        from . import api
         if self.enumvalues:
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
@@ -549,8 +542,8 @@
         if (smallest_value >= ((-1) << (8*size2-1)) and
             largest_value < (1 << (8*size2-sign))):
             return btype2
-        raise api.CDefError("%s values don't all fit into either 'long' "
-                            "or 'unsigned long'" % self._get_c_name())
+        raise CDefError("%s values don't all fit into either 'long' "
+                        "or 'unsigned long'" % self._get_c_name())
 
 def unknown_type(name, structname=None):
     if structname is None:
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -1,5 +1,6 @@
 import os, sys, io
 from . import ffiplatform, model
+from .error import VerificationError
 from .cffi_opcode import *
 
 VERSION = "0x2601"
@@ -211,7 +212,7 @@
                 method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in recompile(): %r" % name)
             try:
                 self._current_quals = quals
@@ -354,12 +355,12 @@
                     included_module_name, included_source = (
                         ffi_to_include._assigned_source[:2])
                 except AttributeError:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "ffi object %r includes %r, but the latter has not "
                         "been prepared with set_source()" % (
                             self.ffi, ffi_to_include,))
                 if included_source is None:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "not implemented yet: ffi.include() of a Python-based "
                         "ffi inside a C-based ffi")
                 prnt('  "%s",' % (included_module_name,))
@@ -391,6 +392,10 @@
         prnt()
         #
         # the init function
+        prnt('#ifdef __GNUC__')
+        prnt('#  pragma GCC visibility push(default)  /* for -fvisibility= */')
+        prnt('#endif')
+        prnt()
         prnt('#ifdef PYPY_VERSION')
         prnt('PyMODINIT_FUNC')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
@@ -429,6 +434,10 @@
             self.module_name, version))
         prnt('}')
         prnt('#endif')
+        prnt()
+        prnt('#ifdef __GNUC__')
+        prnt('#  pragma GCC visibility pop')
+        prnt('#endif')
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -456,12 +465,12 @@
                 included_module_name, included_source = (
                     ffi_to_include._assigned_source[:2])
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "ffi object %r includes %r, but the latter has not "
                     "been prepared with set_source()" % (
                         self.ffi, ffi_to_include,))
             if included_source is not None:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented yet: ffi.include() of a C-based "
                     "ffi inside a Python-based ffi")
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
@@ -831,7 +840,7 @@
                 prnt('  { %s = &p->%s; (void)tmp; }' % (
                     ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                     fname))
-            except ffiplatform.VerificationError as e:
+            except VerificationError as e:
                 prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname))
@@ -994,7 +1003,7 @@
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
                             check_value=None):
         if (category, name) in self._seen_constants:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "duplicate declaration of %s '%s'" % (category, name))
         self._seen_constants.add((category, name))
         #
@@ -1093,7 +1102,7 @@
     def _generate_cpy_macro_ctx(self, tp, name):
         if tp == '...':
             if self.target_is_python:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "cannot use the syntax '...' in '#define %s ...' when "
                     "using the ABI mode" % (name,))
             check_value = None
@@ -1226,7 +1235,7 @@
 
     def _generate_cpy_extern_python_ctx(self, tp, name):
         if self.target_is_python:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "cannot use 'extern \"Python\"' in the ABI mode")
         if tp.ellipsis:
             raise NotImplementedError("a vararg function is extern \"Python\"")
@@ -1307,7 +1316,7 @@
         if tp.length is None:
             self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index)
         elif tp.length == '...':
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "type %s badly placed: the '...' array length can only be "
                 "used on global arrays or on fields of structures" % (
                     str(tp).replace('/*...*/', '...'),))
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -79,9 +79,10 @@
     CPython itself should ignore the flag in a debugging version
     (by not listing .abi3.so in the extensions it supports), but
     it doesn't so far, creating troubles.  That's why we check
-    for "not sys.flags.debug". (http://bugs.python.org/issue28401)
+    for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
+    of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
     """
-    if 'py_limited_api' not in kwds and not sys.flags.debug:
+    if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -2,7 +2,8 @@
 # DEPRECATED: implementation for ffi.verify()
 #
 import sys, imp
-from . import model, ffiplatform
+from . import model
+from .error import VerificationError
 
 
 class VCPythonEngine(object):
@@ -155,7 +156,7 @@
                                           self.verifier.modulefilename)
             except ImportError as e:
                 error = "importing %r: %s" % (self.verifier.modulefilename, e)
-                raise ffiplatform.VerificationError(error)
+                raise VerificationError(error)
             finally:
                 if hasattr(sys, "setdlopenflags"):
                     sys.setdlopenflags(previous_flags)
@@ -185,7 +186,7 @@
             def __dir__(self):
                 return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
-        if module._cffi_setup(lst, ffiplatform.VerificationError, library):
+        if module._cffi_setup(lst, VerificationError, library):
             import warnings
             warnings.warn("reimporting %r might overwrite older definitions"
                           % (self.verifier.get_module_name()))
@@ -212,7 +213,7 @@
                 method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in verify(): %r" % name)
             try:
                 method(tp, realname)
@@ -485,7 +486,7 @@
                     prnt('  { %s = &p->%s; (void)tmp; }' % (
                         ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                         fname))
-                except ffiplatform.VerificationError as e:
+                except VerificationError as e:
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         prnt('static PyObject *')
@@ -550,7 +551,7 @@
             # check that the layout sizes and offsets match the real ones
             def check(realvalue, expectedvalue, msg):
                 if realvalue != expectedvalue:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "%s (we have %d, but C compiler says %d)"
                         % (msg, expectedvalue, realvalue))
             ffi = self.ffi
@@ -771,7 +772,7 @@
                 BItemType = self.ffi._get_cached_btype(tp.item)
                 length, rest = divmod(size, self.ffi.sizeof(BItemType))
                 if rest != 0:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "bad size: %r does not seem to be an array of %s" %
                         (name, tp.item))
                 tp = tp.resolve_length(length)
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -4,7 +4,8 @@
 import sys, os
 import types
 
-from . import model, ffiplatform
+from . import model
+from .error import VerificationError
 
 
 class VGenericEngine(object):
@@ -102,7 +103,7 @@
                 method = getattr(self, '_generate_gen_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in verify(): %r" % name)
             try:
                 method(tp, realname)
@@ -281,7 +282,7 @@
                     prnt('  { %s = &p->%s; (void)tmp; }' % (
                         ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                         fname))
-                except ffiplatform.VerificationError as e:
+                except VerificationError as e:
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         self.export_symbols.append(layoutfuncname)
@@ -344,7 +345,7 @@
             # check that the layout sizes and offsets match the real ones
             def check(realvalue, expectedvalue, msg):
                 if realvalue != expectedvalue:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "%s (we have %d, but C compiler says %d)"
                         % (msg, expectedvalue, realvalue))
             ffi = self.ffi
@@ -498,7 +499,7 @@
             error = self.ffi.string(p)
             if sys.version_info >= (3,):
                 error = str(error, 'utf-8')
-            raise ffiplatform.VerificationError(error)
+            raise VerificationError(error)
 
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
@@ -591,7 +592,7 @@
                 BItemType = self.ffi._get_cached_btype(tp.item)
                 length, rest = divmod(size, self.ffi.sizeof(BItemType))
                 if rest != 0:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "bad size: %r does not seem to be an array of %s" %
                         (name, tp.item))
                 tp = tp.resolve_length(length)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -4,6 +4,7 @@
 import sys, os, binascii, shutil, io
 from . import __version_verifier_modules__
 from . import ffiplatform
+from .error import VerificationError
 
 if sys.version_info >= (3, 3):
     import importlib.machinery
@@ -42,7 +43,7 @@
                  ext_package=None, tag='', force_generic_engine=False,
                  source_extension='.c', flags=None, relative_to=None, **kwds):
         if ffi._parser._uses_new_feature:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "feature not supported with ffi.verify(), but only "
                 "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,))
         self.ffi = ffi
@@ -83,7 +84,7 @@
         which can be tweaked beforehand."""
         with self.ffi._lock:
             if self._has_source and file is None:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "source code already written")
             self._write_source(file)
 
@@ -92,7 +93,7 @@
         This produces a dynamic link library in 'self.modulefilename'."""
         with self.ffi._lock:
             if self._has_module:
-                raise ffiplatform.VerificationError("module already compiled")
+                raise VerificationError("module already compiled")
             if not self._has_source:
                 self._write_source()
             self._compile_module()
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.9.2"
+VERSION = "1.10.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,9 +1,9 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.9.2", ("This test_c.py file is for testing a version"
-                                " of cffi that differs from the one that we"
-                                " get from 'import _cffi_backend'")
+assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+                                 " of cffi that differs from the one that we"
+                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
     type_or_class = "type"
     mandatory_b_prefix = ''
@@ -3733,7 +3733,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
@@ -29,7 +29,7 @@
     content = open(p).read()
     #
     v = cffi.__version__
-    assert ("version = '%s'\n" % v[:3]) in content
+    assert ("version = '%s'\n" % v[:4]) in content
     assert ("release = '%s'\n" % v) in content
 
 def test_doc_version_file():
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -157,20 +157,21 @@
         except ImportError as e:
             py.test.skip(str(e))
         orig_version = setuptools.__version__
+        expecting_limited_api = not hasattr(sys, 'gettotalrefcount')
         try:
             setuptools.__version__ = '26.0.0'
             from setuptools import Extension
 
             kwds = _set_py_limited_api(Extension, {})
-            assert kwds['py_limited_api'] == True
+            assert kwds.get('py_limited_api', False) == expecting_limited_api
 
             setuptools.__version__ = '25.0'
             kwds = _set_py_limited_api(Extension, {})
-            assert not kwds
+            assert kwds.get('py_limited_api', False) == False
 
             setuptools.__version__ = 'development'
             kwds = _set_py_limited_api(Extension, {})
-            assert kwds['py_limited_api'] == True
+            assert kwds.get('py_limited_api', False) == expecting_limited_api
 
         finally:
             setuptools.__version__ = orig_version
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -7,6 +7,11 @@
 from pypy.module.test_lib_pypy.cffi_tests.support import u, long
 from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture
 
+try:
+    import importlib
+except ImportError:
+    importlib = None
+
 
 def check_type_table(input, expected_output, included=None):
     ffi = FFI()
@@ -27,6 +32,7 @@
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
     else:
+        # add '-Werror' to the existing 'extra_compile_args' flags
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
                                       ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
@@ -522,6 +528,8 @@
         assert os.path.exists(str(package_dir.join('mymod.c')))
         package_dir.join('__init__.py').write('')
         #
+        getattr(importlib, 'invalidate_caches', object)()
+        #
         sys.path.insert(0, str(udir))
         import test_module_name_in_package.mymod
         assert test_module_name_in_package.mymod.lib.foo(10) == 42
@@ -2160,7 +2168,7 @@
             return s;
         }
     """)
-    assert lib.f().y == chr(40)
+    assert ord(lib.f().y) == 40
     assert lib.f().x == 200
     e = py.test.raises(NotImplementedError, lib.g, 0)
     assert str(e.value) == (
@@ -2169,3 +2177,15 @@
         "  Such structs are only supported as return value if the function is "
         "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()"
         "+ffibuilder.set_source() and not taking a final '...' argument)")
+
+def test_gcc_visibility_hidden():
+    if sys.platform == 'win32':
+        py.test.skip("test for gcc/clang")
+    ffi = FFI()
+    ffi.cdef("""
+    int f(int);
+    """)
+    lib = verify(ffi, "test_gcc_visibility_hidden", """
+    int f(int a) { return a + 40; }
+    """, extra_compile_args=['-fvisibility=hidden'])
+    assert lib.f(2) == 42

From pypy.commits at gmail.com  Tue Jan 24 07:32:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 04:32:59 -0800 (PST)
Subject: [pypy-commit] pypy default: cffi issue300: return _Bool as Python
 booleans; related fixes
Message-ID: <5887497b.d185df0a.d2d2c.732e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89731:b07132257e83
Date: 2017-01-24 13:32 +0100
http://bitbucket.org/pypy/pypy/changeset/b07132257e83/

Log:	cffi issue300: return _Bool as Python booleans; related fixes

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -370,6 +370,19 @@
         # bypass the method 'string' implemented in W_CTypePrimitive
         return W_CType.string(self, cdataobj, maxlen)
 
+    def convert_to_object(self, cdata):
+        space = self.space
+        value = ord(cdata[0])
+        if value < 2:
+            return space.newbool(value != 0)
+        else:
+            raise oefmt(space.w_ValueError,
+                        "got a _Bool of value %d, expected 0 or 1",
+                        value)
+
+    def unpack_list_of_int_items(self, ptr, length):
+        return None
+
 
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -82,6 +82,8 @@
                 raise oefmt(space.w_IndexError,
                             "initializer string is too long for '%s' (got %d "
                             "characters)", self.name, n)
+            if isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool):
+                self._must_be_string_of_zero_or_one(s)
             copy_string_to_raw(llstr(s), cdata, 0, n)
             if n != self.length:
                 cdata[n] = '\x00'
@@ -101,9 +103,16 @@
         else:
             raise self._convert_error("list or tuple", w_ob)
 
+    def _must_be_string_of_zero_or_one(self, s):
+        for c in s:
+            if ord(c) > 1:
+                raise oefmt(self.space.w_ValueError,
+                            "an array of _Bool can only contain \\x00 or \\x01")
+
     def string(self, cdataobj, maxlen):
         space = self.space
-        if isinstance(self.ctitem, ctypeprim.W_CTypePrimitive):
+        if (isinstance(self.ctitem, ctypeprim.W_CTypePrimitive) and
+               not isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool)):
             with cdataobj as ptr:
                 if not ptr:
                     raise oefmt(space.w_RuntimeError,
@@ -283,6 +292,8 @@
         if self.accept_str and space.isinstance_w(w_init, space.w_str):
             # special case to optimize strings passed to a "char *" argument
             value = w_init.str_w(space)
+            if isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool):
+                self._must_be_string_of_zero_or_one(value)
             keepalives[i] = value
             buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
             rffi.cast(rffi.CCHARPP, cdata)[0] = buf
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -885,6 +885,15 @@
     py.test.raises(OverflowError, f, 128, 0)
     py.test.raises(OverflowError, f, 0, 128)
 
+def test_call_function_0_pretend_bool_result():
+    BSignedChar = new_primitive_type("signed char")
+    BBool = new_primitive_type("_Bool")
+    BFunc0 = new_function_type((BSignedChar, BSignedChar), BBool, False)
+    f = cast(BFunc0, _testfunc(0))
+    assert f(40, -39) is True
+    assert f(40, -40) is False
+    py.test.raises(ValueError, f, 40, 2)
+
 def test_call_function_1():
     BInt = new_primitive_type("int")
     BLong = new_primitive_type("long")
@@ -1047,6 +1056,17 @@
     res = f(b"foo")
     assert res == 1000 * ord(b'f')
 
+def test_call_function_23_bool_array():
+    # declaring the function as int(_Bool*)
+    BBool = new_primitive_type("_Bool")
+    BBoolP = new_pointer_type(BBool)
+    BInt = new_primitive_type("int")
+    BFunc23 = new_function_type((BBoolP,), BInt, False)
+    f = cast(BFunc23, _testfunc(23))
+    res = f(b"\x01\x01")
+    assert res == 1000
+    py.test.raises(ValueError, f, b"\x02\x02")
+
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
@@ -2617,13 +2637,38 @@
     py.test.raises(OverflowError, newp, BBoolP, 2)
     py.test.raises(OverflowError, newp, BBoolP, -1)
     BCharP = new_pointer_type(new_primitive_type("char"))
-    p = newp(BCharP, b'X')
+    p = newp(BCharP, b'\x01')
     q = cast(BBoolP, p)
-    assert q[0] == ord(b'X')
+    assert q[0] is True
+    p = newp(BCharP, b'\x00')
+    q = cast(BBoolP, p)
+    assert q[0] is False
     py.test.raises(TypeError, string, cast(BBool, False))
     BDouble = new_primitive_type("double")
     assert int(cast(BBool, cast(BDouble, 0.1))) == 1
     assert int(cast(BBool, cast(BDouble, 0.0))) == 0
+    BBoolA = new_array_type(BBoolP, None)
+    p = newp(BBoolA, b'\x01\x00')
+    assert p[0] is True
+    assert p[1] is False
+
+def test_bool_forbidden_cases():
+    BBool = new_primitive_type("_Bool")
+    BBoolP = new_pointer_type(BBool)
+    BBoolA = new_array_type(BBoolP, None)
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    p = newp(BCharP, b'X')
+    q = cast(BBoolP, p)
+    py.test.raises(ValueError, "q[0]")
+    py.test.raises(TypeError, newp, BBoolP, b'\x00')
+    assert newp(BBoolP, 0)[0] is False
+    assert newp(BBoolP, 1)[0] is True
+    py.test.raises(OverflowError, newp, BBoolP, 2)
+    py.test.raises(OverflowError, newp, BBoolP, -1)
+    py.test.raises(ValueError, newp, BBoolA, b'\x00\x01\x02')
+    py.test.raises(OverflowError, newp, BBoolA, [0, 1, 2])
+    py.test.raises(TypeError, string, newp(BBoolP, 1))
+    py.test.raises(TypeError, string, newp(BBoolA, [1]))
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
@@ -3663,7 +3708,7 @@
             ("int16_t", [-2**15, 2**15-1]),
             ("int32_t", [-2**31, 2**31-1]),
             ("int64_t", [-2**63, 2**63-1]),
-            ("_Bool", [0, 1]),
+            ("_Bool", [False, True]),
             ("float", [0.0, 10.5]),
             ("double", [12.34, 56.78]),
             ]:
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -109,9 +109,8 @@
         P_BOOL = _cffi_backend.new_pointer_type(BOOL)
         BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
         buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
-        assert buf[0] == 1
-        assert buf[1] == 0
-        assert type(buf[1]) is int
+        assert buf[0] is True
+        assert buf[1] is False
         raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
         raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
 

From pypy.commits at gmail.com  Tue Jan 24 07:57:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 04:57:01 -0800 (PST)
Subject: [pypy-commit] cffi default: reintroduce a fast case for
 ffi.unpack(arrays of _Bool)
Message-ID: <58874f1d.828bdf0a.2e8d7.d9e3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2872:19591d33fdb2
Date: 2017-01-24 13:56 +0100
http://bitbucket.org/cffi/cffi/changeset/19591d33fdb2/

Log:	reintroduce a fast case for ffi.unpack(arrays of _Bool)

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5944,12 +5944,12 @@
             else if (itemsize == sizeof(short))       casenum = 1;
             else if (itemsize == sizeof(signed char)) casenum = 0;
         }
-        else if ((ctitem->ct_flags & (CT_PRIMITIVE_UNSIGNED | CT_IS_BOOL))
-                 == CT_PRIMITIVE_UNSIGNED) {
+        else if (ctitem->ct_flags & CT_PRIMITIVE_UNSIGNED) {
             /* Note: we never pick case 6 if sizeof(int) == sizeof(long),
                so that case 6 below can assume that the 'unsigned int' result
                would always fit in a 'signed long'. */
-            if      (itemsize == sizeof(unsigned long))  casenum = 7;
+            if (ctitem->ct_flags & CT_IS_BOOL)           casenum = 11;
+            else if (itemsize == sizeof(unsigned long))  casenum = 7;
             else if (itemsize == sizeof(unsigned int))   casenum = 6;
             else if (itemsize == sizeof(unsigned short)) casenum = 5;
             else if (itemsize == sizeof(unsigned char))  casenum = 4;
@@ -5982,6 +5982,13 @@
         case 8: x = PyFloat_FromDouble(*(float *)src); break;
         case 9: x = PyFloat_FromDouble(*(double *)src); break;
         case 10: x = new_simple_cdata(*(char **)src, ctitem); break;
+        case 11:
+            switch (*(unsigned char *)src) {
+            case 0: x = Py_False; Py_INCREF(x); break;
+            case 1: x = Py_True;  Py_INCREF(x); break;
+            default: x = convert_to_object(src, ctitem); /* error */
+            }
+            break;
         }
         if (x == NULL) {
             Py_DECREF(result);

From pypy.commits at gmail.com  Tue Jan 24 09:03:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 24 Jan 2017 06:03:30 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix tests
Message-ID: <58875eb2.810b1c0a.681cb.ed03@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89732:321e73239593
Date: 2017-01-24 14:02 +0000
http://bitbucket.org/pypy/pypy/changeset/321e73239593/

Log:	fix tests

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -13,8 +13,8 @@
         w_hello = space.newbytes("hello")
         #assert api.PyObject_CheckBuffer(w_hello)
         w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
-        w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
-        assert space.eq_w(w_char, space.wrap('h'))
+        w_byte = space.call_method(w_view, '__getitem__', space.wrap(0))
+        assert space.eq_w(w_byte, space.wrap(ord('h')))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
@@ -72,7 +72,6 @@
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol_app(self):
-        import struct
         module = self.import_module(name='buffer_test')
         arr = module.PyMyArray(10)
         y = memoryview(arr)
@@ -80,8 +79,7 @@
         assert y.shape == (10,)
         assert len(y) == 10
         s = y[3]
-        assert len(s) == struct.calcsize('i')
-        assert s == struct.pack('i', 3)
+        assert s == 3
 
     def test_buffer_protocol_capi(self):
         foo = self.import_extension('foo', [

From pypy.commits at gmail.com  Tue Jan 24 09:17:56 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 24 Jan 2017 06:17:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix type creation in test
Message-ID: <58876214.91abdf0a.eeb38.a201@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89733:0e5a44ac25a3
Date: 2017-01-24 14:11 +0000
http://bitbucket.org/pypy/pypy/changeset/0e5a44ac25a3/

Log:	Fix type creation in test

diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -42,7 +42,7 @@
         w_year = space.getattr(w_obj, space.newbytes('year'))
         assert space.int_w(w_year) == 1
 
-        w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype, 
+        w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype,
                                  arg, space.newdict({}))
         w_year = space.getattr(w_obj, space.newbytes('year'))
         assert space.int_w(w_year) == 1
@@ -137,25 +137,23 @@
                     return datetime_cls->tp_new(t, a, k);
                 }
 
-                static void 
+                static void
                 _timestamp_dealloc(PyObject *op)
                 {
                     foocnt --;
                     datetime_cls->tp_dealloc(op);
                 }
-                 
+
 
                 static PyTypeObject _Timestamp = {
-                    PyObject_HEAD_INIT(NULL)
-                    0,                            /* ob_size */
+                    PyVarObject_HEAD_INIT(NULL, 0)
                     "foo._Timestamp",   /* tp_name*/
                     0,                  /* tp_basicsize*/
                     0,                  /* tp_itemsize */
                     _timestamp_dealloc  /* tp_dealloc  */
                 };
                 static PyTypeObject Timestamp = {
-                    PyObject_HEAD_INIT(NULL)
-                    0,                            /* ob_size */
+                    PyVarObject_HEAD_INIT(NULL, 0)
                     "foo.Timestamp",   /* tp_name*/
                     0,                  /* tp_basicsize*/
                     0                  /* tp_itemsize */
@@ -164,7 +162,7 @@
                 PyObject * mod1 = PyImport_ImportModule("datetime");
                 if (mod1 == NULL) INITERROR;
                 PyObject * dt = PyUnicode_FromString("datetime");
-                datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod1, dt); 
+                datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod1, dt);
                 if (datetime_cls == NULL) INITERROR;
                 _Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
                 _Timestamp.tp_base = datetime_cls;
@@ -179,7 +177,7 @@
                 Timestamp.tp_dealloc = datetime_cls->tp_dealloc;
                 if (PyType_Ready(&Timestamp) < 0) INITERROR;
             ''')
-        # _Timestamp has __new__, __del__ and 
+        # _Timestamp has __new__, __del__ and
         #      inherits from datetime.datetime
         # Timestamp has __new__, default __del__ (subtype_dealloc) and
         #      inherits from _Timestamp

From pypy.commits at gmail.com  Tue Jan 24 09:18:32 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 24 Jan 2017 06:18:32 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix type creation in test
Message-ID: <58876238.496f1c0a.87fe9.e4e1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89734:4dbc89b22496
Date: 2017-01-24 14:11 +0000
http://bitbucket.org/pypy/pypy/changeset/4dbc89b22496/

Log:	Fix type creation in test (grafted from
	0e5a44ac25a3501025d3198b7d6806672182aff0)

diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -42,7 +42,7 @@
         w_year = space.getattr(w_obj, space.newbytes('year'))
         assert space.int_w(w_year) == 1
 
-        w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype, 
+        w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype,
                                  arg, space.newdict({}))
         w_year = space.getattr(w_obj, space.newbytes('year'))
         assert space.int_w(w_year) == 1
@@ -137,25 +137,23 @@
                     return datetime_cls->tp_new(t, a, k);
                 }
 
-                static void 
+                static void
                 _timestamp_dealloc(PyObject *op)
                 {
                     foocnt --;
                     datetime_cls->tp_dealloc(op);
                 }
-                 
+
 
                 static PyTypeObject _Timestamp = {
-                    PyObject_HEAD_INIT(NULL)
-                    0,                            /* ob_size */
+                    PyVarObject_HEAD_INIT(NULL, 0)
                     "foo._Timestamp",   /* tp_name*/
                     0,                  /* tp_basicsize*/
                     0,                  /* tp_itemsize */
                     _timestamp_dealloc  /* tp_dealloc  */
                 };
                 static PyTypeObject Timestamp = {
-                    PyObject_HEAD_INIT(NULL)
-                    0,                            /* ob_size */
+                    PyVarObject_HEAD_INIT(NULL, 0)
                     "foo.Timestamp",   /* tp_name*/
                     0,                  /* tp_basicsize*/
                     0                  /* tp_itemsize */
@@ -179,7 +177,7 @@
                 Timestamp.tp_dealloc = datetime_cls->tp_dealloc;
                 if (PyType_Ready(&Timestamp) < 0) INITERROR;
             ''')
-        # _Timestamp has __new__, __del__ and 
+        # _Timestamp has __new__, __del__ and
         #      inherits from datetime.datetime
         # Timestamp has __new__, default __del__ (subtype_dealloc) and
         #      inherits from _Timestamp

From pypy.commits at gmail.com  Tue Jan 24 09:21:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 06:21:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: test and fix
Message-ID: <588762d8.9d711c0a.3f964.dfe3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89735:9069e5841365
Date: 2017-01-24 15:20 +0100
http://bitbucket.org/pypy/pypy/changeset/9069e5841365/

Log:	test and fix

diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -260,8 +260,12 @@
     fstr.current_index = i
     literal = builder.build()
     if not fstr.raw_mode and u'\\' in literal:
+        # xxx messy
+        space = astbuilder.space
         literal = literal.encode('utf-8')
-        literal = unicodehelper.decode_unicode_escape(astbuilder.space, literal)
+        literal = parsestring.decode_unicode_utf8(space, literal, 0,
+                                                  len(literal))
+        literal = unicodehelper.decode_unicode_escape(space, literal)
     return literal
 
 
@@ -328,22 +332,26 @@
     space = astbuilder.space
     encoding = astbuilder.compile_info.encoding
     joined_pieces = []
-    for i in range(atom_node.num_children()):
-        try:
+    try:
+        for i in range(atom_node.num_children()):
             w_next = parsestring.parsestr(
                     space, encoding, atom_node.get_child(i).get_value())
-        except error.OperationError as e:
-            if not (e.match(space, space.w_UnicodeError) or
-                    e.match(space, space.w_ValueError)):
-                raise
-            # Unicode/ValueError in literal: turn into SyntaxError
-            raise astbuilder.error(e.errorstr(space), atom_node)
-        if not isinstance(w_next, parsestring.W_FString):
-            add_constant_string(astbuilder, joined_pieces, w_next, atom_node)
-        else:
-            parse_f_string(astbuilder, joined_pieces, w_next, atom_node)
+            if not isinstance(w_next, parsestring.W_FString):
+                add_constant_string(astbuilder, joined_pieces, w_next,
+                                    atom_node)
+            else:
+                parse_f_string(astbuilder, joined_pieces, w_next, atom_node)
+
+    except error.OperationError as e:
+        if not (e.match(space, space.w_UnicodeError) or
+                e.match(space, space.w_ValueError)):
+            raise
+        # Unicode/ValueError in literal: turn into SyntaxError
+        raise astbuilder.error(e.errorstr(space), atom_node)
+
     if len(joined_pieces) == 1:   # <= the common path
         return joined_pieces[0]   # ast.Str, Bytes or FormattedValue
+
     # with more than one piece, it is a combination of Str and
     # FormattedValue pieces---if there is a Bytes, then we got
     # an invalid mixture of bytes and unicode literals
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1192,6 +1192,7 @@
         raises(SyntaxError, self.run, "f'{5#}'")
         raises(SyntaxError, self.run, "f'{5)#}'")
         raises(SyntaxError, self.run, "f'''{5)\n#}'''")
+        raises(SyntaxError, self.run, "f'\\x'")
 
 
 class AppTestCompiler:

From pypy.commits at gmail.com  Tue Jan 24 09:41:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 06:41:59 -0800 (PST)
Subject: [pypy-commit] pypy default: test and fix
Message-ID: <588767b7.5190df0a.16278.a692@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89736:ad54ed3f27d2
Date: 2017-01-24 15:31 +0100
http://bitbucket.org/pypy/pypy/changeset/ad54ed3f27d2/

Log:	test and fix

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1104,8 +1104,9 @@
                 if not e.match(space, space.w_UnicodeError):
                     raise
                 # UnicodeError in literal: turn into SyntaxError
-                self.error(e.errorstr(space), atom_node)
-                sub_strings_w = [] # please annotator
+                e.normalize_exception(space)
+                errmsg = space.str_w(space.str(e.get_w_value(space)))
+                raise self.error('(unicode error) %s' % errmsg, atom_node)
             # This implements implicit string concatenation.
             if len(sub_strings_w) > 1:
                 w_sub_strings = space.newlist(sub_strings_w)
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1240,3 +1240,9 @@
         if1, if2 = comps[0].ifs
         assert isinstance(if1, ast.Name)
         assert isinstance(if2, ast.Name)
+
+    def test_decode_error_in_string_literal(self):
+        input = "u'\\x'"
+        exc = py.test.raises(SyntaxError, self.get_ast, input).value
+        assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode"
+                           " bytes in position 0-1: truncated \\xXX escape")

From pypy.commits at gmail.com  Tue Jan 24 09:42:02 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 06:42:02 -0800 (PST)
Subject: [pypy-commit] pypy default: OperationError.errorstr() gave results
 depending on whether the exception
Message-ID: <588767ba.04abdf0a.cc1f8.acb0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89737:a6f3468a583e
Date: 2017-01-24 15:40 +0100
http://bitbucket.org/pypy/pypy/changeset/a6f3468a583e/

Log:	OperationError.errorstr() gave results depending on whether the
	exception was already normalized or not. Fix

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -71,6 +71,7 @@
 
     def errorstr(self, space, use_repr=False):
         "The exception class and value, as a string."
+        self.normalize_exception(space)
         w_value = self.get_w_value(space)
         if space is None:
             # this part NOT_RPYTHON
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -81,7 +81,18 @@
 def test_errorstr(space):
     operr = OperationError(space.w_ValueError, space.wrap("message"))
     assert operr.errorstr(space) == "ValueError: message"
-    assert operr.errorstr(space, use_repr=True) == "ValueError: 'message'"
+    assert operr.errorstr(space, use_repr=True) == (
+        "ValueError: ValueError('message',)")
+    operr = OperationError(space.w_ValueError, space.w_None)
+    assert operr.errorstr(space) == "ValueError"
+    operr = OperationError(space.w_ValueError,
+        space.newtuple([space.wrap(6), space.wrap(7)]))
+    assert operr.errorstr(space) == "ValueError: (6, 7)"
+    operr = OperationError(space.w_UnicodeDecodeError,
+        space.wrap(('unicodeescape', r'\\x', 0, 2, r'truncated \\xXX escape')))
+    assert operr.errorstr(space) == (
+        "UnicodeDecodeError: 'unicodeescape' codec can't decode "
+        "bytes in position 0-1: truncated \\\\xXX escape")
 
 def test_wrap_oserror():
     class FakeSpace:

From pypy.commits at gmail.com  Tue Jan 24 09:42:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 06:42:03 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <588767bb.4d821c0a.cb48e.0bd8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89738:2a1d5f7642dc
Date: 2017-01-24 15:41 +0100
http://bitbucket.org/pypy/pypy/changeset/2a1d5f7642dc/

Log:	merge heads

diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -42,7 +42,7 @@
         w_year = space.getattr(w_obj, space.newbytes('year'))
         assert space.int_w(w_year) == 1
 
-        w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype, 
+        w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype,
                                  arg, space.newdict({}))
         w_year = space.getattr(w_obj, space.newbytes('year'))
         assert space.int_w(w_year) == 1
@@ -137,25 +137,23 @@
                     return datetime_cls->tp_new(t, a, k);
                 }
 
-                static void 
+                static void
                 _timestamp_dealloc(PyObject *op)
                 {
                     foocnt --;
                     datetime_cls->tp_dealloc(op);
                 }
-                 
+
 
                 static PyTypeObject _Timestamp = {
-                    PyObject_HEAD_INIT(NULL)
-                    0,                            /* ob_size */
+                    PyVarObject_HEAD_INIT(NULL, 0)
                     "foo._Timestamp",   /* tp_name*/
                     0,                  /* tp_basicsize*/
                     0,                  /* tp_itemsize */
                     _timestamp_dealloc  /* tp_dealloc  */
                 };
                 static PyTypeObject Timestamp = {
-                    PyObject_HEAD_INIT(NULL)
-                    0,                            /* ob_size */
+                    PyVarObject_HEAD_INIT(NULL, 0)
                     "foo.Timestamp",   /* tp_name*/
                     0,                  /* tp_basicsize*/
                     0                  /* tp_itemsize */
@@ -179,7 +177,7 @@
                 Timestamp.tp_dealloc = datetime_cls->tp_dealloc;
                 if (PyType_Ready(&Timestamp) < 0) INITERROR;
             ''')
-        # _Timestamp has __new__, __del__ and 
+        # _Timestamp has __new__, __del__ and
         #      inherits from datetime.datetime
         # Timestamp has __new__, default __del__ (subtype_dealloc) and
         #      inherits from _Timestamp

From pypy.commits at gmail.com  Tue Jan 24 09:54:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 06:54:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58876a90.068f1c0a.a87f1.f5b1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89739:a0364a1c7f67
Date: 2017-01-24 15:46 +0100
http://bitbucket.org/pypy/pypy/changeset/a0364a1c7f67/

Log:	hg merge default

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.9.2
+Version: 1.10.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -1,11 +1,11 @@
 __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError',
            'FFIError']
 
-from .api import FFI, CDefError, FFIError
-from .ffiplatform import VerificationError, VerificationMissing
+from .api import FFI
+from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.9.2"
-__version_info__ = (1, 9, 2)
+__version__ = "1.10.0"
+__version_info__ = (1, 10, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.9.2"
+                               "\ncompiled with cffi version: 1.10.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -1,5 +1,7 @@
 import sys, types
 from .lock import allocate_lock
+from .error import CDefError
+from . import model
 
 try:
     callable
@@ -15,17 +17,6 @@
     basestring = str
 
 
-class FFIError(Exception):
-    pass
-
-class CDefError(Exception):
-    def __str__(self):
-        try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
-        except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
-
 
 class FFI(object):
     r'''
@@ -49,18 +40,27 @@
         """Create an FFI instance.  The 'backend' argument is used to
         select a non-default backend, mostly for tests.
         """
-        from . import cparser, model
         if backend is None:
             # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with
             # _cffi_backend.so compiled.
             import _cffi_backend as backend
             from . import __version__
-            assert backend.__version__ == __version__, \
-               "version mismatch, %s != %s" % (backend.__version__, __version__)
+            if backend.__version__ != __version__:
+                # bad version!  Try to be as explicit as possible.
+                if hasattr(backend, '__file__'):
+                    # CPython
+                    raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r.  When we import the top-level '_cffi_backend' extension module, we get version %s, located in %r.  The two versions should be equal; check your installation." % (
+                        __version__, __file__,
+                        backend.__version__, backend.__file__))
+                else:
+                    # PyPy
+                    raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r.  This interpreter comes with a built-in '_cffi_backend' module, which is version %s.  The two versions should be equal; check your installation." % (
+                        __version__, __file__, backend.__version__))
             # (If you insist you can also try to pass the option
             # 'backend=backend_ctypes.CTypesBackend()', but don't
             # rely on it!  It's probably not going to work well.)
 
+        from . import cparser
         self._backend = backend
         self._lock = allocate_lock()
         self._parser = cparser.Parser()
@@ -212,7 +212,7 @@
 
     def offsetof(self, cdecl, *fields_or_indexes):
         """Return the offset of the named field inside the given
-        structure or array, which must be given as a C type name.  
+        structure or array, which must be given as a C type name.
         You can give several field names in case of nested structures.
         You can also give numeric values which correspond to array
         items, in case of an array type.
@@ -300,7 +300,7 @@
         return self._backend.string(cdata, maxlen)
 
     def unpack(self, cdata, length):
-        """Unpack an array of C data of the given length, 
+        """Unpack an array of C data of the given length,
         returning a Python string/unicode/list.
 
         If 'cdata' is a pointer to 'char', returns a byte string.
@@ -452,7 +452,6 @@
         return self._backend.getwinerror(code)
 
     def _pointer_to(self, ctype):
-        from . import model
         with self._lock:
             return model.pointer_cache(self, ctype)
 
@@ -764,7 +763,6 @@
         return backend.load_library(path, flags)
 
 def _make_ffi_library(ffi, libname, flags):
-    import os
     backend = ffi._backend
     backendlib = _load_backend_lib(backend, libname, flags)
     #
@@ -802,7 +800,6 @@
         if accessors_version[0] is ffi._cdef_version:
             return
         #
-        from . import model
         for key, (tp, _) in ffi._parser._declarations.items():
             if not isinstance(tp, model.EnumType):
                 tag, name = key.split(' ', 1)
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -1,3 +1,4 @@
+from .error import VerificationError
 
 class CffiOp(object):
     def __init__(self, op, arg):
@@ -19,7 +20,6 @@
                                     % (self.arg,))
             return format_four_bytes(value)
         if isinstance(self.arg, str):
-            from .ffiplatform import VerificationError
             raise VerificationError("cannot emit to Python: %r" % (self.arg,))
         return format_four_bytes((self.arg << 8) | self.op)
 
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -1,5 +1,6 @@
 import sys
-from . import api, model
+from . import model
+from .error import FFIError
 
 
 COMMON_TYPES = {}
@@ -31,11 +32,11 @@
         elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result, quals = model.PrimitiveType(cdecl), 0
         elif cdecl == 'set-unicode-needed':
-            raise api.FFIError("The Windows type %r is only available after "
-                               "you call ffi.set_unicode()" % (commontype,))
+            raise FFIError("The Windows type %r is only available after "
+                           "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == cdecl:
-                raise api.FFIError(
+                raise FFIError(
                     "Unsupported type: %r.  Please look at "
         "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
                     "and file an issue if you think this type should really "
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -1,5 +1,6 @@
-from . import api, model
+from . import model
 from .commontypes import COMMON_TYPES, resolve_common_type
+from .error import FFIError, CDefError
 try:
     from . import _pycparser as pycparser
 except ImportError:
@@ -113,7 +114,7 @@
             # grouping variant
             closing = csource.find('}', endpos)
             if closing < 0:
-                raise api.CDefError("'extern \"Python\" {': no '}' found")
+                raise CDefError("'extern \"Python\" {': no '}' found")
             if csource.find('{', endpos + 1, closing) >= 0:
                 raise NotImplementedError("cannot use { } inside a block "
                                           "'extern \"Python\" { ... }'")
@@ -123,7 +124,7 @@
             # non-grouping variant
             semicolon = csource.find(';', endpos)
             if semicolon < 0:
-                raise api.CDefError("'extern \"Python\": no ';' found")
+                raise CDefError("'extern \"Python\": no ';' found")
             parts.append(csource[endpos:semicolon+1])
             csource = csource[semicolon+1:]
         parts.append(' void __cffi_extern_python_stop;')
@@ -288,7 +289,7 @@
             msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
-        raise api.CDefError(msg)
+        raise CDefError(msg)
 
     def parse(self, csource, override=False, packed=False, dllexport=False):
         prev_options = self._options
@@ -318,8 +319,8 @@
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
                     if not decl.name:
-                        raise api.CDefError("typedef does not declare any name",
-                                            decl)
+                        raise CDefError("typedef does not declare any name",
+                                        decl)
                     quals = 0
                     if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
                             and decl.type.type.names[-1] == '__dotdotdot__'):
@@ -337,8 +338,8 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise api.CDefError("unrecognized construct", decl)
-        except api.FFIError as e:
+                    raise CDefError("unrecognized construct", decl)
+        except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
                 e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
@@ -348,7 +349,7 @@
         if key in self._int_constants:
             if self._int_constants[key] == val:
                 return     # ignore identical double declarations
-            raise api.FFIError(
+            raise FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
 
@@ -375,7 +376,7 @@
             elif value == '...':
                 self._declare('macro ' + key, value)
             else:
-                raise api.CDefError(
+                raise CDefError(
                     'only supports one of the following syntax:\n'
                     '  #define %s ...     (literally dot-dot-dot)\n'
                     '  #define %s NUMBER  (with NUMBER an integer'
@@ -410,8 +411,8 @@
             elif isinstance(node, pycparser.c_ast.Enum):
                 self._get_struct_union_enum_type('enum', node)
             elif not decl.name:
-                raise api.CDefError("construct does not declare any variable",
-                                    decl)
+                raise CDefError("construct does not declare any variable",
+                                decl)
             #
             if decl.name:
                 tp, quals = self._get_type_and_quals(node,
@@ -438,7 +439,7 @@
                     self._inside_extern_python = decl.name
                 else:
                     if self._inside_extern_python !='__cffi_extern_python_stop':
-                        raise api.CDefError(
+                        raise CDefError(
                             "cannot declare constants or "
                             "variables with 'extern \"Python\"'")
                     if (quals & model.Q_CONST) and not tp.is_array_type:
@@ -454,7 +455,7 @@
         assert not macros
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
-            raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
+            raise CDefError("unknown identifier '%s'" % (exprnode.name,))
         return self._get_type_and_quals(exprnode.type)
 
     def _declare(self, name, obj, included=False, quals=0):
@@ -463,7 +464,7 @@
             if prevobj is obj and prevquals == quals:
                 return
             if not self._options.get('override'):
-                raise api.FFIError(
+                raise FFIError(
                     "multiple declarations of %s (for interactive usage, "
                     "try cdef(xx, override=True))" % (name,))
         assert '__dotdotdot__' not in name.split()
@@ -551,7 +552,7 @@
                 if ident == 'void':
                     return model.void_type, quals
                 if ident == '__dotdotdot__':
-                    raise api.FFIError(':%d: bad usage of "..."' %
+                    raise FFIError(':%d: bad usage of "..."' %
                             typenode.coord.line)
                 tp0, quals0 = resolve_common_type(self, ident)
                 return tp0, (quals | quals0)
@@ -583,14 +584,14 @@
             return self._get_struct_union_enum_type('union', typenode, name,
                                                     nested=True), 0
         #
-        raise api.FFIError(":%d: bad or unsupported type declaration" %
+        raise FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
     def _parse_function_type(self, typenode, funcname=None):
         params = list(getattr(typenode.args, 'params', []))
         for i, arg in enumerate(params):
             if not hasattr(arg, 'type'):
-                raise api.CDefError("%s arg %d: unknown type '%s'"
+                raise CDefError("%s arg %d: unknown type '%s'"
                     " (if you meant to use the old C syntax of giving"
                     " untyped arguments, it is not supported)"
                     % (funcname or 'in expression', i + 1,
@@ -604,7 +605,7 @@
         if ellipsis:
             params.pop()
             if not params:
-                raise api.CDefError(
+                raise CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
         args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type))
@@ -705,7 +706,7 @@
             return tp
         #
         if tp.fldnames is not None:
-            raise api.CDefError("duplicate declaration of struct %s" % name)
+            raise CDefError("duplicate declaration of struct %s" % name)
         fldnames = []
         fldtypes = []
         fldbitsize = []
@@ -749,7 +750,7 @@
 
     def _make_partial(self, tp, nested):
         if not isinstance(tp, model.StructOrUnion):
-            raise api.CDefError("%s cannot be partial" % (tp,))
+            raise CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
         tp.partial = True
@@ -769,7 +770,7 @@
                     len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
                 return ord(s[-2])
             else:
-                raise api.CDefError("invalid constant %r" % (s,))
+                raise CDefError("invalid constant %r" % (s,))
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '+'):
@@ -788,12 +789,12 @@
             if partial_length_ok:
                 self._partial_length = True
                 return '...'
-            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
-                               "the actual array length in this context"
-                               % exprnode.coord.line)
+            raise FFIError(":%d: unsupported '[...]' here, cannot derive "
+                           "the actual array length in this context"
+                           % exprnode.coord.line)
         #
-        raise api.FFIError(":%d: unsupported expression: expected a "
-                           "simple numeric constant" % exprnode.coord.line)
+        raise FFIError(":%d: unsupported expression: expected a "
+                       "simple numeric constant" % exprnode.coord.line)
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
@@ -843,8 +844,8 @@
             for t in typenames[:-1]:
                 if t not in ['int', 'short', 'long', 'signed',
                              'unsigned', 'char']:
-                    raise api.FFIError(':%d: bad usage of "..."' %
-                                       decl.coord.line)
+                    raise FFIError(':%d: bad usage of "..."' %
+                                   decl.coord.line)
             result = model.UnknownIntegerType(decl.name)
 
         if self._uses_new_feature is None:
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -1,14 +1,5 @@
 import sys, os
-
-
-class VerificationError(Exception):
-    """ An error raised when verification fails
-    """
-
-class VerificationMissing(Exception):
-    """ An error raised when incomplete structures are passed into
-    cdef, but no verification has been done
-    """
+from .error import VerificationError
 
 
 LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -1,8 +1,8 @@
-import types, sys
+import types
 import weakref
 
 from .lock import allocate_lock
-
+from .error import CDefError, VerificationError, VerificationMissing
 
 # type qualifiers
 Q_CONST    = 0x01
@@ -39,7 +39,6 @@
         replace_with = qualify(quals, replace_with)
         result = result.replace('&', replace_with)
         if '$' in result:
-            from .ffiplatform import VerificationError
             raise VerificationError(
                 "cannot generate '%s' in %s: unknown type name"
                 % (self._get_c_name(), context))
@@ -223,9 +222,8 @@
     is_raw_function = True
 
     def build_backend_type(self, ffi, finishlist):
-        from . import api
-        raise api.CDefError("cannot render the type %r: it is a function "
-                            "type, not a pointer-to-function type" % (self,))
+        raise CDefError("cannot render the type %r: it is a function "
+                        "type, not a pointer-to-function type" % (self,))
 
     def as_function_pointer(self):
         return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi)
@@ -307,9 +305,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         if self.length == '...':
-            from . import api
-            raise api.CDefError("cannot render the type %r: unknown length" %
-                                (self,))
+            raise CDefError("cannot render the type %r: unknown length" %
+                            (self,))
         self.item.get_cached_btype(ffi, finishlist)   # force the item BType
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
@@ -455,13 +452,11 @@
         self.completed = 2
 
     def _verification_error(self, msg):
-        from .ffiplatform import VerificationError
         raise VerificationError(msg)
 
     def check_not_partial(self):
         if self.partial and self.fixedlayout is None:
-            from . import ffiplatform
-            raise ffiplatform.VerificationMissing(self._get_c_name())
+            raise VerificationMissing(self._get_c_name())
 
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
@@ -499,8 +494,7 @@
 
     def check_not_partial(self):
         if self.partial and not self.partial_resolved:
-            from . import ffiplatform
-            raise ffiplatform.VerificationMissing(self._get_c_name())
+            raise VerificationMissing(self._get_c_name())
 
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
@@ -514,7 +508,6 @@
         if self.baseinttype is not None:
             return self.baseinttype.get_cached_btype(ffi, finishlist)
         #
-        from . import api
         if self.enumvalues:
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
@@ -549,8 +542,8 @@
         if (smallest_value >= ((-1) << (8*size2-1)) and
             largest_value < (1 << (8*size2-sign))):
             return btype2
-        raise api.CDefError("%s values don't all fit into either 'long' "
-                            "or 'unsigned long'" % self._get_c_name())
+        raise CDefError("%s values don't all fit into either 'long' "
+                        "or 'unsigned long'" % self._get_c_name())
 
 def unknown_type(name, structname=None):
     if structname is None:
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -1,5 +1,6 @@
 import os, sys, io
 from . import ffiplatform, model
+from .error import VerificationError
 from .cffi_opcode import *
 
 VERSION = "0x2601"
@@ -211,7 +212,7 @@
                 method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in recompile(): %r" % name)
             try:
                 self._current_quals = quals
@@ -354,12 +355,12 @@
                     included_module_name, included_source = (
                         ffi_to_include._assigned_source[:2])
                 except AttributeError:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "ffi object %r includes %r, but the latter has not "
                         "been prepared with set_source()" % (
                             self.ffi, ffi_to_include,))
                 if included_source is None:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "not implemented yet: ffi.include() of a Python-based "
                         "ffi inside a C-based ffi")
                 prnt('  "%s",' % (included_module_name,))
@@ -391,6 +392,10 @@
         prnt()
         #
         # the init function
+        prnt('#ifdef __GNUC__')
+        prnt('#  pragma GCC visibility push(default)  /* for -fvisibility= */')
+        prnt('#endif')
+        prnt()
         prnt('#ifdef PYPY_VERSION')
         prnt('PyMODINIT_FUNC')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
@@ -429,6 +434,10 @@
             self.module_name, version))
         prnt('}')
         prnt('#endif')
+        prnt()
+        prnt('#ifdef __GNUC__')
+        prnt('#  pragma GCC visibility pop')
+        prnt('#endif')
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -456,12 +465,12 @@
                 included_module_name, included_source = (
                     ffi_to_include._assigned_source[:2])
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "ffi object %r includes %r, but the latter has not "
                     "been prepared with set_source()" % (
                         self.ffi, ffi_to_include,))
             if included_source is not None:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented yet: ffi.include() of a C-based "
                     "ffi inside a Python-based ffi")
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
@@ -831,7 +840,7 @@
                 prnt('  { %s = &p->%s; (void)tmp; }' % (
                     ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                     fname))
-            except ffiplatform.VerificationError as e:
+            except VerificationError as e:
                 prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname))
@@ -994,7 +1003,7 @@
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
                             check_value=None):
         if (category, name) in self._seen_constants:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "duplicate declaration of %s '%s'" % (category, name))
         self._seen_constants.add((category, name))
         #
@@ -1093,7 +1102,7 @@
     def _generate_cpy_macro_ctx(self, tp, name):
         if tp == '...':
             if self.target_is_python:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "cannot use the syntax '...' in '#define %s ...' when "
                     "using the ABI mode" % (name,))
             check_value = None
@@ -1226,7 +1235,7 @@
 
     def _generate_cpy_extern_python_ctx(self, tp, name):
         if self.target_is_python:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "cannot use 'extern \"Python\"' in the ABI mode")
         if tp.ellipsis:
             raise NotImplementedError("a vararg function is extern \"Python\"")
@@ -1307,7 +1316,7 @@
         if tp.length is None:
             self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index)
         elif tp.length == '...':
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "type %s badly placed: the '...' array length can only be "
                 "used on global arrays or on fields of structures" % (
                     str(tp).replace('/*...*/', '...'),))
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -79,9 +79,10 @@
     CPython itself should ignore the flag in a debugging version
     (by not listing .abi3.so in the extensions it supports), but
     it doesn't so far, creating troubles.  That's why we check
-    for "not sys.flags.debug". (http://bugs.python.org/issue28401)
+    for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
+    of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
     """
-    if 'py_limited_api' not in kwds and not sys.flags.debug:
+    if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -2,7 +2,8 @@
 # DEPRECATED: implementation for ffi.verify()
 #
 import sys, imp
-from . import model, ffiplatform
+from . import model
+from .error import VerificationError
 
 
 class VCPythonEngine(object):
@@ -155,7 +156,7 @@
                                           self.verifier.modulefilename)
             except ImportError as e:
                 error = "importing %r: %s" % (self.verifier.modulefilename, e)
-                raise ffiplatform.VerificationError(error)
+                raise VerificationError(error)
             finally:
                 if hasattr(sys, "setdlopenflags"):
                     sys.setdlopenflags(previous_flags)
@@ -185,7 +186,7 @@
             def __dir__(self):
                 return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
-        if module._cffi_setup(lst, ffiplatform.VerificationError, library):
+        if module._cffi_setup(lst, VerificationError, library):
             import warnings
             warnings.warn("reimporting %r might overwrite older definitions"
                           % (self.verifier.get_module_name()))
@@ -212,7 +213,7 @@
                 method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in verify(): %r" % name)
             try:
                 method(tp, realname)
@@ -485,7 +486,7 @@
                     prnt('  { %s = &p->%s; (void)tmp; }' % (
                         ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                         fname))
-                except ffiplatform.VerificationError as e:
+                except VerificationError as e:
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         prnt('static PyObject *')
@@ -550,7 +551,7 @@
             # check that the layout sizes and offsets match the real ones
             def check(realvalue, expectedvalue, msg):
                 if realvalue != expectedvalue:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "%s (we have %d, but C compiler says %d)"
                         % (msg, expectedvalue, realvalue))
             ffi = self.ffi
@@ -771,7 +772,7 @@
                 BItemType = self.ffi._get_cached_btype(tp.item)
                 length, rest = divmod(size, self.ffi.sizeof(BItemType))
                 if rest != 0:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "bad size: %r does not seem to be an array of %s" %
                         (name, tp.item))
                 tp = tp.resolve_length(length)
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -4,7 +4,8 @@
 import sys, os
 import types
 
-from . import model, ffiplatform
+from . import model
+from .error import VerificationError
 
 
 class VGenericEngine(object):
@@ -102,7 +103,7 @@
                 method = getattr(self, '_generate_gen_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in verify(): %r" % name)
             try:
                 method(tp, realname)
@@ -281,7 +282,7 @@
                     prnt('  { %s = &p->%s; (void)tmp; }' % (
                         ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                         fname))
-                except ffiplatform.VerificationError as e:
+                except VerificationError as e:
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         self.export_symbols.append(layoutfuncname)
@@ -344,7 +345,7 @@
             # check that the layout sizes and offsets match the real ones
             def check(realvalue, expectedvalue, msg):
                 if realvalue != expectedvalue:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "%s (we have %d, but C compiler says %d)"
                         % (msg, expectedvalue, realvalue))
             ffi = self.ffi
@@ -498,7 +499,7 @@
             error = self.ffi.string(p)
             if sys.version_info >= (3,):
                 error = str(error, 'utf-8')
-            raise ffiplatform.VerificationError(error)
+            raise VerificationError(error)
 
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
@@ -591,7 +592,7 @@
                 BItemType = self.ffi._get_cached_btype(tp.item)
                 length, rest = divmod(size, self.ffi.sizeof(BItemType))
                 if rest != 0:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "bad size: %r does not seem to be an array of %s" %
                         (name, tp.item))
                 tp = tp.resolve_length(length)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -4,6 +4,7 @@
 import sys, os, binascii, shutil, io
 from . import __version_verifier_modules__
 from . import ffiplatform
+from .error import VerificationError
 
 if sys.version_info >= (3, 3):
     import importlib.machinery
@@ -42,7 +43,7 @@
                  ext_package=None, tag='', force_generic_engine=False,
                  source_extension='.c', flags=None, relative_to=None, **kwds):
         if ffi._parser._uses_new_feature:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "feature not supported with ffi.verify(), but only "
                 "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,))
         self.ffi = ffi
@@ -83,7 +84,7 @@
         which can be tweaked beforehand."""
         with self.ffi._lock:
             if self._has_source and file is None:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "source code already written")
             self._write_source(file)
 
@@ -92,7 +93,7 @@
         This produces a dynamic link library in 'self.modulefilename'."""
         with self.ffi._lock:
             if self._has_module:
-                raise ffiplatform.VerificationError("module already compiled")
+                raise VerificationError("module already compiled")
             if not self._has_source:
                 self._write_source()
             self._compile_module()
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1216,12 +1216,16 @@
                             space, encoding, atom_node.get_child(i).get_value())
                         for i in range(atom_node.num_children())]
             except error.OperationError as e:
-                if not (e.match(space, space.w_UnicodeError) or
-                        e.match(space, space.w_ValueError)):
+                if e.match(space, space.w_UnicodeError):
+                    kind = 'unicode error'
+                elif e.match(space, space.w_ValueError):
+                    kind = 'value error'
+                else:
                     raise
                 # Unicode/ValueError in literal: turn into SyntaxError
-                self.error(e.errorstr(space), atom_node)
-                sub_strings_w = [] # please annotator
+                e.normalize_exception(space)
+                errmsg = space.str_w(space.str(e.get_w_value(space)))
+                raise self.error('(%s) %s' % (kind, errmsg), atom_node)
             # Implement implicit string concatenation.
             w_string = sub_strings_w[0]
             for i in range(1, len(sub_strings_w)):
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1398,3 +1398,9 @@
         assert len(asyncwith.body) == 1
         assert isinstance(asyncwith.body[0], ast.Expr)
         assert isinstance(asyncwith.body[0].value, ast.Num)
+
+    def test_decode_error_in_string_literal(self):
+        input = "u'\\x'"
+        exc = py.test.raises(SyntaxError, self.get_ast, input).value
+        assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode"
+                           " bytes in position 0-1: truncated \\xXX escape")
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -80,6 +80,7 @@
 
     def errorstr(self, space, use_repr=False):
         "The exception class and value, as a string."
+        self.normalize_exception(space)
         w_value = self.get_w_value(space)
         if space is None:
             # this part NOT_RPYTHON
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -96,7 +96,18 @@
 def test_errorstr(space):
     operr = OperationError(space.w_ValueError, space.wrap("message"))
     assert operr.errorstr(space) == "ValueError: message"
-    assert operr.errorstr(space, use_repr=True) == "ValueError: 'message'"
+    assert operr.errorstr(space, use_repr=True) == (
+        "ValueError: ValueError('message',)")
+    operr = OperationError(space.w_ValueError, space.w_None)
+    assert operr.errorstr(space) == "ValueError"
+    operr = OperationError(space.w_ValueError,
+        space.newtuple([space.wrap(6), space.wrap(7)]))
+    assert operr.errorstr(space) == "ValueError: (6, 7)"
+    operr = OperationError(space.w_UnicodeDecodeError,
+        space.wrap(('unicodeescape', r'\\x', 0, 2, r'truncated \\xXX escape')))
+    assert operr.errorstr(space) == (
+        "UnicodeDecodeError: 'unicodeescape' codec can't decode "
+        "bytes in position 0-1: truncated \\\\xXX escape")
 
 def test_wrap_oserror():
     class FakeSpace:
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.9.2"
+VERSION = "1.10.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -370,6 +370,19 @@
         # bypass the method 'string' implemented in W_CTypePrimitive
         return W_CType.string(self, cdataobj, maxlen)
 
+    def convert_to_object(self, cdata):
+        space = self.space
+        value = ord(cdata[0])
+        if value < 2:
+            return space.newbool(value != 0)
+        else:
+            raise oefmt(space.w_ValueError,
+                        "got a _Bool of value %d, expected 0 or 1",
+                        value)
+
+    def unpack_list_of_int_items(self, ptr, length):
+        return None
+
 
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -84,6 +84,8 @@
                 raise oefmt(space.w_IndexError,
                             "initializer string is too long for '%s' (got %d "
                             "characters)", self.name, n)
+            if isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool):
+                self._must_be_string_of_zero_or_one(s)
             copy_string_to_raw(llstr(s), cdata, 0, n)
             if n != self.length:
                 cdata[n] = '\x00'
@@ -103,9 +105,16 @@
         else:
             raise self._convert_error("list or tuple", w_ob)
 
+    def _must_be_string_of_zero_or_one(self, s):
+        for c in s:
+            if ord(c) > 1:
+                raise oefmt(self.space.w_ValueError,
+                            "an array of _Bool can only contain \\x00 or \\x01")
+
     def string(self, cdataobj, maxlen):
         space = self.space
-        if isinstance(self.ctitem, ctypeprim.W_CTypePrimitive):
+        if (isinstance(self.ctitem, ctypeprim.W_CTypePrimitive) and
+               not isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool)):
             with cdataobj as ptr:
                 if not ptr:
                     raise oefmt(space.w_RuntimeError,
@@ -285,6 +294,8 @@
         if self.accept_str and space.isinstance_w(w_init, space.w_str):
             # special case to optimize strings passed to a "char *" argument
             value = space.bytes_w(w_init)
+            if isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool):
+                self._must_be_string_of_zero_or_one(value)
             keepalives[i] = value
             buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
             rffi.cast(rffi.CCHARPP, cdata)[0] = buf
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,9 +1,9 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.9.2", ("This test_c.py file is for testing a version"
-                                " of cffi that differs from the one that we"
-                                " get from 'import _cffi_backend'")
+assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+                                 " of cffi that differs from the one that we"
+                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
     type_or_class = "type"
     mandatory_b_prefix = ''
@@ -885,6 +885,15 @@
     py.test.raises(OverflowError, f, 128, 0)
     py.test.raises(OverflowError, f, 0, 128)
 
+def test_call_function_0_pretend_bool_result():
+    BSignedChar = new_primitive_type("signed char")
+    BBool = new_primitive_type("_Bool")
+    BFunc0 = new_function_type((BSignedChar, BSignedChar), BBool, False)
+    f = cast(BFunc0, _testfunc(0))
+    assert f(40, -39) is True
+    assert f(40, -40) is False
+    py.test.raises(ValueError, f, 40, 2)
+
 def test_call_function_1():
     BInt = new_primitive_type("int")
     BLong = new_primitive_type("long")
@@ -1047,6 +1056,17 @@
     res = f(b"foo")
     assert res == 1000 * ord(b'f')
 
+def test_call_function_23_bool_array():
+    # declaring the function as int(_Bool*)
+    BBool = new_primitive_type("_Bool")
+    BBoolP = new_pointer_type(BBool)
+    BInt = new_primitive_type("int")
+    BFunc23 = new_function_type((BBoolP,), BInt, False)
+    f = cast(BFunc23, _testfunc(23))
+    res = f(b"\x01\x01")
+    assert res == 1000
+    py.test.raises(ValueError, f, b"\x02\x02")
+
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
@@ -2617,13 +2637,38 @@
     py.test.raises(OverflowError, newp, BBoolP, 2)
     py.test.raises(OverflowError, newp, BBoolP, -1)
     BCharP = new_pointer_type(new_primitive_type("char"))
-    p = newp(BCharP, b'X')
+    p = newp(BCharP, b'\x01')
     q = cast(BBoolP, p)
-    assert q[0] == ord(b'X')
+    assert q[0] is True
+    p = newp(BCharP, b'\x00')
+    q = cast(BBoolP, p)
+    assert q[0] is False
     py.test.raises(TypeError, string, cast(BBool, False))
     BDouble = new_primitive_type("double")
     assert int(cast(BBool, cast(BDouble, 0.1))) == 1
     assert int(cast(BBool, cast(BDouble, 0.0))) == 0
+    BBoolA = new_array_type(BBoolP, None)
+    p = newp(BBoolA, b'\x01\x00')
+    assert p[0] is True
+    assert p[1] is False
+
+def test_bool_forbidden_cases():
+    BBool = new_primitive_type("_Bool")
+    BBoolP = new_pointer_type(BBool)
+    BBoolA = new_array_type(BBoolP, None)
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    p = newp(BCharP, b'X')
+    q = cast(BBoolP, p)
+    py.test.raises(ValueError, "q[0]")
+    py.test.raises(TypeError, newp, BBoolP, b'\x00')
+    assert newp(BBoolP, 0)[0] is False
+    assert newp(BBoolP, 1)[0] is True
+    py.test.raises(OverflowError, newp, BBoolP, 2)
+    py.test.raises(OverflowError, newp, BBoolP, -1)
+    py.test.raises(ValueError, newp, BBoolA, b'\x00\x01\x02')
+    py.test.raises(OverflowError, newp, BBoolA, [0, 1, 2])
+    py.test.raises(TypeError, string, newp(BBoolP, 1))
+    py.test.raises(TypeError, string, newp(BBoolA, [1]))
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
@@ -3663,7 +3708,7 @@
             ("int16_t", [-2**15, 2**15-1]),
             ("int32_t", [-2**31, 2**31-1]),
             ("int64_t", [-2**63, 2**63-1]),
-            ("_Bool", [0, 1]),
+            ("_Bool", [False, True]),
             ("float", [0.0, 10.5]),
             ("double", [12.34, 56.78]),
             ]:
@@ -3733,7 +3778,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -109,9 +109,8 @@
         P_BOOL = _cffi_backend.new_pointer_type(BOOL)
         BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
         buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
-        assert buf[0] == 1
-        assert buf[1] == 0
-        assert type(buf[1]) is int
+        assert buf[0] is True
+        assert buf[1] is False
         raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
         raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
@@ -29,7 +29,7 @@
     content = open(p).read()
     #
     v = cffi.__version__
-    assert ("version = '%s'\n" % v[:3]) in content
+    assert ("version = '%s'\n" % v[:4]) in content
     assert ("release = '%s'\n" % v) in content
 
 def test_doc_version_file():
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -157,20 +157,21 @@
         except ImportError as e:
             py.test.skip(str(e))
         orig_version = setuptools.__version__
+        expecting_limited_api = not hasattr(sys, 'gettotalrefcount')
         try:
             setuptools.__version__ = '26.0.0'
             from setuptools import Extension
 
             kwds = _set_py_limited_api(Extension, {})
-            assert kwds['py_limited_api'] == True
+            assert kwds.get('py_limited_api', False) == expecting_limited_api
 
             setuptools.__version__ = '25.0'
             kwds = _set_py_limited_api(Extension, {})
-            assert not kwds
+            assert kwds.get('py_limited_api', False) == False
 
             setuptools.__version__ = 'development'
             kwds = _set_py_limited_api(Extension, {})
-            assert kwds['py_limited_api'] == True
+            assert kwds.get('py_limited_api', False) == expecting_limited_api
 
         finally:
             setuptools.__version__ = orig_version
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -7,6 +7,11 @@
 from pypy.module.test_lib_pypy.cffi_tests.support import u, long
 from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture
 
+try:
+    import importlib
+except ImportError:
+    importlib = None
+
 
 def check_type_table(input, expected_output, included=None):
     ffi = FFI()
@@ -27,6 +32,7 @@
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
     else:
+        # add '-Werror' to the existing 'extra_compile_args' flags
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
                                       ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
@@ -522,6 +528,8 @@
         assert os.path.exists(str(package_dir.join('mymod.c')))
         package_dir.join('__init__.py').write('')
         #
+        getattr(importlib, 'invalidate_caches', object)()
+        #
         sys.path.insert(0, str(udir))
         import test_module_name_in_package.mymod
         assert test_module_name_in_package.mymod.lib.foo(10) == 42
@@ -2160,7 +2168,7 @@
             return s;
         }
     """)
-    assert lib.f().y == chr(40)
+    assert ord(lib.f().y) == 40
     assert lib.f().x == 200
     e = py.test.raises(NotImplementedError, lib.g, 0)
     assert str(e.value) == (
@@ -2169,3 +2177,15 @@
         "  Such structs are only supported as return value if the function is "
         "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()"
         "+ffibuilder.set_source() and not taking a final '...' argument)")
+
+def test_gcc_visibility_hidden():
+    if sys.platform == 'win32':
+        py.test.skip("test for gcc/clang")
+    ffi = FFI()
+    ffi.cdef("""
+    int f(int);
+    """)
+    lib = verify(ffi, "test_gcc_visibility_hidden", """
+    int f(int a) { return a + 40; }
+    """, extra_compile_args=['-fvisibility=hidden'])
+    assert lib.f(2) == 42
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -259,7 +259,7 @@
                 pass
             except RZlibError as e:
                 # in this case, CPython raises the direct exception coming
-                # from the zlib module: let's to the same
+                # from the zlib module: let's do the same
                 raise zlib_error(space, e.msg)
             else:
                 if is_package:
@@ -292,7 +292,7 @@
             raise oefmt(space.w_IOError, "Error reading file")
         except RZlibError as e:
             # in this case, CPython raises the direct exception coming
-            # from the zlib module: let's to the same
+            # from the zlib module: let's do the same
             raise zlib_error(space, e.msg)
 
     def get_code(self, space, w_fullname):
@@ -428,7 +428,7 @@
                     space.wrap_fsdecoded(filename))
     except RZlibError as e:
         # in this case, CPython raises the direct exception coming
-        # from the zlib module: let's to the same
+        # from the zlib module: let's do the same
         raise zlib_error(space, e.msg)
 
     prefix = name[len(filename):]

From pypy.commits at gmail.com  Tue Jan 24 09:54:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 06:54:10 -0800 (PST)
Subject: [pypy-commit] pypy default: forgot to check in this file
Message-ID: <58876a92.85e11c0a.1b84b.f16e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89740:e80e3f733c30
Date: 2017-01-24 15:49 +0100
http://bitbucket.org/pypy/pypy/changeset/e80e3f733c30/

Log:	forgot to check in this file

diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/cffi/error.py
@@ -0,0 +1,20 @@
+
+class FFIError(Exception):
+    pass
+
+class CDefError(Exception):
+    def __str__(self):
+        try:
+            line = 'line %d: ' % (self.args[1].coord.line,)
+        except (AttributeError, TypeError, IndexError):
+            line = ''
+        return '%s%s' % (line, self.args[0])
+
+class VerificationError(Exception):
+    """ An error raised when verification fails
+    """
+
+class VerificationMissing(Exception):
+    """ An error raised when incomplete structures are passed into
+    cdef, but no verification has been done
+    """

From pypy.commits at gmail.com  Tue Jan 24 09:54:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 06:54:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58876a94.2d88df0a.f10f0.a76b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89741:40f6115fdf6d
Date: 2017-01-24 15:49 +0100
http://bitbucket.org/pypy/pypy/changeset/40f6115fdf6d/

Log:	hg merge default

diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/cffi/error.py
@@ -0,0 +1,20 @@
+
+class FFIError(Exception):
+    pass
+
+class CDefError(Exception):
+    def __str__(self):
+        try:
+            line = 'line %d: ' % (self.args[1].coord.line,)
+        except (AttributeError, TypeError, IndexError):
+            line = ''
+        return '%s%s' % (line, self.args[0])
+
+class VerificationError(Exception):
+    """ An error raised when verification fails
+    """
+
+class VerificationMissing(Exception):
+    """ An error raised when incomplete structures are passed into
+    cdef, but no verification has been done
+    """

From pypy.commits at gmail.com  Tue Jan 24 09:54:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 06:54:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: hg merge py3.5
Message-ID: <58876a97.09bb1c0a.e9e8b.f4ac@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89742:3d484dcb6db5
Date: 2017-01-24 15:51 +0100
http://bitbucket.org/pypy/pypy/changeset/3d484dcb6db5/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 3413 lines

diff --git a/lib-python/3/test/test_asyncio/test_events.py b/lib-python/3/test/test_asyncio/test_events.py
--- a/lib-python/3/test/test_asyncio/test_events.py
+++ b/lib-python/3/test/test_asyncio/test_events.py
@@ -825,9 +825,15 @@
         server = self.loop.run_until_complete(f)
         self.assertEqual(len(server.sockets), 1)
         sock = server.sockets[0]
-        self.assertFalse(
-            sock.getsockopt(
-                socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        try:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        except OSError:
+            raise unittest.SkipTest(
+                "Python's socket module was compiled using modern headers "
+                "thus defining SO_REUSEPORT but this process is running "
+                "under an older kernel that does not support SO_REUSEPORT.")
         server.close()
 
         test_utils.run_briefly(self.loop)
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.9.2
+Version: 1.10.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -1,11 +1,11 @@
 __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError',
            'FFIError']
 
-from .api import FFI, CDefError, FFIError
-from .ffiplatform import VerificationError, VerificationMissing
+from .api import FFI
+from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.9.2"
-__version_info__ = (1, 9, 2)
+__version__ = "1.10.0"
+__version_info__ = (1, 10, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.9.2"
+                               "\ncompiled with cffi version: 1.10.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -1,5 +1,7 @@
 import sys, types
 from .lock import allocate_lock
+from .error import CDefError
+from . import model
 
 try:
     callable
@@ -15,17 +17,6 @@
     basestring = str
 
 
-class FFIError(Exception):
-    pass
-
-class CDefError(Exception):
-    def __str__(self):
-        try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
-        except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
-
 
 class FFI(object):
     r'''
@@ -49,18 +40,27 @@
         """Create an FFI instance.  The 'backend' argument is used to
         select a non-default backend, mostly for tests.
         """
-        from . import cparser, model
         if backend is None:
             # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with
             # _cffi_backend.so compiled.
             import _cffi_backend as backend
             from . import __version__
-            assert backend.__version__ == __version__, \
-               "version mismatch, %s != %s" % (backend.__version__, __version__)
+            if backend.__version__ != __version__:
+                # bad version!  Try to be as explicit as possible.
+                if hasattr(backend, '__file__'):
+                    # CPython
+                    raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r.  When we import the top-level '_cffi_backend' extension module, we get version %s, located in %r.  The two versions should be equal; check your installation." % (
+                        __version__, __file__,
+                        backend.__version__, backend.__file__))
+                else:
+                    # PyPy
+                    raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r.  This interpreter comes with a built-in '_cffi_backend' module, which is version %s.  The two versions should be equal; check your installation." % (
+                        __version__, __file__, backend.__version__))
             # (If you insist you can also try to pass the option
             # 'backend=backend_ctypes.CTypesBackend()', but don't
             # rely on it!  It's probably not going to work well.)
 
+        from . import cparser
         self._backend = backend
         self._lock = allocate_lock()
         self._parser = cparser.Parser()
@@ -212,7 +212,7 @@
 
     def offsetof(self, cdecl, *fields_or_indexes):
         """Return the offset of the named field inside the given
-        structure or array, which must be given as a C type name.  
+        structure or array, which must be given as a C type name.
         You can give several field names in case of nested structures.
         You can also give numeric values which correspond to array
         items, in case of an array type.
@@ -300,7 +300,7 @@
         return self._backend.string(cdata, maxlen)
 
     def unpack(self, cdata, length):
-        """Unpack an array of C data of the given length, 
+        """Unpack an array of C data of the given length,
         returning a Python string/unicode/list.
 
         If 'cdata' is a pointer to 'char', returns a byte string.
@@ -452,7 +452,6 @@
         return self._backend.getwinerror(code)
 
     def _pointer_to(self, ctype):
-        from . import model
         with self._lock:
             return model.pointer_cache(self, ctype)
 
@@ -764,7 +763,6 @@
         return backend.load_library(path, flags)
 
 def _make_ffi_library(ffi, libname, flags):
-    import os
     backend = ffi._backend
     backendlib = _load_backend_lib(backend, libname, flags)
     #
@@ -802,7 +800,6 @@
         if accessors_version[0] is ffi._cdef_version:
             return
         #
-        from . import model
         for key, (tp, _) in ffi._parser._declarations.items():
             if not isinstance(tp, model.EnumType):
                 tag, name = key.split(' ', 1)
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -1,3 +1,4 @@
+from .error import VerificationError
 
 class CffiOp(object):
     def __init__(self, op, arg):
@@ -19,7 +20,6 @@
                                     % (self.arg,))
             return format_four_bytes(value)
         if isinstance(self.arg, str):
-            from .ffiplatform import VerificationError
             raise VerificationError("cannot emit to Python: %r" % (self.arg,))
         return format_four_bytes((self.arg << 8) | self.op)
 
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -1,5 +1,6 @@
 import sys
-from . import api, model
+from . import model
+from .error import FFIError
 
 
 COMMON_TYPES = {}
@@ -31,11 +32,11 @@
         elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result, quals = model.PrimitiveType(cdecl), 0
         elif cdecl == 'set-unicode-needed':
-            raise api.FFIError("The Windows type %r is only available after "
-                               "you call ffi.set_unicode()" % (commontype,))
+            raise FFIError("The Windows type %r is only available after "
+                           "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == cdecl:
-                raise api.FFIError(
+                raise FFIError(
                     "Unsupported type: %r.  Please look at "
         "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
                     "and file an issue if you think this type should really "
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -1,5 +1,6 @@
-from . import api, model
+from . import model
 from .commontypes import COMMON_TYPES, resolve_common_type
+from .error import FFIError, CDefError
 try:
     from . import _pycparser as pycparser
 except ImportError:
@@ -113,7 +114,7 @@
             # grouping variant
             closing = csource.find('}', endpos)
             if closing < 0:
-                raise api.CDefError("'extern \"Python\" {': no '}' found")
+                raise CDefError("'extern \"Python\" {': no '}' found")
             if csource.find('{', endpos + 1, closing) >= 0:
                 raise NotImplementedError("cannot use { } inside a block "
                                           "'extern \"Python\" { ... }'")
@@ -123,7 +124,7 @@
             # non-grouping variant
             semicolon = csource.find(';', endpos)
             if semicolon < 0:
-                raise api.CDefError("'extern \"Python\": no ';' found")
+                raise CDefError("'extern \"Python\": no ';' found")
             parts.append(csource[endpos:semicolon+1])
             csource = csource[semicolon+1:]
         parts.append(' void __cffi_extern_python_stop;')
@@ -288,7 +289,7 @@
             msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
-        raise api.CDefError(msg)
+        raise CDefError(msg)
 
     def parse(self, csource, override=False, packed=False, dllexport=False):
         prev_options = self._options
@@ -318,8 +319,8 @@
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
                     if not decl.name:
-                        raise api.CDefError("typedef does not declare any name",
-                                            decl)
+                        raise CDefError("typedef does not declare any name",
+                                        decl)
                     quals = 0
                     if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
                             and decl.type.type.names[-1] == '__dotdotdot__'):
@@ -337,8 +338,8 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise api.CDefError("unrecognized construct", decl)
-        except api.FFIError as e:
+                    raise CDefError("unrecognized construct", decl)
+        except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
                 e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
@@ -348,7 +349,7 @@
         if key in self._int_constants:
             if self._int_constants[key] == val:
                 return     # ignore identical double declarations
-            raise api.FFIError(
+            raise FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
 
@@ -375,7 +376,7 @@
             elif value == '...':
                 self._declare('macro ' + key, value)
             else:
-                raise api.CDefError(
+                raise CDefError(
                     'only supports one of the following syntax:\n'
                     '  #define %s ...     (literally dot-dot-dot)\n'
                     '  #define %s NUMBER  (with NUMBER an integer'
@@ -410,8 +411,8 @@
             elif isinstance(node, pycparser.c_ast.Enum):
                 self._get_struct_union_enum_type('enum', node)
             elif not decl.name:
-                raise api.CDefError("construct does not declare any variable",
-                                    decl)
+                raise CDefError("construct does not declare any variable",
+                                decl)
             #
             if decl.name:
                 tp, quals = self._get_type_and_quals(node,
@@ -438,7 +439,7 @@
                     self._inside_extern_python = decl.name
                 else:
                     if self._inside_extern_python !='__cffi_extern_python_stop':
-                        raise api.CDefError(
+                        raise CDefError(
                             "cannot declare constants or "
                             "variables with 'extern \"Python\"'")
                     if (quals & model.Q_CONST) and not tp.is_array_type:
@@ -454,7 +455,7 @@
         assert not macros
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
-            raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
+            raise CDefError("unknown identifier '%s'" % (exprnode.name,))
         return self._get_type_and_quals(exprnode.type)
 
     def _declare(self, name, obj, included=False, quals=0):
@@ -463,7 +464,7 @@
             if prevobj is obj and prevquals == quals:
                 return
             if not self._options.get('override'):
-                raise api.FFIError(
+                raise FFIError(
                     "multiple declarations of %s (for interactive usage, "
                     "try cdef(xx, override=True))" % (name,))
         assert '__dotdotdot__' not in name.split()
@@ -551,7 +552,7 @@
                 if ident == 'void':
                     return model.void_type, quals
                 if ident == '__dotdotdot__':
-                    raise api.FFIError(':%d: bad usage of "..."' %
+                    raise FFIError(':%d: bad usage of "..."' %
                             typenode.coord.line)
                 tp0, quals0 = resolve_common_type(self, ident)
                 return tp0, (quals | quals0)
@@ -583,14 +584,14 @@
             return self._get_struct_union_enum_type('union', typenode, name,
                                                     nested=True), 0
         #
-        raise api.FFIError(":%d: bad or unsupported type declaration" %
+        raise FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
     def _parse_function_type(self, typenode, funcname=None):
         params = list(getattr(typenode.args, 'params', []))
         for i, arg in enumerate(params):
             if not hasattr(arg, 'type'):
-                raise api.CDefError("%s arg %d: unknown type '%s'"
+                raise CDefError("%s arg %d: unknown type '%s'"
                     " (if you meant to use the old C syntax of giving"
                     " untyped arguments, it is not supported)"
                     % (funcname or 'in expression', i + 1,
@@ -604,7 +605,7 @@
         if ellipsis:
             params.pop()
             if not params:
-                raise api.CDefError(
+                raise CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
         args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type))
@@ -705,7 +706,7 @@
             return tp
         #
         if tp.fldnames is not None:
-            raise api.CDefError("duplicate declaration of struct %s" % name)
+            raise CDefError("duplicate declaration of struct %s" % name)
         fldnames = []
         fldtypes = []
         fldbitsize = []
@@ -749,7 +750,7 @@
 
     def _make_partial(self, tp, nested):
         if not isinstance(tp, model.StructOrUnion):
-            raise api.CDefError("%s cannot be partial" % (tp,))
+            raise CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
         tp.partial = True
@@ -769,7 +770,7 @@
                     len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
                 return ord(s[-2])
             else:
-                raise api.CDefError("invalid constant %r" % (s,))
+                raise CDefError("invalid constant %r" % (s,))
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '+'):
@@ -788,12 +789,12 @@
             if partial_length_ok:
                 self._partial_length = True
                 return '...'
-            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
-                               "the actual array length in this context"
-                               % exprnode.coord.line)
+            raise FFIError(":%d: unsupported '[...]' here, cannot derive "
+                           "the actual array length in this context"
+                           % exprnode.coord.line)
         #
-        raise api.FFIError(":%d: unsupported expression: expected a "
-                           "simple numeric constant" % exprnode.coord.line)
+        raise FFIError(":%d: unsupported expression: expected a "
+                       "simple numeric constant" % exprnode.coord.line)
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
@@ -843,8 +844,8 @@
             for t in typenames[:-1]:
                 if t not in ['int', 'short', 'long', 'signed',
                              'unsigned', 'char']:
-                    raise api.FFIError(':%d: bad usage of "..."' %
-                                       decl.coord.line)
+                    raise FFIError(':%d: bad usage of "..."' %
+                                   decl.coord.line)
             result = model.UnknownIntegerType(decl.name)
 
         if self._uses_new_feature is None:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/cffi/error.py
@@ -0,0 +1,20 @@
+
+class FFIError(Exception):
+    pass
+
+class CDefError(Exception):
+    def __str__(self):
+        try:
+            line = 'line %d: ' % (self.args[1].coord.line,)
+        except (AttributeError, TypeError, IndexError):
+            line = ''
+        return '%s%s' % (line, self.args[0])
+
+class VerificationError(Exception):
+    """ An error raised when verification fails
+    """
+
+class VerificationMissing(Exception):
+    """ An error raised when incomplete structures are passed into
+    cdef, but no verification has been done
+    """
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -1,14 +1,5 @@
 import sys, os
-
-
-class VerificationError(Exception):
-    """ An error raised when verification fails
-    """
-
-class VerificationMissing(Exception):
-    """ An error raised when incomplete structures are passed into
-    cdef, but no verification has been done
-    """
+from .error import VerificationError
 
 
 LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -1,8 +1,8 @@
-import types, sys
+import types
 import weakref
 
 from .lock import allocate_lock
-
+from .error import CDefError, VerificationError, VerificationMissing
 
 # type qualifiers
 Q_CONST    = 0x01
@@ -39,7 +39,6 @@
         replace_with = qualify(quals, replace_with)
         result = result.replace('&', replace_with)
         if '$' in result:
-            from .ffiplatform import VerificationError
             raise VerificationError(
                 "cannot generate '%s' in %s: unknown type name"
                 % (self._get_c_name(), context))
@@ -223,9 +222,8 @@
     is_raw_function = True
 
     def build_backend_type(self, ffi, finishlist):
-        from . import api
-        raise api.CDefError("cannot render the type %r: it is a function "
-                            "type, not a pointer-to-function type" % (self,))
+        raise CDefError("cannot render the type %r: it is a function "
+                        "type, not a pointer-to-function type" % (self,))
 
     def as_function_pointer(self):
         return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi)
@@ -307,9 +305,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         if self.length == '...':
-            from . import api
-            raise api.CDefError("cannot render the type %r: unknown length" %
-                                (self,))
+            raise CDefError("cannot render the type %r: unknown length" %
+                            (self,))
         self.item.get_cached_btype(ffi, finishlist)   # force the item BType
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
@@ -455,13 +452,11 @@
         self.completed = 2
 
     def _verification_error(self, msg):
-        from .ffiplatform import VerificationError
         raise VerificationError(msg)
 
     def check_not_partial(self):
         if self.partial and self.fixedlayout is None:
-            from . import ffiplatform
-            raise ffiplatform.VerificationMissing(self._get_c_name())
+            raise VerificationMissing(self._get_c_name())
 
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
@@ -499,8 +494,7 @@
 
     def check_not_partial(self):
         if self.partial and not self.partial_resolved:
-            from . import ffiplatform
-            raise ffiplatform.VerificationMissing(self._get_c_name())
+            raise VerificationMissing(self._get_c_name())
 
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
@@ -514,7 +508,6 @@
         if self.baseinttype is not None:
             return self.baseinttype.get_cached_btype(ffi, finishlist)
         #
-        from . import api
         if self.enumvalues:
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
@@ -549,8 +542,8 @@
         if (smallest_value >= ((-1) << (8*size2-1)) and
             largest_value < (1 << (8*size2-sign))):
             return btype2
-        raise api.CDefError("%s values don't all fit into either 'long' "
-                            "or 'unsigned long'" % self._get_c_name())
+        raise CDefError("%s values don't all fit into either 'long' "
+                        "or 'unsigned long'" % self._get_c_name())
 
 def unknown_type(name, structname=None):
     if structname is None:
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -1,5 +1,6 @@
 import os, sys, io
 from . import ffiplatform, model
+from .error import VerificationError
 from .cffi_opcode import *
 
 VERSION = "0x2601"
@@ -211,7 +212,7 @@
                 method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in recompile(): %r" % name)
             try:
                 self._current_quals = quals
@@ -354,12 +355,12 @@
                     included_module_name, included_source = (
                         ffi_to_include._assigned_source[:2])
                 except AttributeError:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "ffi object %r includes %r, but the latter has not "
                         "been prepared with set_source()" % (
                             self.ffi, ffi_to_include,))
                 if included_source is None:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "not implemented yet: ffi.include() of a Python-based "
                         "ffi inside a C-based ffi")
                 prnt('  "%s",' % (included_module_name,))
@@ -391,6 +392,10 @@
         prnt()
         #
         # the init function
+        prnt('#ifdef __GNUC__')
+        prnt('#  pragma GCC visibility push(default)  /* for -fvisibility= */')
+        prnt('#endif')
+        prnt()
         prnt('#ifdef PYPY_VERSION')
         prnt('PyMODINIT_FUNC')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
@@ -429,6 +434,10 @@
             self.module_name, version))
         prnt('}')
         prnt('#endif')
+        prnt()
+        prnt('#ifdef __GNUC__')
+        prnt('#  pragma GCC visibility pop')
+        prnt('#endif')
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -456,12 +465,12 @@
                 included_module_name, included_source = (
                     ffi_to_include._assigned_source[:2])
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "ffi object %r includes %r, but the latter has not "
                     "been prepared with set_source()" % (
                         self.ffi, ffi_to_include,))
             if included_source is not None:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented yet: ffi.include() of a C-based "
                     "ffi inside a Python-based ffi")
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
@@ -831,7 +840,7 @@
                 prnt('  { %s = &p->%s; (void)tmp; }' % (
                     ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                     fname))
-            except ffiplatform.VerificationError as e:
+            except VerificationError as e:
                 prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname))
@@ -994,7 +1003,7 @@
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
                             check_value=None):
         if (category, name) in self._seen_constants:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "duplicate declaration of %s '%s'" % (category, name))
         self._seen_constants.add((category, name))
         #
@@ -1093,7 +1102,7 @@
     def _generate_cpy_macro_ctx(self, tp, name):
         if tp == '...':
             if self.target_is_python:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "cannot use the syntax '...' in '#define %s ...' when "
                     "using the ABI mode" % (name,))
             check_value = None
@@ -1226,7 +1235,7 @@
 
     def _generate_cpy_extern_python_ctx(self, tp, name):
         if self.target_is_python:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "cannot use 'extern \"Python\"' in the ABI mode")
         if tp.ellipsis:
             raise NotImplementedError("a vararg function is extern \"Python\"")
@@ -1307,7 +1316,7 @@
         if tp.length is None:
             self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index)
         elif tp.length == '...':
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "type %s badly placed: the '...' array length can only be "
                 "used on global arrays or on fields of structures" % (
                     str(tp).replace('/*...*/', '...'),))
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -79,9 +79,10 @@
     CPython itself should ignore the flag in a debugging version
     (by not listing .abi3.so in the extensions it supports), but
     it doesn't so far, creating troubles.  That's why we check
-    for "not sys.flags.debug". (http://bugs.python.org/issue28401)
+    for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
+    of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
     """
-    if 'py_limited_api' not in kwds and not sys.flags.debug:
+    if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -2,7 +2,8 @@
 # DEPRECATED: implementation for ffi.verify()
 #
 import sys, imp
-from . import model, ffiplatform
+from . import model
+from .error import VerificationError
 
 
 class VCPythonEngine(object):
@@ -155,7 +156,7 @@
                                           self.verifier.modulefilename)
             except ImportError as e:
                 error = "importing %r: %s" % (self.verifier.modulefilename, e)
-                raise ffiplatform.VerificationError(error)
+                raise VerificationError(error)
             finally:
                 if hasattr(sys, "setdlopenflags"):
                     sys.setdlopenflags(previous_flags)
@@ -185,7 +186,7 @@
             def __dir__(self):
                 return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
-        if module._cffi_setup(lst, ffiplatform.VerificationError, library):
+        if module._cffi_setup(lst, VerificationError, library):
             import warnings
             warnings.warn("reimporting %r might overwrite older definitions"
                           % (self.verifier.get_module_name()))
@@ -212,7 +213,7 @@
                 method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in verify(): %r" % name)
             try:
                 method(tp, realname)
@@ -485,7 +486,7 @@
                     prnt('  { %s = &p->%s; (void)tmp; }' % (
                         ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                         fname))
-                except ffiplatform.VerificationError as e:
+                except VerificationError as e:
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         prnt('static PyObject *')
@@ -550,7 +551,7 @@
             # check that the layout sizes and offsets match the real ones
             def check(realvalue, expectedvalue, msg):
                 if realvalue != expectedvalue:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "%s (we have %d, but C compiler says %d)"
                         % (msg, expectedvalue, realvalue))
             ffi = self.ffi
@@ -771,7 +772,7 @@
                 BItemType = self.ffi._get_cached_btype(tp.item)
                 length, rest = divmod(size, self.ffi.sizeof(BItemType))
                 if rest != 0:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "bad size: %r does not seem to be an array of %s" %
                         (name, tp.item))
                 tp = tp.resolve_length(length)
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -4,7 +4,8 @@
 import sys, os
 import types
 
-from . import model, ffiplatform
+from . import model
+from .error import VerificationError
 
 
 class VGenericEngine(object):
@@ -102,7 +103,7 @@
                 method = getattr(self, '_generate_gen_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "not implemented in verify(): %r" % name)
             try:
                 method(tp, realname)
@@ -281,7 +282,7 @@
                     prnt('  { %s = &p->%s; (void)tmp; }' % (
                         ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                         fname))
-                except ffiplatform.VerificationError as e:
+                except VerificationError as e:
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         self.export_symbols.append(layoutfuncname)
@@ -344,7 +345,7 @@
             # check that the layout sizes and offsets match the real ones
             def check(realvalue, expectedvalue, msg):
                 if realvalue != expectedvalue:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "%s (we have %d, but C compiler says %d)"
                         % (msg, expectedvalue, realvalue))
             ffi = self.ffi
@@ -498,7 +499,7 @@
             error = self.ffi.string(p)
             if sys.version_info >= (3,):
                 error = str(error, 'utf-8')
-            raise ffiplatform.VerificationError(error)
+            raise VerificationError(error)
 
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
@@ -591,7 +592,7 @@
                 BItemType = self.ffi._get_cached_btype(tp.item)
                 length, rest = divmod(size, self.ffi.sizeof(BItemType))
                 if rest != 0:
-                    raise ffiplatform.VerificationError(
+                    raise VerificationError(
                         "bad size: %r does not seem to be an array of %s" %
                         (name, tp.item))
                 tp = tp.resolve_length(length)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -4,6 +4,7 @@
 import sys, os, binascii, shutil, io
 from . import __version_verifier_modules__
 from . import ffiplatform
+from .error import VerificationError
 
 if sys.version_info >= (3, 3):
     import importlib.machinery
@@ -42,7 +43,7 @@
                  ext_package=None, tag='', force_generic_engine=False,
                  source_extension='.c', flags=None, relative_to=None, **kwds):
         if ffi._parser._uses_new_feature:
-            raise ffiplatform.VerificationError(
+            raise VerificationError(
                 "feature not supported with ffi.verify(), but only "
                 "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,))
         self.ffi = ffi
@@ -83,7 +84,7 @@
         which can be tweaked beforehand."""
         with self.ffi._lock:
             if self._has_source and file is None:
-                raise ffiplatform.VerificationError(
+                raise VerificationError(
                     "source code already written")
             self._write_source(file)
 
@@ -92,7 +93,7 @@
         This produces a dynamic link library in 'self.modulefilename'."""
         with self.ffi._lock:
             if self._has_module:
-                raise ffiplatform.VerificationError("module already compiled")
+                raise VerificationError("module already compiled")
             if not self._has_source:
                 self._write_source()
             self._compile_module()
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -31,6 +31,7 @@
 
 class AST(object):
     __metaclass__ = extendabletype
+    _attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")
@@ -138,7 +139,7 @@
         self.w_AST = space.gettypeobject(W_AST.typedef)
         for (name, base, fields, attributes) in self.AST_TYPES:
             self.make_new_type(space, name, base, fields, attributes)
-        
+
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
@@ -150,7 +151,7 @@
             space.setitem_str(w_dict, "_attributes",
                               space.newtuple([space.wrap(a) for a in attributes]))
         w_type = space.call_function(
-            space.w_type, 
+            space.w_type,
             space.wrap(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
@@ -184,7 +185,9 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_Module(self)
 
     def to_object(self, space):
@@ -217,7 +220,9 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_Interactive(self)
 
     def to_object(self, space):
@@ -279,7 +284,9 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_Suite(self)
 
     def to_object(self, space):
@@ -380,9 +387,13 @@
     def mutate_over(self, visitor):
         self.args = self.args.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.decorator_list:
-            visitor._mutate_sequence(self.decorator_list)
+            for i in range(len(self.decorator_list)):
+                if self.decorator_list[i] is not None:
+                    self.decorator_list[i] = self.decorator_list[i].mutate_over(visitor)
         if self.returns:
             self.returns = self.returns.mutate_over(visitor)
         return visitor.visit_FunctionDef(self)
@@ -456,9 +467,13 @@
     def mutate_over(self, visitor):
         self.args = self.args.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.decorator_list:
-            visitor._mutate_sequence(self.decorator_list)
+            for i in range(len(self.decorator_list)):
+                if self.decorator_list[i] is not None:
+                    self.decorator_list[i] = self.decorator_list[i].mutate_over(visitor)
         if self.returns:
             self.returns = self.returns.mutate_over(visitor)
         return visitor.visit_AsyncFunctionDef(self)
@@ -531,13 +546,21 @@
 
     def mutate_over(self, visitor):
         if self.bases:
-            visitor._mutate_sequence(self.bases)
+            for i in range(len(self.bases)):
+                if self.bases[i] is not None:
+                    self.bases[i] = self.bases[i].mutate_over(visitor)
         if self.keywords:
-            visitor._mutate_sequence(self.keywords)
+            for i in range(len(self.keywords)):
+                if self.keywords[i] is not None:
+                    self.keywords[i] = self.keywords[i].mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.decorator_list:
-            visitor._mutate_sequence(self.decorator_list)
+            for i in range(len(self.decorator_list)):
+                if self.decorator_list[i] is not None:
+                    self.decorator_list[i] = self.decorator_list[i].mutate_over(visitor)
         return visitor.visit_ClassDef(self)
 
     def to_object(self, space):
@@ -649,7 +672,9 @@
 
     def mutate_over(self, visitor):
         if self.targets:
-            visitor._mutate_sequence(self.targets)
+            for i in range(len(self.targets)):
+                if self.targets[i] is not None:
+                    self.targets[i] = self.targets[i].mutate_over(visitor)
         return visitor.visit_Delete(self)
 
     def to_object(self, space):
@@ -692,7 +717,9 @@
 
     def mutate_over(self, visitor):
         if self.targets:
-            visitor._mutate_sequence(self.targets)
+            for i in range(len(self.targets)):
+                if self.targets[i] is not None:
+                    self.targets[i] = self.targets[i].mutate_over(visitor)
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Assign(self)
 
@@ -799,9 +826,13 @@
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_For(self)
 
     def to_object(self, space):
@@ -869,9 +900,13 @@
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_AsyncFor(self)
 
     def to_object(self, space):
@@ -937,9 +972,13 @@
     def mutate_over(self, visitor):
         self.test = self.test.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_While(self)
 
     def to_object(self, space):
@@ -999,9 +1038,13 @@
     def mutate_over(self, visitor):
         self.test = self.test.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         return visitor.visit_If(self)
 
     def to_object(self, space):
@@ -1059,9 +1102,13 @@
 
     def mutate_over(self, visitor):
         if self.items:
-            visitor._mutate_sequence(self.items)
+            for i in range(len(self.items)):
+                if self.items[i] is not None:
+                    self.items[i] = self.items[i].mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_With(self)
 
     def to_object(self, space):
@@ -1113,9 +1160,13 @@
 
     def mutate_over(self, visitor):
         if self.items:
-            visitor._mutate_sequence(self.items)
+            for i in range(len(self.items)):
+                if self.items[i] is not None:
+                    self.items[i] = self.items[i].mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_AsyncWith(self)
 
     def to_object(self, space):
@@ -1213,13 +1264,21 @@
 
     def mutate_over(self, visitor):
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         if self.handlers:
-            visitor._mutate_sequence(self.handlers)
+            for i in range(len(self.handlers)):
+                if self.handlers[i] is not None:
+                    self.handlers[i] = self.handlers[i].mutate_over(visitor)
         if self.orelse:
-            visitor._mutate_sequence(self.orelse)
+            for i in range(len(self.orelse)):
+                if self.orelse[i] is not None:
+                    self.orelse[i] = self.orelse[i].mutate_over(visitor)
         if self.finalbody:
-            visitor._mutate_sequence(self.finalbody)
+            for i in range(len(self.finalbody)):
+                if self.finalbody[i] is not None:
+                    self.finalbody[i] = self.finalbody[i].mutate_over(visitor)
         return visitor.visit_Try(self)
 
     def to_object(self, space):
@@ -1333,7 +1392,9 @@
 
     def mutate_over(self, visitor):
         if self.names:
-            visitor._mutate_sequence(self.names)
+            for i in range(len(self.names)):
+                if self.names[i] is not None:
+                    self.names[i] = self.names[i].mutate_over(visitor)
         return visitor.visit_Import(self)
 
     def to_object(self, space):
@@ -1377,7 +1438,9 @@
 
     def mutate_over(self, visitor):
         if self.names:
-            visitor._mutate_sequence(self.names)
+            for i in range(len(self.names)):
+                if self.names[i] is not None:
+                    self.names[i] = self.names[i].mutate_over(visitor)
         return visitor.visit_ImportFrom(self)
 
     def to_object(self, space):
@@ -1710,7 +1773,9 @@
 
     def mutate_over(self, visitor):
         if self.values:
-            visitor._mutate_sequence(self.values)
+            for i in range(len(self.values)):
+                if self.values[i] is not None:
+                    self.values[i] = self.values[i].mutate_over(visitor)
         return visitor.visit_BoolOp(self)
 
     def to_object(self, space):
@@ -1957,9 +2022,13 @@
 
     def mutate_over(self, visitor):
         if self.keys:
-            visitor._mutate_sequence(self.keys)
+            for i in range(len(self.keys)):
+                if self.keys[i] is not None:
+                    self.keys[i] = self.keys[i].mutate_over(visitor)
         if self.values:
-            visitor._mutate_sequence(self.values)
+            for i in range(len(self.values)):
+                if self.values[i] is not None:
+                    self.values[i] = self.values[i].mutate_over(visitor)
         return visitor.visit_Dict(self)
 
     def to_object(self, space):
@@ -2010,7 +2079,9 @@
 
     def mutate_over(self, visitor):
         if self.elts:
-            visitor._mutate_sequence(self.elts)
+            for i in range(len(self.elts)):
+                if self.elts[i] is not None:
+                    self.elts[i] = self.elts[i].mutate_over(visitor)
         return visitor.visit_Set(self)
 
     def to_object(self, space):
@@ -2054,7 +2125,9 @@
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                if self.generators[i] is not None:
+                    self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_ListComp(self)
 
     def to_object(self, space):
@@ -2104,7 +2177,9 @@
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                if self.generators[i] is not None:
+                    self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_SetComp(self)
 
     def to_object(self, space):
@@ -2156,7 +2231,9 @@
         self.key = self.key.mutate_over(visitor)
         self.value = self.value.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                if self.generators[i] is not None:
+                    self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_DictComp(self)
 
     def to_object(self, space):
@@ -2212,7 +2289,9 @@
     def mutate_over(self, visitor):
         self.elt = self.elt.mutate_over(visitor)
         if self.generators:
-            visitor._mutate_sequence(self.generators)
+            for i in range(len(self.generators)):
+                if self.generators[i] is not None:
+                    self.generators[i] = self.generators[i].mutate_over(visitor)
         return visitor.visit_GeneratorExp(self)
 
     def to_object(self, space):
@@ -2376,7 +2455,9 @@
     def mutate_over(self, visitor):
         self.left = self.left.mutate_over(visitor)
         if self.comparators:
-            visitor._mutate_sequence(self.comparators)
+            for i in range(len(self.comparators)):
+                if self.comparators[i] is not None:
+                    self.comparators[i] = self.comparators[i].mutate_over(visitor)
         return visitor.visit_Compare(self)
 
     def to_object(self, space):
@@ -2436,9 +2517,13 @@
     def mutate_over(self, visitor):
         self.func = self.func.mutate_over(visitor)
         if self.args:
-            visitor._mutate_sequence(self.args)
+            for i in range(len(self.args)):
+                if self.args[i] is not None:
+                    self.args[i] = self.args[i].mutate_over(visitor)
         if self.keywords:
-            visitor._mutate_sequence(self.keywords)
+            for i in range(len(self.keywords)):
+                if self.keywords[i] is not None:
+                    self.keywords[i] = self.keywords[i].mutate_over(visitor)
         return visitor.visit_Call(self)
 
     def to_object(self, space):
@@ -2689,8 +2774,8 @@
 
 class NameConstant(expr):
 
-    def __init__(self, single, lineno, col_offset):
-        self.single = single
+    def __init__(self, value, lineno, col_offset):
+        self.value = value
         expr.__init__(self, lineno, col_offset)
 
     def walkabout(self, visitor):
@@ -2701,8 +2786,8 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_NameConstant)
-        w_single = self.single  # singleton
-        space.setattr(w_node, space.wrap('single'), w_single)
+        w_value = self.value  # singleton
+        space.setattr(w_node, space.wrap('value'), w_value)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
         w_col_offset = space.wrap(self.col_offset)  # int
@@ -2711,17 +2796,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        w_single = get_field(space, w_node, 'single', False)
+        w_value = get_field(space, w_node, 'value', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _single = w_single
-        if _single is None:
-            raise_required_value(space, w_node, 'single')
+        _value = w_value
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
-        return NameConstant(_single, _lineno, _col_offset)
-
-State.ast_type('NameConstant', 'expr', ['single'])
+        return NameConstant(_value, _lineno, _col_offset)
+
+State.ast_type('NameConstant', 'expr', ['value'])
 
 
 class Ellipsis(expr):
@@ -2960,7 +3045,9 @@
 
     def mutate_over(self, visitor):
         if self.elts:
-            visitor._mutate_sequence(self.elts)
+            for i in range(len(self.elts)):
+                if self.elts[i] is not None:
+                    self.elts[i] = self.elts[i].mutate_over(visitor)
         return visitor.visit_List(self)
 
     def to_object(self, space):
@@ -3009,7 +3096,9 @@
 
     def mutate_over(self, visitor):
         if self.elts:
-            visitor._mutate_sequence(self.elts)
+            for i in range(len(self.elts)):
+                if self.elts[i] is not None:
+                    self.elts[i] = self.elts[i].mutate_over(visitor)
         return visitor.visit_Tuple(self)
 
     def to_object(self, space):
@@ -3215,7 +3304,9 @@
 
     def mutate_over(self, visitor):
         if self.dims:
-            visitor._mutate_sequence(self.dims)
+            for i in range(len(self.dims)):
+                if self.dims[i] is not None:
+                    self.dims[i] = self.dims[i].mutate_over(visitor)
         return visitor.visit_ExtSlice(self)
 
     def to_object(self, space):
@@ -3583,7 +3674,9 @@
         self.target = self.target.mutate_over(visitor)
         self.iter = self.iter.mutate_over(visitor)
         if self.ifs:
-            visitor._mutate_sequence(self.ifs)
+            for i in range(len(self.ifs)):
+                if self.ifs[i] is not None:
+                    self.ifs[i] = self.ifs[i].mutate_over(visitor)
         return visitor.visit_comprehension(self)
 
     def walkabout(self, visitor):
@@ -3651,7 +3744,9 @@
         if self.type:
             self.type = self.type.mutate_over(visitor)
         if self.body:
-            visitor._mutate_sequence(self.body)
+            for i in range(len(self.body)):
+                if self.body[i] is not None:
+                    self.body[i] = self.body[i].mutate_over(visitor)
         return visitor.visit_ExceptHandler(self)
 
     def to_object(self, space):
@@ -3702,17 +3797,25 @@
 
     def mutate_over(self, visitor):
         if self.args:
-            visitor._mutate_sequence(self.args)
+            for i in range(len(self.args)):
+                if self.args[i] is not None:
+                    self.args[i] = self.args[i].mutate_over(visitor)
         if self.vararg:
             self.vararg = self.vararg.mutate_over(visitor)
         if self.kwonlyargs:
-            visitor._mutate_sequence(self.kwonlyargs)
+            for i in range(len(self.kwonlyargs)):
+                if self.kwonlyargs[i] is not None:
+                    self.kwonlyargs[i] = self.kwonlyargs[i].mutate_over(visitor)
         if self.kw_defaults:
-            visitor._mutate_sequence(self.kw_defaults)
+            for i in range(len(self.kw_defaults)):
+                if self.kw_defaults[i] is not None:
+                    self.kw_defaults[i] = self.kw_defaults[i].mutate_over(visitor)
         if self.kwarg:
             self.kwarg = self.kwarg.mutate_over(visitor)
         if self.defaults:
-            visitor._mutate_sequence(self.defaults)
+            for i in range(len(self.defaults)):
+                if self.defaults[i] is not None:
+                    self.defaults[i] = self.defaults[i].mutate_over(visitor)
         return visitor.visit_arguments(self)
 
     def walkabout(self, visitor):
@@ -3923,11 +4026,6 @@
     def default_visitor(self, node):
         raise NodeVisitorNotImplemented
 
-    def _mutate_sequence(self, seq):
-        for i in range(len(seq)):
-            if seq[i] is not None:
-                seq[i] = seq[i].mutate_over(self)
-
     def visit_Module(self, node):
         return self.default_visitor(node)
     def visit_Interactive(self, node):
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -8,6 +8,7 @@
 # please.
 import struct
 
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.astcompiler import ast, assemble, symtable, consts, misc
 from pypy.interpreter.astcompiler import optimize # For side effects
 from pypy.interpreter.pyparser.error import SyntaxError
@@ -337,6 +338,7 @@
         for i, default in enumerate(args.kw_defaults):
             if default:
                 kwonly = args.kwonlyargs[i]
+                assert isinstance(kwonly, ast.arg)
                 mangled = self.scope.mangle(kwonly.arg).decode('utf-8')
                 self.load_const(self.space.wrap(mangled))
                 default.walkabout(self)
@@ -351,16 +353,20 @@
     def _visit_arg_annotations(self, args, names):
         if args:
             for arg in args:
+                assert isinstance(arg, ast.arg)
                 self._visit_arg_annotation(arg.arg, arg.annotation, names)
 
+    @specialize.argtype(1)
     def _visit_annotations(self, func, args, returns):
         space = self.space
         names = []
         self._visit_arg_annotations(args.args, names)
-        if args.vararg:
-            self._visit_arg_annotation(args.vararg.arg, args.vararg.annotation,
+        vararg = args.vararg
+        if vararg:
+            self._visit_arg_annotation(vararg.arg, vararg.annotation,
                                        names)
         self._visit_arg_annotations(args.kwonlyargs, names)
+        kwarg = args.kwarg
         if args.kwarg:
             self._visit_arg_annotation(args.kwarg.arg, args.kwarg.annotation,
                                        names)
@@ -375,6 +381,7 @@
             l += 1
         return l
 
+    @specialize.arg(2)
     def _visit_function(self, func, function_code_generator):
         self.update_position(func.lineno, True)
         # Load decorators first, but apply them after the function is created.
@@ -923,10 +930,12 @@
         self.update_position(wih.lineno, True)
         self.handle_withitem(wih, 0, is_async=False)
 
+    @specialize.argtype(1)
     def handle_withitem(self, wih, pos, is_async):
         body_block = self.new_block()
         cleanup = self.new_block()
         witem = wih.items[pos]
+        assert isinstance(witem, ast.withitem)
         witem.context_expr.walkabout(self)
         if not is_async:
             self.emit_jump(ops.SETUP_WITH, cleanup)
@@ -1237,7 +1246,7 @@
 
     def visit_NameConstant(self, node):
         self.update_position(node.lineno)
-        self.load_const(node.single)
+        self.load_const(node.value)
 
     def visit_keyword(self, keyword):
         if keyword.arg is not None:
@@ -1288,6 +1297,7 @@
         nseen = 0 # the number of keyword arguments on the stack following
         if keywords is not None:
             for kw in keywords:
+                assert isinstance(kw, ast.keyword)
                 if kw.arg is None:
                     # A keyword argument unpacking.
                     if nseen:
@@ -1345,6 +1355,7 @@
                     return False
         if call.keywords is not None:
             for kw in call.keywords:
+                assert isinstance(kw, ast.keyword)
                 if kw.arg is None:
                     return False
         return True
diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -343,11 +343,16 @@
                 parse_f_string(astbuilder, joined_pieces, w_next, atom_node)
 
     except error.OperationError as e:
-        if not (e.match(space, space.w_UnicodeError) or
-                e.match(space, space.w_ValueError)):
+        if e.match(space, space.w_UnicodeError):
+            kind = 'unicode error'
+        elif e.match(space, space.w_ValueError):
+            kind = 'value error'
+        else:
             raise
         # Unicode/ValueError in literal: turn into SyntaxError
-        raise astbuilder.error(e.errorstr(space), atom_node)
+        e.normalize_exception(space)
+        errmsg = space.str_w(space.str(e.get_w_value(space)))
+        raise self.error('(%s) %s' % (kind, errmsg), atom_node)
 
     if len(joined_pieces) == 1:   # <= the common path
         return joined_pieces[0]   # ast.Str, Bytes or FormattedValue
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.error import OperationError
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.runicode import MAXUNICODE
+from rpython.rlib.objectmodel import specialize
 
 
 def optimize_ast(space, tree, compile_info):
@@ -70,7 +71,7 @@
 class __extend__(ast.NameConstant):
 
     def as_constant(self):
-        return self.single
+        return self.value
 
 class __extend__(ast.Index):
     def as_constant(self):
@@ -177,6 +178,7 @@
         self.space = space
         self.compile_info = compile_info
 
+    @specialize.argtype(1)
     def default_visitor(self, node):
         return node
 
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -89,12 +89,12 @@
         """Called when a yield is found."""
         raise SyntaxError("'yield' outside function", yield_node.lineno,
                           yield_node.col_offset)
-    
+
     def note_yieldFrom(self, yieldFrom_node):
         """Called when a yield from is found."""
         raise SyntaxError("'yield' outside function", yieldFrom_node.lineno,
                           yieldFrom_node.col_offset)
-    
+
     def note_await(self, await_node):
         """Called when await is found."""
         raise SyntaxError("'await' outside function", await_node.lineno,
@@ -260,12 +260,12 @@
         self.is_generator = True
         if self._in_try_body_depth > 0:
             self.has_yield_inside_try = True
-    
+
     def note_yieldFrom(self, yield_node):
         self.is_generator = True
         if self._in_try_body_depth > 0:
             self.has_yield_inside_try = True
-            
+
     def note_await(self, await_node):
         if self.name == '<genexpr>':
             msg = "'await' expressions in comprehensions are not supported"
@@ -315,7 +315,7 @@
     def note_yieldFrom(self, yield_node):
         raise SyntaxError("'yield from' inside async function", yield_node.lineno,
                           yield_node.col_offset)
-        
+
     def note_await(self, await_node):
         # Compatibility with CPython 3.5: set the CO_GENERATOR flag in
         # addition to the CO_COROUTINE flag if the function uses the
@@ -414,7 +414,7 @@
         func.args.walkabout(self)
         self.visit_sequence(func.body)
         self.pop_scope()
-    
+
     def visit_AsyncFunctionDef(self, func):
         self.note_symbol(func.name, SYM_ASSIGNED)
         # Function defaults and decorators happen in the outer scope.
@@ -429,7 +429,7 @@
         func.args.walkabout(self)
         self.visit_sequence(func.body)
         self.pop_scope()
-    
+
     def visit_Await(self, aw):
         self.scope.note_await(aw)
         ast.GenericASTVisitor.visit_Await(self, aw)
@@ -572,7 +572,7 @@
         witem.context_expr.walkabout(self)
         if witem.optional_vars:
             witem.optional_vars.walkabout(self)
-    
+
     def visit_AsyncWith(self, aw):
         self.scope.new_temporary_name()
         self.visit_sequence(aw.items)
@@ -595,8 +595,9 @@
             scope.note_keywords_arg(arguments.kwarg)
 
     def _handle_params(self, params, is_toplevel):
-        for i in range(len(params)):
-            arg = params[i].arg
+        for param in params:
+            assert isinstance(param, ast.arg)
+            arg = param.arg
             self.note_symbol(arg, SYM_PARAM)
 
     def _visit_annotations(self, func):
@@ -611,6 +612,7 @@
 
     def _visit_arg_annotations(self, args):
         for arg in args:
+            assert isinstance(arg, ast.arg)
             if arg.annotation:
                 arg.annotation.walkabout(self)
 
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1398,3 +1398,9 @@
         assert len(asyncwith.body) == 1
         assert isinstance(asyncwith.body[0], ast.Expr)
         assert isinstance(asyncwith.body[0].value, ast.Num)
+
+    def test_decode_error_in_string_literal(self):
+        input = "u'\\x'"
+        exc = py.test.raises(SyntaxError, self.get_ast, input).value
+        assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode"
+                           " bytes in position 0-1: truncated \\xXX escape")
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -73,8 +73,7 @@
          | FormattedValue(expr value, int? conversion, expr? format_spec)
          | JoinedStr(expr* values)
          | Bytes(bytes s)
-         -- PyPy mod. first argument name must not be value
-         | NameConstant(singleton single)
+         | NameConstant(singleton value)
          | Ellipsis
 
          -- the following expression can appear in assignment context
diff --git a/pypy/interpreter/astcompiler/tools/asdl.py b/pypy/interpreter/astcompiler/tools/asdl.py
--- a/pypy/interpreter/astcompiler/tools/asdl.py
+++ b/pypy/interpreter/astcompiler/tools/asdl.py
@@ -33,7 +33,8 @@
 # See the EBNF at the top of the file to understand the logical connection
 # between the various node types.
 
-builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton'}
+builtin_types = {'identifier', 'string', 'bytes', 'int', 'bool', 'object',
+                 'singleton'}
 
 class AST:
     def __repr__(self):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -145,7 +145,7 @@
             if allow_none:
                 wrapper += " if %s is not None else space.w_None" % (value,)
             return wrapper
-        
+
     def get_value_extractor(self, field, value):
         if field.type in self.data.simple_types:
             return "%s.from_object(space, %s)" % (field.type, value)
@@ -207,7 +207,7 @@
                 lines.append("if _%s is None:" % (field.name,))
                 lines.append("    raise_required_value(space, w_node, '%s')"
                              % (field.name,))
-            
+
         return lines
 
     def make_converters(self, fields, name, extras=None):
@@ -245,7 +245,7 @@
             if extras:
                 base_args = ", ".join(str(field.name) for field in extras)
                 self.emit("%s.__init__(self, %s)" % (base, base_args), 2)
-    
+
     def make_mutate_over(self, cons, name):
         self.emit("def mutate_over(self, visitor):", 1)
         for field in cons.fields:
@@ -257,12 +257,19 @@
                 else:
                     level = 2
                 if field.seq:
-                    sub = (field.name,)
-                    self.emit("visitor._mutate_sequence(self.%s)" % sub, level)
+                    sub = field.name
+                    self.emit("for i in range(len(self.{})):".format(sub),
+                        level)
+                    self.emit("if self.{}[i] is not None:".format(sub),
+                        level + 1)
+                    self.emit(
+                        "self.{0}[i] = self.{0}[i].mutate_over(visitor)".format(sub),
+                        level + 2)
                 else:
-                    sub = (field.name, field.name)
-                    self.emit("self.%s = self.%s.mutate_over(visitor)" % sub,
-                              level)
+                    sub = field.name
+                    self.emit(
+                        "self.{0} = self.{0}.mutate_over(visitor)".format(sub),
+                        level)
         self.emit("return visitor.visit_%s(self)" % (name,), 2)
         self.emit("")
 
@@ -276,7 +283,7 @@
         self.emit("")
         self.make_mutate_over(cons, cons.name)
         self.make_converters(cons.fields, cons.name, extra_attributes)
-        self.emit("State.ast_type(%r, '%s', %s)" % 
+        self.emit("State.ast_type(%r, '%s', %s)" %
                   (cons.name, base, [f.name for f in cons.fields]))
         self.emit("")
 
@@ -305,11 +312,6 @@
         self.emit("def default_visitor(self, node):", 1)
         self.emit("raise NodeVisitorNotImplemented", 2)
         self.emit("")
-        self.emit("def _mutate_sequence(self, seq):", 1)
-        self.emit("for i in range(len(seq)):", 2)
-        self.emit("if seq[i] is not None:", 3)
-        self.emit("seq[i] = seq[i].mutate_over(self)", 4)
-        self.emit("")
         super(ASTVisitorVisitor, self).visitModule(mod)
         self.emit("")
 
@@ -357,7 +359,7 @@
         self.emit("")
 
     def visitField(self, field):
-        if (field.type not in asdl.builtin_types and 
+        if (field.type not in asdl.builtin_types and
             field.type not in self.data.simple_types):
             level = 2
             template = "node.%s.walkabout(self)"
@@ -451,6 +453,7 @@
 
 class AST(object):
     __metaclass__ = extendabletype
+    _attrs_ = ['lineno', 'col_offset']
 
     def walkabout(self, visitor):
         raise AssertionError("walkabout() implementation not provided")
@@ -558,7 +561,7 @@
         self.w_AST = space.gettypeobject(W_AST.typedef)
         for (name, base, fields, attributes) in self.AST_TYPES:
             self.make_new_type(space, name, base, fields, attributes)
-        
+
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
@@ -570,7 +573,7 @@
             space.setitem_str(w_dict, "_attributes",
                               space.newtuple([space.wrap(a) for a in attributes]))
         w_type = space.call_function(
-            space.w_type, 
+            space.w_type,
             space.wrap(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
diff --git a/pypy/interpreter/astcompiler/tools/spark.py b/pypy/interpreter/astcompiler/tools/spark.py
deleted file mode 100644
--- a/pypy/interpreter/astcompiler/tools/spark.py
+++ /dev/null
@@ -1,839 +0,0 @@
-#  Copyright (c) 1998-2002 John Aycock
-#
-#  Permission is hereby granted, free of charge, to any person obtaining
-#  a copy of this software and associated documentation files (the
-#  "Software"), to deal in the Software without restriction, including
-#  without limitation the rights to use, copy, modify, merge, publish,
-#  distribute, sublicense, and/or sell copies of the Software, and to
-#  permit persons to whom the Software is furnished to do so, subject to
-#  the following conditions:
-#
-#  The above copyright notice and this permission notice shall be
-#  included in all copies or substantial portions of the Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-#  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-#  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-#  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-__version__ = 'SPARK-0.7 (pre-alpha-5)'
-
-import re
-import string
-
-def _namelist(instance):
-    namelist, namedict, classlist = [], {}, [instance.__class__]
-    for c in classlist:
-        for b in c.__bases__:
-            classlist.append(b)
-        for name in c.__dict__.keys():
-            if not namedict.has_key(name):
-                namelist.append(name)
-                namedict[name] = 1
-    return namelist
-
-class GenericScanner:
-    def __init__(self, flags=0):
-        pattern = self.reflect()
-        self.re = re.compile(pattern, re.VERBOSE|flags)
-
-        self.index2func = {}
-        for name, number in self.re.groupindex.items():
-            self.index2func[number-1] = getattr(self, 't_' + name)
-
-    def makeRE(self, name):
-        doc = getattr(self, name).__doc__
-        rv = '(?P<%s>%s)' % (name[2:], doc)
-        return rv
-
-    def reflect(self):
-        rv = []
-        for name in _namelist(self):
-            if name[:2] == 't_' and name != 't_default':
-                rv.append(self.makeRE(name))
-
-        rv.append(self.makeRE('t_default'))
-        return string.join(rv, '|')
-
-    def error(self, s, pos):
-        print "Lexical error at position %s" % pos
-        raise SystemExit
-
-    def tokenize(self, s):
-        pos = 0
-        n = len(s)
-        while pos < n:
-            m = self.re.match(s, pos)
-            if m is None:
-                self.error(s, pos)
-
-            groups = m.groups()
-            for i in range(len(groups)):
-                if groups[i] and self.index2func.has_key(i):
-                    self.index2func[i](groups[i])
-            pos = m.end()
-
-    def t_default(self, s):
-        r'( . | \n )+'
-        print "Specification error: unmatched input"
-        raise SystemExit
-
-#
-#  Extracted from GenericParser and made global so that [un]picking works.
-#
-class _State:
-    def __init__(self, stateno, items):
-        self.T, self.complete, self.items = [], [], items
-        self.stateno = stateno
-
-class GenericParser:
-    #
-    #  An Earley parser, as per J. Earley, "An Efficient Context-Free
-    #  Parsing Algorithm", CACM 13(2), pp. 94-102.  Also J. C. Earley,
-    #  "An Efficient Context-Free Parsing Algorithm", Ph.D. thesis,
-    #  Carnegie-Mellon University, August 1968.  New formulation of
-    #  the parser according to J. Aycock, "Practical Earley Parsing
-    #  and the SPARK Toolkit", Ph.D. thesis, University of Victoria,
-    #  2001, and J. Aycock and R. N. Horspool, "Practical Earley
-    #  Parsing", unpublished paper, 2001.
-    #
-
-    def __init__(self, start):
-        self.rules = {}
-        self.rule2func = {}
-        self.rule2name = {}
-        self.collectRules()
-        self.augment(start)
-        self.ruleschanged = 1
-
-    _NULLABLE = '\e_'
-    _START = 'START'
-    _BOF = '|-'
-
-    #
-    #  When pickling, take the time to generate the full state machine;
-    #  some information is then extraneous, too.  Unfortunately we
-    #  can't save the rule2func map.
-    #
-    def __getstate__(self):
-        if self.ruleschanged:
-            #
-            #  XXX - duplicated from parse()
-            #
-            self.computeNull()
-            self.newrules = {}
-            self.new2old = {}
-            self.makeNewRules()
-            self.ruleschanged = 0
-            self.edges, self.cores = {}, {}
-            self.states = { 0: self.makeState0() }
-            self.makeState(0, self._BOF)
-        #
-        #  XXX - should find a better way to do this..
-        #
-        changes = 1
-        while changes:
-            changes = 0
-            for k, v in self.edges.items():
-                if v is None:
-                    state, sym = k
-                    if self.states.has_key(state):
-                        self.goto(state, sym)
-                        changes = 1
-        rv = self.__dict__.copy()
-        for s in self.states.values():
-            del s.items
-        del rv['rule2func']
-        del rv['nullable']
-        del rv['cores']
-        return rv
-
-    def __setstate__(self, D):
-        self.rules = {}
-        self.rule2func = {}
-        self.rule2name = {}
-        self.collectRules()
-        start = D['rules'][self._START][0][1][1]        # Blech.
-        self.augment(start)
-        D['rule2func'] = self.rule2func
-        D['makeSet'] = self.makeSet_fast
-        self.__dict__ = D
-
-    #
-    #  A hook for GenericASTBuilder and GenericASTMatcher.  Mess
-    #  thee not with this; nor shall thee toucheth the _preprocess
-    #  argument to addRule.
-    #
-    def preprocess(self, rule, func):       return rule, func
-
-    def addRule(self, doc, func, _preprocess=1):
-        fn = func
-        rules = string.split(doc)
-
-        index = []
-        for i in range(len(rules)):
-            if rules[i] == '::=':
-                index.append(i-1)
-        index.append(len(rules))
-
-        for i in range(len(index)-1):
-            lhs = rules[index[i]]
-            rhs = rules[index[i]+2:index[i+1]]
-            rule = (lhs, tuple(rhs))
-
-            if _preprocess:
-                rule, fn = self.preprocess(rule, func)
-
-            if self.rules.has_key(lhs):
-                self.rules[lhs].append(rule)
-            else:
-                self.rules[lhs] = [ rule ]
-            self.rule2func[rule] = fn
-            self.rule2name[rule] = func.__name__[2:]
-        self.ruleschanged = 1
-
-    def collectRules(self):
-        for name in _namelist(self):
-            if name[:2] == 'p_':
-                func = getattr(self, name)
-                doc = func.__doc__
-                self.addRule(doc, func)
-
-    def augment(self, start):
-        rule = '%s ::= %s %s' % (self._START, self._BOF, start)
-        self.addRule(rule, lambda args: args[1], 0)
-
-    def computeNull(self):
-        self.nullable = {}
-        tbd = []
-
-        for rulelist in self.rules.values():
-            lhs = rulelist[0][0]
-            self.nullable[lhs] = 0
-            for rule in rulelist:
-                rhs = rule[1]
-                if len(rhs) == 0:
-                    self.nullable[lhs] = 1
-                    continue
-                #
-                #  We only need to consider rules which
-                #  consist entirely of nonterminal symbols.
-                #  This should be a savings on typical
-                #  grammars.
-                #
-                for sym in rhs:
-                    if not self.rules.has_key(sym):
-                        break
-                else:
-                    tbd.append(rule)
-        changes = 1
-        while changes:
-            changes = 0

From pypy.commits at gmail.com  Tue Jan 24 09:59:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 06:59:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: Add test file from CPython
 3.6.0, unmodified so far
Message-ID: <58876bb9.57bc1c0a.d809c.567d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89743:7d80e3347a86
Date: 2017-01-24 15:58 +0100
http://bitbucket.org/pypy/pypy/changeset/7d80e3347a86/

Log:	Add test file from CPython 3.6.0, unmodified so far

diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/test_fstring.py
@@ -0,0 +1,762 @@
+# This test file is from CPython 3.6.0
+
+import ast
+import types
+import decimal
+import unittest
+
+a_global = 'global variable'
+
+# You could argue that I'm too strict in looking for specific error
+#  values with assertRaisesRegex, but without it it's way too easy to
+#  make a syntax error in the test strings. Especially with all of the
+#  triple quotes, raw strings, backslashes, etc. I think it's a
+#  worthwhile tradeoff. When I switched to this method, I found many
+#  examples where I wasn't testing what I thought I was.
+
+class TestCase(unittest.TestCase):
+    def assertAllRaise(self, exception_type, regex, error_strings):
+        for str in error_strings:
+            with self.subTest(str=str):
+                with self.assertRaisesRegex(exception_type, regex):
+                    eval(str)
+
+    def test__format__lookup(self):
+        # Make sure __format__ is looked up on the type, not the instance.
+        class X:
+            def __format__(self, spec):
+                return 'class'
+
+        x = X()
+
+        # Add a bound __format__ method to the 'y' instance, but not
+        #  the 'x' instance.
+        y = X()
+        y.__format__ = types.MethodType(lambda self, spec: 'instance', y)
+
+        self.assertEqual(f'{y}', format(y))
+        self.assertEqual(f'{y}', 'class')
+        self.assertEqual(format(x), format(y))
+
+        # __format__ is not called this way, but still make sure it
+        #  returns what we expect (so we can make sure we're bypassing
+        #  it).
+        self.assertEqual(x.__format__(''), 'class')
+        self.assertEqual(y.__format__(''), 'instance')
+
+        # This is how __format__ is actually called.
+        self.assertEqual(type(x).__format__(x, ''), 'class')
+        self.assertEqual(type(y).__format__(y, ''), 'class')
+
+    def test_ast(self):
+        # Inspired by http://bugs.python.org/issue24975
+        class X:
+            def __init__(self):
+                self.called = False
+            def __call__(self):
+                self.called = True
+                return 4
+        x = X()
+        expr = """
+a = 10
+f'{a * x()}'"""
+        t = ast.parse(expr)
+        c = compile(t, '', 'exec')
+
+        # Make sure x was not called.
+        self.assertFalse(x.called)
+
+        # Actually run the code.
+        exec(c)
+
+        # Make sure x was called.
+        self.assertTrue(x.called)
+
+    def test_literal_eval(self):
+        # With no expressions, an f-string is okay.
+        self.assertEqual(ast.literal_eval("f'x'"), 'x')
+        self.assertEqual(ast.literal_eval("f'x' 'y'"), 'xy')
+
+        # But this should raise an error.
+        with self.assertRaisesRegex(ValueError, 'malformed node or string'):
+            ast.literal_eval("f'x{3}'")
+
+        # As should this, which uses a different ast node
+        with self.assertRaisesRegex(ValueError, 'malformed node or string'):
+            ast.literal_eval("f'{3}'")
+
+    def test_ast_compile_time_concat(self):
+        x = ['']
+
+        expr = """x[0] = 'foo' f'{3}'"""
+        t = ast.parse(expr)
+        c = compile(t, '', 'exec')
+        exec(c)
+        self.assertEqual(x[0], 'foo3')
+
+    def test_compile_time_concat_errors(self):
+        self.assertAllRaise(SyntaxError,
+                            'cannot mix bytes and nonbytes literals',
+                            [r"""f'' b''""",
+                             r"""b'' f''""",
+                             ])
+
+    def test_literal(self):
+        self.assertEqual(f'', '')
+        self.assertEqual(f'a', 'a')
+        self.assertEqual(f' ', ' ')
+
+    def test_unterminated_string(self):
+        self.assertAllRaise(SyntaxError, 'f-string: unterminated string',
+                            [r"""f'{"x'""",
+                             r"""f'{"x}'""",
+                             r"""f'{("x'""",
+                             r"""f'{("x}'""",
+                             ])
+
+    def test_mismatched_parens(self):
+        self.assertAllRaise(SyntaxError, 'f-string: mismatched',
+                            ["f'{((}'",
+                             ])
+
+    def test_double_braces(self):
+        self.assertEqual(f'{{', '{')
+        self.assertEqual(f'a{{', 'a{')
+        self.assertEqual(f'{{b', '{b')
+        self.assertEqual(f'a{{b', 'a{b')
+        self.assertEqual(f'}}', '}')
+        self.assertEqual(f'a}}', 'a}')
+        self.assertEqual(f'}}b', '}b')
+        self.assertEqual(f'a}}b', 'a}b')
+        self.assertEqual(f'{{}}', '{}')
+        self.assertEqual(f'a{{}}', 'a{}')
+        self.assertEqual(f'{{b}}', '{b}')
+        self.assertEqual(f'{{}}c', '{}c')
+        self.assertEqual(f'a{{b}}', 'a{b}')
+        self.assertEqual(f'a{{}}c', 'a{}c')
+        self.assertEqual(f'{{b}}c', '{b}c')
+        self.assertEqual(f'a{{b}}c', 'a{b}c')
+
+        self.assertEqual(f'{{{10}', '{10')
+        self.assertEqual(f'}}{10}', '}10')
+        self.assertEqual(f'}}{{{10}', '}{10')
+        self.assertEqual(f'}}a{{{10}', '}a{10')
+
+        self.assertEqual(f'{10}{{', '10{')
+        self.assertEqual(f'{10}}}', '10}')
+        self.assertEqual(f'{10}}}{{', '10}{')
+        self.assertEqual(f'{10}}}a{{' '}', '10}a{}')
+
+        # Inside of strings, don't interpret doubled brackets.
+        self.assertEqual(f'{"{{}}"}', '{{}}')
+
+        self.assertAllRaise(TypeError, 'unhashable type',
+                            ["f'{ {{}} }'", # dict in a set
+                             ])
+
+    def test_compile_time_concat(self):
+        x = 'def'
+        self.assertEqual('abc' f'## {x}ghi', 'abc## defghi')
+        self.assertEqual('abc' f'{x}' 'ghi', 'abcdefghi')
+        self.assertEqual('abc' f'{x}' 'gh' f'i{x:4}', 'abcdefghidef ')
+        self.assertEqual('{x}' f'{x}', '{x}def')
+        self.assertEqual('{x' f'{x}', '{xdef')
+        self.assertEqual('{x}' f'{x}', '{x}def')
+        self.assertEqual('{{x}}' f'{x}', '{{x}}def')
+        self.assertEqual('{{x' f'{x}', '{{xdef')
+        self.assertEqual('x}}' f'{x}', 'x}}def')
+        self.assertEqual(f'{x}' 'x}}', 'defx}}')
+        self.assertEqual(f'{x}' '', 'def')
+        self.assertEqual('' f'{x}' '', 'def')
+        self.assertEqual('' f'{x}', 'def')
+        self.assertEqual(f'{x}' '2', 'def2')
+        self.assertEqual('1' f'{x}' '2', '1def2')
+        self.assertEqual('1' f'{x}', '1def')
+        self.assertEqual(f'{x}' f'-{x}', 'def-def')
+        self.assertEqual('' f'', '')
+        self.assertEqual('' f'' '', '')
+        self.assertEqual('' f'' '' f'', '')
+        self.assertEqual(f'', '')
+        self.assertEqual(f'' '', '')
+        self.assertEqual(f'' '' f'', '')
+        self.assertEqual(f'' '' f'' '', '')
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{3' f'}'",  # can't concat to get a valid f-string
+                             ])
+
+    def test_comments(self):
+        # These aren't comments, since they're in strings.
+        d = {'#': 'hash'}
+        self.assertEqual(f'{"#"}', '#')
+        self.assertEqual(f'{d["#"]}', 'hash')
+
+        self.assertAllRaise(SyntaxError, "f-string expression part cannot include '#'",
+                            ["f'{1#}'",   # error because the expression becomes "(1#)"
+                             "f'{3(#)}'",
+                             "f'{#}'",
+                             "f'{)#}'",   # When wrapped in parens, this becomes
+                                          #  '()#)'.  Make sure that doesn't compile.
+                             ])
+
+    def test_many_expressions(self):
+        # Create a string with many expressions in it. Note that
+        #  because we have a space in here as a literal, we're actually
+        #  going to use twice as many ast nodes: one for each literal
+        #  plus one for each expression.
+        def build_fstr(n, extra=''):
+            return "f'" + ('{x} ' * n) + extra + "'"
+
+        x = 'X'
+        width = 1
+
+        # Test around 256.
+        for i in range(250, 260):
+            self.assertEqual(eval(build_fstr(i)), (x+' ')*i)
+
+        # Test concatenating 2 largs fstrings.
+        self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256))
+
+        s = build_fstr(253, '{x:{width}} ')
+        self.assertEqual(eval(s), (x+' ')*254)
+
+        # Test lots of expressions and constants, concatenated.
+        s = "f'{1}' 'x' 'y'" * 1024
+        self.assertEqual(eval(s), '1xy' * 1024)
+
+    def test_format_specifier_expressions(self):
+        width = 10
+        precision = 4
+        value = decimal.Decimal('12.34567')
+        self.assertEqual(f'result: {value:{width}.{precision}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result:      12.35')
+        self.assertEqual(f'{10:#{1}0x}', '       0xa')
+        self.assertEqual(f'{10:{"#"}1{0}{"x"}}', '       0xa')
+        self.assertEqual(f'{-10:-{"#"}1{0}x}', '      -0xa')
+        self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', '      -0xa')
+        self.assertEqual(f'{10:#{3 != {4:5} and width}x}', '       0xa')
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["""f'{"s"!r{":10"}}'""",
+
+                             # This looks like a nested format spec.
+                             ])
+
+        self.assertAllRaise(SyntaxError, "invalid syntax",
+                            [# Invalid syntax inside a nested spec.
+                             "f'{4:{/5}}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply",
+                            [# Can't nest format specifiers.
+                             "f'result: {value:{width:{0}}.{precision:1}}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character',
+                            [# No expansion inside conversion or for
+                             #  the : or ! itself.
+                             """f'{"s"!{"r"}}'""",
+                             ])
+
+    def test_side_effect_order(self):
+        class X:
+            def __init__(self):
+                self.i = 0
+            def __format__(self, spec):
+                self.i += 1
+                return str(self.i)
+
+        x = X()
+        self.assertEqual(f'{x} {x}', '1 2')
+
+    def test_missing_expression(self):
+        self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed',
+                            ["f'{}'",
+                             "f'{ }'"
+                             "f' {} '",
+                             "f'{!r}'",
+                             "f'{ !r}'",
+                             "f'{10:{ }}'",
+                             "f' { } '",
+
+                             # Catch the empty expression before the
+                             #  invalid conversion.
+                             "f'{!x}'",
+                             "f'{ !xr}'",
+                             "f'{!x:}'",
+                             "f'{!x:a}'",
+                             "f'{ !xr:}'",
+                             "f'{ !xr:a}'",
+
+                             "f'{!}'",
+                             "f'{:}'",
+
+                             # We find the empty expression before the
+                             #  missing closing brace.
+                             "f'{!'",
+                             "f'{!s:'",
+                             "f'{:'",
+                             "f'{:x'",
+                             ])
+
+    def test_parens_in_expressions(self):
+        self.assertEqual(f'{3,}', '(3,)')
+
+        # Add these because when an expression is evaluated, parens
+        #  are added around it. But we shouldn't go from an invalid
+        #  expression to a valid one. The added parens are just
+        #  supposed to allow whitespace (including newlines).
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["f'{,}'",
+                             "f'{,}'",  # this is (,), which is an error
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{3)+(4}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, 'EOL while scanning string literal',
+                            ["f'{\n}'",
+                             ])
+
+    def test_backslashes_in_string_part(self):
+        self.assertEqual(f'\t', '\t')
+        self.assertEqual(r'\t', '\\t')
+        self.assertEqual(rf'\t', '\\t')
+        self.assertEqual(f'{2}\t', '2\t')
+        self.assertEqual(f'{2}\t{3}', '2\t3')
+        self.assertEqual(f'\t{3}', '\t3')
+
+        self.assertEqual(f'\u0394', '\u0394')
+        self.assertEqual(r'\u0394', '\\u0394')
+        self.assertEqual(rf'\u0394', '\\u0394')
+        self.assertEqual(f'{2}\u0394', '2\u0394')
+        self.assertEqual(f'{2}\u0394{3}', '2\u03943')
+        self.assertEqual(f'\u0394{3}', '\u03943')
+
+        self.assertEqual(f'\U00000394', '\u0394')
+        self.assertEqual(r'\U00000394', '\\U00000394')
+        self.assertEqual(rf'\U00000394', '\\U00000394')
+        self.assertEqual(f'{2}\U00000394', '2\u0394')
+        self.assertEqual(f'{2}\U00000394{3}', '2\u03943')
+        self.assertEqual(f'\U00000394{3}', '\u03943')
+
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', '\u0394')
+        self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}', '2\u0394')
+        self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}{3}', '2\u03943')
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}{3}', '\u03943')
+        self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}', '2\u0394')
+        self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}3', '2\u03943')
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}3', '\u03943')
+
+        self.assertEqual(f'\x20', ' ')
+        self.assertEqual(r'\x20', '\\x20')
+        self.assertEqual(rf'\x20', '\\x20')
+        self.assertEqual(f'{2}\x20', '2 ')
+        self.assertEqual(f'{2}\x20{3}', '2 3')
+        self.assertEqual(f'\x20{3}', ' 3')
+
+        self.assertEqual(f'2\x20', '2 ')
+        self.assertEqual(f'2\x203', '2 3')
+        self.assertEqual(f'\x203', ' 3')
+
+    def test_misformed_unicode_character_name(self):
+        # These test are needed because unicode names are parsed
+        # differently inside f-strings.
+        self.assertAllRaise(SyntaxError, r"\(unicode error\) 'unicodeescape' codec can't decode bytes in position .*: malformed \\N character escape",
+                            [r"f'\N'",
+                             r"f'\N{'",
+                             r"f'\N{GREEK CAPITAL LETTER DELTA'",
+
+                             # Here are the non-f-string versions,
+                             #  which should give the same errors.
+                             r"'\N'",
+                             r"'\N{'",
+                             r"'\N{GREEK CAPITAL LETTER DELTA'",
+                             ])
+
+    def test_no_backslashes_in_expression_part(self):
+        self.assertAllRaise(SyntaxError, 'f-string expression part cannot include a backslash',
+                            [r"f'{\'a\'}'",
+                             r"f'{\t3}'",
+                             r"f'{\}'",
+                             r"rf'{\'a\'}'",
+                             r"rf'{\t3}'",
+                             r"rf'{\}'",
+                             r"""rf'{"\N{LEFT CURLY BRACKET}"}'""",
+                             r"f'{\n}'",
+                             ])
+
+    def test_no_escapes_for_braces(self):
+        """
+        Only literal curly braces begin an expression.
+        """
+        # \x7b is '{'.
+        self.assertEqual(f'\x7b1+1}}', '{1+1}')
+        self.assertEqual(f'\x7b1+1', '{1+1')
+        self.assertEqual(f'\u007b1+1', '{1+1')
+        self.assertEqual(f'\N{LEFT CURLY BRACKET}1+1\N{RIGHT CURLY BRACKET}', '{1+1}')
+
+    def test_newlines_in_expressions(self):
+        self.assertEqual(f'{0}', '0')
+        self.assertEqual(rf'''{3+
+4}''', '7')
+
+    def test_lambda(self):
+        x = 5
+        self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'")
+        self.assertEqual(f'{(lambda y:x*y)("8")!r:10}', "'88888'   ")
+        self.assertEqual(f'{(lambda y:x*y)("8"):10}', "88888     ")
+
+        # lambda doesn't work without parens, because the colon
+        #  makes the parser think it's a format_spec
+        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+                            ["f'{lambda x:x}'",
+                             ])
+
+    def test_yield(self):
+        # Not terribly useful, but make sure the yield turns
+        #  a function into a generator
+        def fn(y):
+            f'y:{yield y*2}'
+
+        g = fn(4)
+        self.assertEqual(next(g), 8)
+
+    def test_yield_send(self):
+        def fn(x):
+            yield f'x:{yield (lambda i: x * i)}'
+
+        g = fn(10)
+        the_lambda = next(g)
+        self.assertEqual(the_lambda(4), 40)
+        self.assertEqual(g.send('string'), 'x:string')
+
+    def test_expressions_with_triple_quoted_strings(self):
+        self.assertEqual(f"{'''x'''}", 'x')
+        self.assertEqual(f"{'''eric's'''}", "eric's")
+
+        # Test concatenation within an expression
+        self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy')
+        self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s')
+        self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy')
+        self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy')
+        self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy')
+        self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy')
+
+    def test_multiple_vars(self):
+        x = 98
+        y = 'abc'
+        self.assertEqual(f'{x}{y}', '98abc')
+
+        self.assertEqual(f'X{x}{y}', 'X98abc')
+        self.assertEqual(f'{x}X{y}', '98Xabc')
+        self.assertEqual(f'{x}{y}X', '98abcX')
+
+        self.assertEqual(f'X{x}Y{y}', 'X98Yabc')
+        self.assertEqual(f'X{x}{y}Y', 'X98abcY')
+        self.assertEqual(f'{x}X{y}Y', '98XabcY')
+
+        self.assertEqual(f'X{x}Y{y}Z', 'X98YabcZ')
+
+    def test_closure(self):
+        def outer(x):
+            def inner():
+                return f'x:{x}'
+            return inner
+
+        self.assertEqual(outer('987')(), 'x:987')
+        self.assertEqual(outer(7)(), 'x:7')
+
+    def test_arguments(self):
+        y = 2
+        def f(x, width):
+            return f'x={x*y:{width}}'
+
+        self.assertEqual(f('foo', 10), 'x=foofoo    ')
+        x = 'bar'
+        self.assertEqual(f(10, 10), 'x=        20')
+
+    def test_locals(self):
+        value = 123
+        self.assertEqual(f'v:{value}', 'v:123')
+
+    def test_missing_variable(self):
+        with self.assertRaises(NameError):
+            f'v:{value}'
+
+    def test_missing_format_spec(self):
+        class O:
+            def __format__(self, spec):
+                if not spec:
+                    return '*'
+                return spec
+
+        self.assertEqual(f'{O():x}', 'x')
+        self.assertEqual(f'{O()}', '*')
+        self.assertEqual(f'{O():}', '*')
+
+        self.assertEqual(f'{3:}', '3')
+        self.assertEqual(f'{3!s:}', '3')
+
+    def test_global(self):
+        self.assertEqual(f'g:{a_global}', 'g:global variable')
+        self.assertEqual(f'g:{a_global!r}', "g:'global variable'")
+
+        a_local = 'local variable'
+        self.assertEqual(f'g:{a_global} l:{a_local}',
+                         'g:global variable l:local variable')
+        self.assertEqual(f'g:{a_global!r}',
+                         "g:'global variable'")
+        self.assertEqual(f'g:{a_global} l:{a_local!r}',
+                         "g:global variable l:'local variable'")
+
+        self.assertIn("module 'unittest' from", f'{unittest}')
+
+    def test_shadowed_global(self):
+        a_global = 'really a local'
+        self.assertEqual(f'g:{a_global}', 'g:really a local')
+        self.assertEqual(f'g:{a_global!r}', "g:'really a local'")
+
+        a_local = 'local variable'
+        self.assertEqual(f'g:{a_global} l:{a_local}',
+                         'g:really a local l:local variable')
+        self.assertEqual(f'g:{a_global!r}',
+                         "g:'really a local'")
+        self.assertEqual(f'g:{a_global} l:{a_local!r}',
+                         "g:really a local l:'local variable'")
+
+    def test_call(self):
+        def foo(x):
+            return 'x=' + str(x)
+
+        self.assertEqual(f'{foo(10)}', 'x=10')
+
+    def test_nested_fstrings(self):
+        y = 5
+        self.assertEqual(f'{f"{0}"*3}', '000')
+        self.assertEqual(f'{f"{y}"*3}', '555')
+
+    def test_invalid_string_prefixes(self):
+        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+                            ["fu''",
+                             "uf''",
+                             "Fu''",
+                             "fU''",
+                             "Uf''",
+                             "uF''",
+                             "ufr''",
+                             "urf''",
+                             "fur''",
+                             "fru''",
+                             "rfu''",
+                             "ruf''",
+                             "FUR''",
+                             "Fur''",
+                             "fb''",
+                             "fB''",
+                             "Fb''",
+                             "FB''",
+                             "bf''",
+                             "bF''",
+                             "Bf''",
+                             "BF''",
+                             ])
+
+    def test_leading_trailing_spaces(self):
+        self.assertEqual(f'{ 3}', '3')
+        self.assertEqual(f'{  3}', '3')
+        self.assertEqual(f'{3 }', '3')
+        self.assertEqual(f'{3  }', '3')
+
+        self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}',
+                         'expr={1: 2}')
+        self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }',
+                         'expr={1: 2}')
+
+    def test_not_equal(self):
+        # There's a special test for this because there's a special
+        #  case in the f-string parser to look for != as not ending an
+        #  expression. Normally it would, while looking for !s or !r.
+
+        self.assertEqual(f'{3!=4}', 'True')
+        self.assertEqual(f'{3!=4:}', 'True')
+        self.assertEqual(f'{3!=4!s}', 'True')
+        self.assertEqual(f'{3!=4!s:.3}', 'Tru')
+
+    def test_conversions(self):
+        self.assertEqual(f'{3.14:10.10}', '      3.14')
+        self.assertEqual(f'{3.14!s:10.10}', '3.14      ')
+        self.assertEqual(f'{3.14!r:10.10}', '3.14      ')
+        self.assertEqual(f'{3.14!a:10.10}', '3.14      ')
+
+        self.assertEqual(f'{"a"}', 'a')
+        self.assertEqual(f'{"a"!r}', "'a'")
+        self.assertEqual(f'{"a"!a}', "'a'")
+
+        # Not a conversion.
+        self.assertEqual(f'{"a!r"}', "a!r")
+
+        # Not a conversion, but show that ! is allowed in a format spec.
+        self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!')
+
+        self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character',
+                            ["f'{3!g}'",
+                             "f'{3!A}'",
+                             "f'{3!3}'",
+                             "f'{3!G}'",
+                             "f'{3!!}'",
+                             "f'{3!:}'",
+                             "f'{3! s}'",  # no space before conversion char
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{x!s{y}}'",
+                             "f'{3!ss}'",
+                             "f'{3!ss:}'",
+                             "f'{3!ss:s}'",
+                             ])
+
+    def test_assignment(self):
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["f'' = 3",
+                             "f'{0}' = x",
+                             "f'{x}' = x",
+                             ])
+
+    def test_del(self):
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["del f''",
+                             "del '' f''",
+                             ])
+
+    def test_mismatched_braces(self):
+        self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed",
+                            ["f'{{}'",
+                             "f'{{}}}'",
+                             "f'}'",
+                             "f'x}'",
+                             "f'x}x'",
+                             r"f'\u007b}'",
+
+                             # Can't have { or } in a format spec.
+                             "f'{3:}>10}'",
+                             "f'{3:}}>10}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{3:{{>10}'",
+                             "f'{3'",
+                             "f'{3!'",
+                             "f'{3:'",
+                             "f'{3!s'",
+                             "f'{3!s:'",
+                             "f'{3!s:3'",
+                             "f'x{'",
+                             "f'x{x'",
+                             "f'{x'",
+                             "f'{3:s'",
+                             "f'{{{'",
+                             "f'{{}}{'",
+                             "f'{'",
+                             ])
+
+        # But these are just normal strings.
+        self.assertEqual(f'{"{"}', '{')
+        self.assertEqual(f'{"}"}', '}')
+        self.assertEqual(f'{3:{"}"}>10}', '}}}}}}}}}3')
+        self.assertEqual(f'{2:{"{"}>10}', '{{{{{{{{{2')
+
+    def test_if_conditional(self):
+        # There's special logic in compile.c to test if the
+        #  conditional for an if (and while) are constants. Exercise
+        #  that code.
+
+        def test_fstring(x, expected):
+            flag = 0
+            if f'{x}':
+                flag = 1
+            else:
+                flag = 2
+            self.assertEqual(flag, expected)
+
+        def test_concat_empty(x, expected):
+            flag = 0
+            if '' f'{x}':
+                flag = 1
+            else:
+                flag = 2
+            self.assertEqual(flag, expected)
+
+        def test_concat_non_empty(x, expected):
+            flag = 0
+            if ' ' f'{x}':
+                flag = 1
+            else:
+                flag = 2
+            self.assertEqual(flag, expected)
+
+        test_fstring('', 2)
+        test_fstring(' ', 1)
+
+        test_concat_empty('', 2)
+        test_concat_empty(' ', 1)
+
+        test_concat_non_empty('', 1)
+        test_concat_non_empty(' ', 1)
+
+    def test_empty_format_specifier(self):
+        x = 'test'
+        self.assertEqual(f'{x}', 'test')
+        self.assertEqual(f'{x:}', 'test')
+        self.assertEqual(f'{x!s:}', 'test')
+        self.assertEqual(f'{x!r:}', "'test'")
+
+    def test_str_format_differences(self):
+        d = {'a': 'string',
+             0: 'integer',
+             }
+        a = 0
+        self.assertEqual(f'{d[0]}', 'integer')
+        self.assertEqual(f'{d["a"]}', 'string')
+        self.assertEqual(f'{d[a]}', 'integer')
+        self.assertEqual('{d[a]}'.format(d=d), 'string')
+        self.assertEqual('{d[0]}'.format(d=d), 'integer')
+
+    def test_invalid_expressions(self):
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            [r"f'{a[4)}'",
+                             r"f'{a(4]}'",
+                            ])
+
+    def test_errors(self):
+        # see issue 26287
+        self.assertAllRaise(TypeError, 'unsupported',
+                            [r"f'{(lambda: 0):x}'",
+                             r"f'{(0,):x}'",
+                             ])
+        self.assertAllRaise(ValueError, 'Unknown format code',
+                            [r"f'{1000:j}'",
+                             r"f'{1000:j}'",
+                            ])
+
+    def test_loop(self):
+        for i in range(1000):
+            self.assertEqual(f'i:{i}', 'i:' + str(i))
+
+    def test_dict(self):
+        d = {'"': 'dquote',
+             "'": 'squote',
+             'foo': 'bar',
+             }
+        self.assertEqual(f'''{d["'"]}''', 'squote')
+        self.assertEqual(f"""{d['"']}""", 'dquote')
+
+        self.assertEqual(f'{d["foo"]}', 'bar')
+        self.assertEqual(f"{d['foo']}", 'bar')
+
+if __name__ == '__main__':
+    unittest.main()

From pypy.commits at gmail.com  Tue Jan 24 11:11:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 08:11:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: translation fix
Message-ID: <58877c94.0d1a1c0a.7ab00.30f3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89745:207d98a326e0
Date: 2017-01-24 17:09 +0100
http://bitbucket.org/pypy/pypy/changeset/207d98a326e0/

Log:	translation fix

diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -352,7 +352,7 @@
         # Unicode/ValueError in literal: turn into SyntaxError
         e.normalize_exception(space)
         errmsg = space.str_w(space.str(e.get_w_value(space)))
-        raise self.error('(%s) %s' % (kind, errmsg), atom_node)
+        raise astbuilder.error('(%s) %s' % (kind, errmsg), atom_node)
 
     if len(joined_pieces) == 1:   # <= the common path
         return joined_pieces[0]   # ast.Str, Bytes or FormattedValue

From pypy.commits at gmail.com  Tue Jan 24 11:11:02 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 08:11:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: Can now translate a pypy
 with --no-objspace-fstrings. Might be useful
Message-ID: <58877c96.0e821c0a.5ffe5.2a44@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89746:b8a7efb2f114
Date: 2017-01-24 17:10 +0100
http://bitbucket.org/pypy/pypy/changeset/b8a7efb2f114/

Log:	Can now translate a pypy with --no-objspace-fstrings. Might be
	useful if you're absolutely convinced f-strings pose a danger or for
	some other reason don't want them in your PyPy 3.5.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -196,6 +196,11 @@
                default=False,
                requires=[("objspace.usemodules.cpyext", False)]),
 
+    BoolOption("fstrings",
+               "if you are really convinced that f-strings are a security "
+               "issue, you can disable them here",
+               default=True),
+
     OptionDescription("std", "Standard Object Space Options", [
         BoolOption("withtproxy", "support transparent proxies",
                    default=True),
diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -292,6 +292,16 @@
     # could be merged into a single function with a clearer logic.  It's
     # done this way to follow CPython's source code more closely.
 
+    space = astbuilder.space
+    if not space.config.objspace.fstrings:
+        raise oefmt(space.w_SystemError,
+                    "f-strings have been disabled in this version of pypy "
+                    "with the translation option --no-objspace-fstrings.  "
+                    "The PyPy team (and CPython) thinks f-strings don't "
+                    "add any security risks, but we leave it to you to "
+                    "convince whoever translated this pypy that it is "
+                    "really the case")
+
     while True:
         literal, expr = fstring_find_literal_and_expr(astbuilder, fstr,
                                                       atom_node, rec)

From pypy.commits at gmail.com  Tue Jan 24 11:10:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 08:10:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: regenerate this file
Message-ID: <58877c92.958c1c0a.222df.2376@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89744:8bfc9e97b86b
Date: 2017-01-24 17:09 +0100
http://bitbucket.org/pypy/pypy/changeset/8bfc9e97b86b/

Log:	regenerate this file

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2704,7 +2704,9 @@
 
     def mutate_over(self, visitor):
         if self.values:
-            visitor._mutate_sequence(self.values)
+            for i in range(len(self.values)):
+                if self.values[i] is not None:
+                    self.values[i] = self.values[i].mutate_over(visitor)
         return visitor.visit_JoinedStr(self)
 
     def to_object(self, space):

From pypy.commits at gmail.com  Tue Jan 24 11:31:23 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 24 Jan 2017 08:31:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add #defines for the PyMem_Raw family of
 functions
Message-ID: <5887815b.6f98df0a.d4ac7.d219@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89747:ecc2c1ad70a9
Date: 2017-01-24 16:29 +0000
http://bitbucket.org/pypy/pypy/changeset/ecc2c1ad70a9/

Log:	Add #defines for the PyMem_Raw family of functions

diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -15,6 +15,10 @@
 #define PyMem_Free  PyMem_FREE
 #define PyMem_Realloc  PyMem_REALLOC
 
+#define PyMem_RawMalloc PyMem_Malloc
+#define PyMem_RawFree PyMem_Free
+#define PyMem_RawRealloc PyMem_Realloc
+
 /*
  * Type-oriented memory interface
  * ==============================

From pypy.commits at gmail.com  Tue Jan 24 11:32:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 08:32:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add f-strings to PyPy 3.5. In CPython
 they are only there from 3.6, but
Message-ID: <5887819f.52a5df0a.b21bc.d265@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89749:a19390150925
Date: 2017-01-24 17:31 +0100
http://bitbucket.org/pypy/pypy/changeset/a19390150925/

Log:	Add f-strings to PyPy 3.5. In CPython they are only there from 3.6,
	but the idea is that if it is the only new feature that you really
	need to run your Python 3.x programs, then using PyPy 3.5 should
	work too.

	For people that, for some reason, think f-strings are a security
	issue and would like to disable them, translate with --no-objspace-
	fstrings.

diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -214,6 +214,9 @@
 def_op('BUILD_TUPLE_UNPACK', 152)
 def_op('BUILD_SET_UNPACK', 153)
 
+def_op('FORMAT_VALUE', 155)   # in CPython 3.6, but available in PyPy from 3.5
+def_op('BUILD_STRING', 157)   # in CPython 3.6, but available in PyPy from 3.5
+
 # pypy modification, experimental bytecode
 def_op('LOOKUP_METHOD', 201)          # Index in name list
 hasname.append(201)
diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/test_fstring.py
@@ -0,0 +1,762 @@
+# This test file is from CPython 3.6.0
+
+import ast
+import types
+import decimal
+import unittest
+
+a_global = 'global variable'
+
+# You could argue that I'm too strict in looking for specific error
+#  values with assertRaisesRegex, but without it it's way too easy to
+#  make a syntax error in the test strings. Especially with all of the
+#  triple quotes, raw strings, backslashes, etc. I think it's a
+#  worthwhile tradeoff. When I switched to this method, I found many
+#  examples where I wasn't testing what I thought I was.
+
+class TestCase(unittest.TestCase):
+    def assertAllRaise(self, exception_type, regex, error_strings):
+        for str in error_strings:
+            with self.subTest(str=str):
+                with self.assertRaisesRegex(exception_type, regex):
+                    eval(str)
+
+    def test__format__lookup(self):
+        # Make sure __format__ is looked up on the type, not the instance.
+        class X:
+            def __format__(self, spec):
+                return 'class'
+
+        x = X()
+
+        # Add a bound __format__ method to the 'y' instance, but not
+        #  the 'x' instance.
+        y = X()
+        y.__format__ = types.MethodType(lambda self, spec: 'instance', y)
+
+        self.assertEqual(f'{y}', format(y))
+        self.assertEqual(f'{y}', 'class')
+        self.assertEqual(format(x), format(y))
+
+        # __format__ is not called this way, but still make sure it
+        #  returns what we expect (so we can make sure we're bypassing
+        #  it).
+        self.assertEqual(x.__format__(''), 'class')
+        self.assertEqual(y.__format__(''), 'instance')
+
+        # This is how __format__ is actually called.
+        self.assertEqual(type(x).__format__(x, ''), 'class')
+        self.assertEqual(type(y).__format__(y, ''), 'class')
+
+    def test_ast(self):
+        # Inspired by http://bugs.python.org/issue24975
+        class X:
+            def __init__(self):
+                self.called = False
+            def __call__(self):
+                self.called = True
+                return 4
+        x = X()
+        expr = """
+a = 10
+f'{a * x()}'"""
+        t = ast.parse(expr)
+        c = compile(t, '', 'exec')
+
+        # Make sure x was not called.
+        self.assertFalse(x.called)
+
+        # Actually run the code.
+        exec(c)
+
+        # Make sure x was called.
+        self.assertTrue(x.called)
+
+    def test_literal_eval(self):
+        # With no expressions, an f-string is okay.
+        self.assertEqual(ast.literal_eval("f'x'"), 'x')
+        self.assertEqual(ast.literal_eval("f'x' 'y'"), 'xy')
+
+        # But this should raise an error.
+        with self.assertRaisesRegex(ValueError, 'malformed node or string'):
+            ast.literal_eval("f'x{3}'")
+
+        # As should this, which uses a different ast node
+        with self.assertRaisesRegex(ValueError, 'malformed node or string'):
+            ast.literal_eval("f'{3}'")
+
+    def test_ast_compile_time_concat(self):
+        x = ['']
+
+        expr = """x[0] = 'foo' f'{3}'"""
+        t = ast.parse(expr)
+        c = compile(t, '', 'exec')
+        exec(c)
+        self.assertEqual(x[0], 'foo3')
+
+    def test_compile_time_concat_errors(self):
+        self.assertAllRaise(SyntaxError,
+                            'cannot mix bytes and nonbytes literals',
+                            [r"""f'' b''""",
+                             r"""b'' f''""",
+                             ])
+
+    def test_literal(self):
+        self.assertEqual(f'', '')
+        self.assertEqual(f'a', 'a')
+        self.assertEqual(f' ', ' ')
+
+    def test_unterminated_string(self):
+        self.assertAllRaise(SyntaxError, 'f-string: unterminated string',
+                            [r"""f'{"x'""",
+                             r"""f'{"x}'""",
+                             r"""f'{("x'""",
+                             r"""f'{("x}'""",
+                             ])
+
+    def test_mismatched_parens(self):
+        self.assertAllRaise(SyntaxError, 'f-string: mismatched',
+                            ["f'{((}'",
+                             ])
+
+    def test_double_braces(self):
+        self.assertEqual(f'{{', '{')
+        self.assertEqual(f'a{{', 'a{')
+        self.assertEqual(f'{{b', '{b')
+        self.assertEqual(f'a{{b', 'a{b')
+        self.assertEqual(f'}}', '}')
+        self.assertEqual(f'a}}', 'a}')
+        self.assertEqual(f'}}b', '}b')
+        self.assertEqual(f'a}}b', 'a}b')
+        self.assertEqual(f'{{}}', '{}')
+        self.assertEqual(f'a{{}}', 'a{}')
+        self.assertEqual(f'{{b}}', '{b}')
+        self.assertEqual(f'{{}}c', '{}c')
+        self.assertEqual(f'a{{b}}', 'a{b}')
+        self.assertEqual(f'a{{}}c', 'a{}c')
+        self.assertEqual(f'{{b}}c', '{b}c')
+        self.assertEqual(f'a{{b}}c', 'a{b}c')
+
+        self.assertEqual(f'{{{10}', '{10')
+        self.assertEqual(f'}}{10}', '}10')
+        self.assertEqual(f'}}{{{10}', '}{10')
+        self.assertEqual(f'}}a{{{10}', '}a{10')
+
+        self.assertEqual(f'{10}{{', '10{')
+        self.assertEqual(f'{10}}}', '10}')
+        self.assertEqual(f'{10}}}{{', '10}{')
+        self.assertEqual(f'{10}}}a{{' '}', '10}a{}')
+
+        # Inside of strings, don't interpret doubled brackets.
+        self.assertEqual(f'{"{{}}"}', '{{}}')
+
+        self.assertAllRaise(TypeError, 'unhashable type',
+                            ["f'{ {{}} }'", # dict in a set
+                             ])
+
+    def test_compile_time_concat(self):
+        x = 'def'
+        self.assertEqual('abc' f'## {x}ghi', 'abc## defghi')
+        self.assertEqual('abc' f'{x}' 'ghi', 'abcdefghi')
+        self.assertEqual('abc' f'{x}' 'gh' f'i{x:4}', 'abcdefghidef ')
+        self.assertEqual('{x}' f'{x}', '{x}def')
+        self.assertEqual('{x' f'{x}', '{xdef')
+        self.assertEqual('{x}' f'{x}', '{x}def')
+        self.assertEqual('{{x}}' f'{x}', '{{x}}def')
+        self.assertEqual('{{x' f'{x}', '{{xdef')
+        self.assertEqual('x}}' f'{x}', 'x}}def')
+        self.assertEqual(f'{x}' 'x}}', 'defx}}')
+        self.assertEqual(f'{x}' '', 'def')
+        self.assertEqual('' f'{x}' '', 'def')
+        self.assertEqual('' f'{x}', 'def')
+        self.assertEqual(f'{x}' '2', 'def2')
+        self.assertEqual('1' f'{x}' '2', '1def2')
+        self.assertEqual('1' f'{x}', '1def')
+        self.assertEqual(f'{x}' f'-{x}', 'def-def')
+        self.assertEqual('' f'', '')
+        self.assertEqual('' f'' '', '')
+        self.assertEqual('' f'' '' f'', '')
+        self.assertEqual(f'', '')
+        self.assertEqual(f'' '', '')
+        self.assertEqual(f'' '' f'', '')
+        self.assertEqual(f'' '' f'' '', '')
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{3' f'}'",  # can't concat to get a valid f-string
+                             ])
+
+    def test_comments(self):
+        # These aren't comments, since they're in strings.
+        d = {'#': 'hash'}
+        self.assertEqual(f'{"#"}', '#')
+        self.assertEqual(f'{d["#"]}', 'hash')
+
+        self.assertAllRaise(SyntaxError, "f-string expression part cannot include '#'",
+                            ["f'{1#}'",   # error because the expression becomes "(1#)"
+                             "f'{3(#)}'",
+                             "f'{#}'",
+                             "f'{)#}'",   # When wrapped in parens, this becomes
+                                          #  '()#)'.  Make sure that doesn't compile.
+                             ])
+
+    def test_many_expressions(self):
+        # Create a string with many expressions in it. Note that
+        #  because we have a space in here as a literal, we're actually
+        #  going to use twice as many ast nodes: one for each literal
+        #  plus one for each expression.
+        def build_fstr(n, extra=''):
+            return "f'" + ('{x} ' * n) + extra + "'"
+
+        x = 'X'
+        width = 1
+
+        # Test around 256.
+        for i in range(250, 260):
+            self.assertEqual(eval(build_fstr(i)), (x+' ')*i)
+
+        # Test concatenating 2 largs fstrings.
+        self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256))
+
+        s = build_fstr(253, '{x:{width}} ')
+        self.assertEqual(eval(s), (x+' ')*254)
+
+        # Test lots of expressions and constants, concatenated.
+        s = "f'{1}' 'x' 'y'" * 1024
+        self.assertEqual(eval(s), '1xy' * 1024)
+
+    def test_format_specifier_expressions(self):
+        width = 10
+        precision = 4
+        value = decimal.Decimal('12.34567')
+        self.assertEqual(f'result: {value:{width}.{precision}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result:      12.35')
+        self.assertEqual(f'{10:#{1}0x}', '       0xa')
+        self.assertEqual(f'{10:{"#"}1{0}{"x"}}', '       0xa')
+        self.assertEqual(f'{-10:-{"#"}1{0}x}', '      -0xa')
+        self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', '      -0xa')
+        self.assertEqual(f'{10:#{3 != {4:5} and width}x}', '       0xa')
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["""f'{"s"!r{":10"}}'""",
+
+                             # This looks like a nested format spec.
+                             ])
+
+        self.assertAllRaise(SyntaxError, "invalid syntax",
+                            [# Invalid syntax inside a nested spec.
+                             "f'{4:{/5}}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply",
+                            [# Can't nest format specifiers.
+                             "f'result: {value:{width:{0}}.{precision:1}}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character',
+                            [# No expansion inside conversion or for
+                             #  the : or ! itself.
+                             """f'{"s"!{"r"}}'""",
+                             ])
+
+    def test_side_effect_order(self):
+        class X:
+            def __init__(self):
+                self.i = 0
+            def __format__(self, spec):
+                self.i += 1
+                return str(self.i)
+
+        x = X()
+        self.assertEqual(f'{x} {x}', '1 2')
+
+    def test_missing_expression(self):
+        self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed',
+                            ["f'{}'",
+                             "f'{ }'"
+                             "f' {} '",
+                             "f'{!r}'",
+                             "f'{ !r}'",
+                             "f'{10:{ }}'",
+                             "f' { } '",
+
+                             # Catch the empty expression before the
+                             #  invalid conversion.
+                             "f'{!x}'",
+                             "f'{ !xr}'",
+                             "f'{!x:}'",
+                             "f'{!x:a}'",
+                             "f'{ !xr:}'",
+                             "f'{ !xr:a}'",
+
+                             "f'{!}'",
+                             "f'{:}'",
+
+                             # We find the empty expression before the
+                             #  missing closing brace.
+                             "f'{!'",
+                             "f'{!s:'",
+                             "f'{:'",
+                             "f'{:x'",
+                             ])
+
+    def test_parens_in_expressions(self):
+        self.assertEqual(f'{3,}', '(3,)')
+
+        # Add these because when an expression is evaluated, parens
+        #  are added around it. But we shouldn't go from an invalid
+        #  expression to a valid one. The added parens are just
+        #  supposed to allow whitespace (including newlines).
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["f'{,}'",
+                             "f'{,}'",  # this is (,), which is an error
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{3)+(4}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, 'EOL while scanning string literal',
+                            ["f'{\n}'",
+                             ])
+
+    def test_backslashes_in_string_part(self):
+        self.assertEqual(f'\t', '\t')
+        self.assertEqual(r'\t', '\\t')
+        self.assertEqual(rf'\t', '\\t')
+        self.assertEqual(f'{2}\t', '2\t')
+        self.assertEqual(f'{2}\t{3}', '2\t3')
+        self.assertEqual(f'\t{3}', '\t3')
+
+        self.assertEqual(f'\u0394', '\u0394')
+        self.assertEqual(r'\u0394', '\\u0394')
+        self.assertEqual(rf'\u0394', '\\u0394')
+        self.assertEqual(f'{2}\u0394', '2\u0394')
+        self.assertEqual(f'{2}\u0394{3}', '2\u03943')
+        self.assertEqual(f'\u0394{3}', '\u03943')
+
+        self.assertEqual(f'\U00000394', '\u0394')
+        self.assertEqual(r'\U00000394', '\\U00000394')
+        self.assertEqual(rf'\U00000394', '\\U00000394')
+        self.assertEqual(f'{2}\U00000394', '2\u0394')
+        self.assertEqual(f'{2}\U00000394{3}', '2\u03943')
+        self.assertEqual(f'\U00000394{3}', '\u03943')
+
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', '\u0394')
+        self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}', '2\u0394')
+        self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}{3}', '2\u03943')
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}{3}', '\u03943')
+        self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}', '2\u0394')
+        self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}3', '2\u03943')
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}3', '\u03943')
+
+        self.assertEqual(f'\x20', ' ')
+        self.assertEqual(r'\x20', '\\x20')
+        self.assertEqual(rf'\x20', '\\x20')
+        self.assertEqual(f'{2}\x20', '2 ')
+        self.assertEqual(f'{2}\x20{3}', '2 3')
+        self.assertEqual(f'\x20{3}', ' 3')
+
+        self.assertEqual(f'2\x20', '2 ')
+        self.assertEqual(f'2\x203', '2 3')
+        self.assertEqual(f'\x203', ' 3')
+
+    def test_misformed_unicode_character_name(self):
+        # These test are needed because unicode names are parsed
+        # differently inside f-strings.
+        self.assertAllRaise(SyntaxError, r"\(unicode error\) 'unicodeescape' codec can't decode bytes in position .*: malformed \\N character escape",
+                            [r"f'\N'",
+                             r"f'\N{'",
+                             r"f'\N{GREEK CAPITAL LETTER DELTA'",
+
+                             # Here are the non-f-string versions,
+                             #  which should give the same errors.
+                             r"'\N'",
+                             r"'\N{'",
+                             r"'\N{GREEK CAPITAL LETTER DELTA'",
+                             ])
+
+    def test_no_backslashes_in_expression_part(self):
+        self.assertAllRaise(SyntaxError, 'f-string expression part cannot include a backslash',
+                            [r"f'{\'a\'}'",
+                             r"f'{\t3}'",
+                             r"f'{\}'",
+                             r"rf'{\'a\'}'",
+                             r"rf'{\t3}'",
+                             r"rf'{\}'",
+                             r"""rf'{"\N{LEFT CURLY BRACKET}"}'""",
+                             r"f'{\n}'",
+                             ])
+
+    def test_no_escapes_for_braces(self):
+        """
+        Only literal curly braces begin an expression.
+        """
+        # \x7b is '{'.
+        self.assertEqual(f'\x7b1+1}}', '{1+1}')
+        self.assertEqual(f'\x7b1+1', '{1+1')
+        self.assertEqual(f'\u007b1+1', '{1+1')
+        self.assertEqual(f'\N{LEFT CURLY BRACKET}1+1\N{RIGHT CURLY BRACKET}', '{1+1}')
+
+    def test_newlines_in_expressions(self):
+        self.assertEqual(f'{0}', '0')
+        self.assertEqual(rf'''{3+
+4}''', '7')
+
+    def test_lambda(self):
+        x = 5
+        self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'")
+        self.assertEqual(f'{(lambda y:x*y)("8")!r:10}', "'88888'   ")
+        self.assertEqual(f'{(lambda y:x*y)("8"):10}', "88888     ")
+
+        # lambda doesn't work without parens, because the colon
+        #  makes the parser think it's a format_spec
+        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+                            ["f'{lambda x:x}'",
+                             ])
+
+    def test_yield(self):
+        # Not terribly useful, but make sure the yield turns
+        #  a function into a generator
+        def fn(y):
+            f'y:{yield y*2}'
+
+        g = fn(4)
+        self.assertEqual(next(g), 8)
+
+    def test_yield_send(self):
+        def fn(x):
+            yield f'x:{yield (lambda i: x * i)}'
+
+        g = fn(10)
+        the_lambda = next(g)
+        self.assertEqual(the_lambda(4), 40)
+        self.assertEqual(g.send('string'), 'x:string')
+
+    def test_expressions_with_triple_quoted_strings(self):
+        self.assertEqual(f"{'''x'''}", 'x')
+        self.assertEqual(f"{'''eric's'''}", "eric's")
+
+        # Test concatenation within an expression
+        self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy')
+        self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s')
+        self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy')
+        self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy')
+        self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy')
+        self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy')
+
+    def test_multiple_vars(self):
+        x = 98
+        y = 'abc'
+        self.assertEqual(f'{x}{y}', '98abc')
+
+        self.assertEqual(f'X{x}{y}', 'X98abc')
+        self.assertEqual(f'{x}X{y}', '98Xabc')
+        self.assertEqual(f'{x}{y}X', '98abcX')
+
+        self.assertEqual(f'X{x}Y{y}', 'X98Yabc')
+        self.assertEqual(f'X{x}{y}Y', 'X98abcY')
+        self.assertEqual(f'{x}X{y}Y', '98XabcY')
+
+        self.assertEqual(f'X{x}Y{y}Z', 'X98YabcZ')
+
+    def test_closure(self):
+        def outer(x):
+            def inner():
+                return f'x:{x}'
+            return inner
+
+        self.assertEqual(outer('987')(), 'x:987')
+        self.assertEqual(outer(7)(), 'x:7')
+
+    def test_arguments(self):
+        y = 2
+        def f(x, width):
+            return f'x={x*y:{width}}'
+
+        self.assertEqual(f('foo', 10), 'x=foofoo    ')
+        x = 'bar'
+        self.assertEqual(f(10, 10), 'x=        20')
+
+    def test_locals(self):
+        value = 123
+        self.assertEqual(f'v:{value}', 'v:123')
+
+    def test_missing_variable(self):
+        with self.assertRaises(NameError):
+            f'v:{value}'
+
+    def test_missing_format_spec(self):
+        class O:
+            def __format__(self, spec):
+                if not spec:
+                    return '*'
+                return spec
+
+        self.assertEqual(f'{O():x}', 'x')
+        self.assertEqual(f'{O()}', '*')
+        self.assertEqual(f'{O():}', '*')
+
+        self.assertEqual(f'{3:}', '3')
+        self.assertEqual(f'{3!s:}', '3')
+
+    def test_global(self):
+        self.assertEqual(f'g:{a_global}', 'g:global variable')
+        self.assertEqual(f'g:{a_global!r}', "g:'global variable'")
+
+        a_local = 'local variable'
+        self.assertEqual(f'g:{a_global} l:{a_local}',
+                         'g:global variable l:local variable')
+        self.assertEqual(f'g:{a_global!r}',
+                         "g:'global variable'")
+        self.assertEqual(f'g:{a_global} l:{a_local!r}',
+                         "g:global variable l:'local variable'")
+
+        self.assertIn("module 'unittest' from", f'{unittest}')
+
+    def test_shadowed_global(self):
+        a_global = 'really a local'
+        self.assertEqual(f'g:{a_global}', 'g:really a local')
+        self.assertEqual(f'g:{a_global!r}', "g:'really a local'")
+
+        a_local = 'local variable'
+        self.assertEqual(f'g:{a_global} l:{a_local}',
+                         'g:really a local l:local variable')
+        self.assertEqual(f'g:{a_global!r}',
+                         "g:'really a local'")
+        self.assertEqual(f'g:{a_global} l:{a_local!r}',
+                         "g:really a local l:'local variable'")
+
+    def test_call(self):
+        def foo(x):
+            return 'x=' + str(x)
+
+        self.assertEqual(f'{foo(10)}', 'x=10')
+
+    def test_nested_fstrings(self):
+        y = 5
+        self.assertEqual(f'{f"{0}"*3}', '000')
+        self.assertEqual(f'{f"{y}"*3}', '555')
+
+    def test_invalid_string_prefixes(self):
+        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+                            ["fu''",
+                             "uf''",
+                             "Fu''",
+                             "fU''",
+                             "Uf''",
+                             "uF''",
+                             "ufr''",
+                             "urf''",
+                             "fur''",
+                             "fru''",
+                             "rfu''",
+                             "ruf''",
+                             "FUR''",
+                             "Fur''",
+                             "fb''",
+                             "fB''",
+                             "Fb''",
+                             "FB''",
+                             "bf''",
+                             "bF''",
+                             "Bf''",
+                             "BF''",
+                             ])
+
+    def test_leading_trailing_spaces(self):
+        self.assertEqual(f'{ 3}', '3')
+        self.assertEqual(f'{  3}', '3')
+        self.assertEqual(f'{3 }', '3')
+        self.assertEqual(f'{3  }', '3')
+
+        self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}',
+                         'expr={1: 2}')
+        self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }',
+                         'expr={1: 2}')
+
+    def test_not_equal(self):
+        # There's a special test for this because there's a special
+        #  case in the f-string parser to look for != as not ending an
+        #  expression. Normally it would, while looking for !s or !r.
+
+        self.assertEqual(f'{3!=4}', 'True')
+        self.assertEqual(f'{3!=4:}', 'True')
+        self.assertEqual(f'{3!=4!s}', 'True')
+        self.assertEqual(f'{3!=4!s:.3}', 'Tru')
+
+    def test_conversions(self):
+        self.assertEqual(f'{3.14:10.10}', '      3.14')
+        self.assertEqual(f'{3.14!s:10.10}', '3.14      ')
+        self.assertEqual(f'{3.14!r:10.10}', '3.14      ')
+        self.assertEqual(f'{3.14!a:10.10}', '3.14      ')
+
+        self.assertEqual(f'{"a"}', 'a')
+        self.assertEqual(f'{"a"!r}', "'a'")
+        self.assertEqual(f'{"a"!a}', "'a'")
+
+        # Not a conversion.
+        self.assertEqual(f'{"a!r"}', "a!r")
+
+        # Not a conversion, but show that ! is allowed in a format spec.
+        self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!')
+
+        self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character',
+                            ["f'{3!g}'",
+                             "f'{3!A}'",
+                             "f'{3!3}'",
+                             "f'{3!G}'",
+                             "f'{3!!}'",
+                             "f'{3!:}'",
+                             "f'{3! s}'",  # no space before conversion char
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{x!s{y}}'",
+                             "f'{3!ss}'",
+                             "f'{3!ss:}'",
+                             "f'{3!ss:s}'",
+                             ])
+
+    def test_assignment(self):
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["f'' = 3",
+                             "f'{0}' = x",
+                             "f'{x}' = x",
+                             ])
+
+    def test_del(self):
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["del f''",
+                             "del '' f''",
+                             ])
+
+    def test_mismatched_braces(self):
+        self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed",
+                            ["f'{{}'",
+                             "f'{{}}}'",
+                             "f'}'",
+                             "f'x}'",
+                             "f'x}x'",
+                             r"f'\u007b}'",
+
+                             # Can't have { or } in a format spec.
+                             "f'{3:}>10}'",
+                             "f'{3:}}>10}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{3:{{>10}'",
+                             "f'{3'",
+                             "f'{3!'",
+                             "f'{3:'",
+                             "f'{3!s'",
+                             "f'{3!s:'",
+                             "f'{3!s:3'",
+                             "f'x{'",
+                             "f'x{x'",
+                             "f'{x'",
+                             "f'{3:s'",
+                             "f'{{{'",
+                             "f'{{}}{'",
+                             "f'{'",
+                             ])
+
+        # But these are just normal strings.
+        self.assertEqual(f'{"{"}', '{')
+        self.assertEqual(f'{"}"}', '}')
+        self.assertEqual(f'{3:{"}"}>10}', '}}}}}}}}}3')
+        self.assertEqual(f'{2:{"{"}>10}', '{{{{{{{{{2')
+
+    def test_if_conditional(self):
+        # There's special logic in compile.c to test if the
+        #  conditional for an if (and while) are constants. Exercise
+        #  that code.
+
+        def test_fstring(x, expected):
+            flag = 0
+            if f'{x}':
+                flag = 1
+            else:
+                flag = 2
+            self.assertEqual(flag, expected)
+
+        def test_concat_empty(x, expected):
+            flag = 0
+            if '' f'{x}':
+                flag = 1
+            else:
+                flag = 2
+            self.assertEqual(flag, expected)
+
+        def test_concat_non_empty(x, expected):
+            flag = 0
+            if ' ' f'{x}':
+                flag = 1
+            else:
+                flag = 2
+            self.assertEqual(flag, expected)
+
+        test_fstring('', 2)
+        test_fstring(' ', 1)
+
+        test_concat_empty('', 2)
+        test_concat_empty(' ', 1)
+
+        test_concat_non_empty('', 1)
+        test_concat_non_empty(' ', 1)
+
+    def test_empty_format_specifier(self):
+        x = 'test'
+        self.assertEqual(f'{x}', 'test')
+        self.assertEqual(f'{x:}', 'test')
+        self.assertEqual(f'{x!s:}', 'test')
+        self.assertEqual(f'{x!r:}', "'test'")
+
+    def test_str_format_differences(self):
+        d = {'a': 'string',
+             0: 'integer',
+             }
+        a = 0
+        self.assertEqual(f'{d[0]}', 'integer')
+        self.assertEqual(f'{d["a"]}', 'string')
+        self.assertEqual(f'{d[a]}', 'integer')
+        self.assertEqual('{d[a]}'.format(d=d), 'string')
+        self.assertEqual('{d[0]}'.format(d=d), 'integer')
+
+    def test_invalid_expressions(self):
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            [r"f'{a[4)}'",
+                             r"f'{a(4]}'",
+                            ])
+
+    def test_errors(self):
+        # see issue 26287
+        self.assertAllRaise(TypeError, 'unsupported',
+                            [r"f'{(lambda: 0):x}'",
+                             r"f'{(0,):x}'",
+                             ])
+        self.assertAllRaise(ValueError, 'Unknown format code',
+                            [r"f'{1000:j}'",
+                             r"f'{1000:j}'",
+                            ])
+
+    def test_loop(self):
+        for i in range(1000):
+            self.assertEqual(f'i:{i}', 'i:' + str(i))
+
+    def test_dict(self):
+        d = {'"': 'dquote',
+             "'": 'squote',
+             'foo': 'bar',
+             }
+        self.assertEqual(f'''{d["'"]}''', 'squote')
+        self.assertEqual(f"""{d['"']}""", 'dquote')
+
+        self.assertEqual(f'{d["foo"]}', 'bar')
+        self.assertEqual(f"{d['foo']}", 'bar')
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -196,6 +196,11 @@
                default=False,
                requires=[("objspace.usemodules.cpyext", False)]),
 
+    BoolOption("fstrings",
+               "if you are really convinced that f-strings are a security "
+               "issue, you can disable them here",
+               default=True),
+
     OptionDescription("std", "Standard Object Space Options", [
         BoolOption("withtproxy", "support transparent proxies",
                    default=True),
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -758,6 +758,14 @@
 def _compute_CALL_METHOD(arg):
     return -_num_args(arg) - 1
 
+def _compute_FORMAT_VALUE(arg):
+    if (arg & consts.FVS_MASK) == consts.FVS_HAVE_SPEC:
+        return -1
+    return 0
+
+def _compute_BUILD_STRING(arg):
+    return 1 - arg
+
 
 _stack_effect_computers = {}
 for name, func in globals().items():
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1733,6 +1733,10 @@
             return Num.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Str):
             return Str.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_FormattedValue):
+            return FormattedValue.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_JoinedStr):
+            return JoinedStr.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Bytes):
             return Bytes.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_NameConstant):
@@ -2639,6 +2643,100 @@
 State.ast_type('Str', 'expr', ['s'])
 
 
+class FormattedValue(expr):
+
+    def __init__(self, value, conversion, format_spec, lineno, col_offset):
+        self.value = value
+        self.conversion = conversion
+        self.format_spec = format_spec
+        expr.__init__(self, lineno, col_offset)
+
+    def walkabout(self, visitor):
+        visitor.visit_FormattedValue(self)
+
+    def mutate_over(self, visitor):
+        self.value = self.value.mutate_over(visitor)
+        if self.format_spec:
+            self.format_spec = self.format_spec.mutate_over(visitor)
+        return visitor.visit_FormattedValue(self)
+
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_FormattedValue)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        w_conversion = space.wrap(self.conversion)  # int
+        space.setattr(w_node, space.wrap('conversion'), w_conversion)
+        w_format_spec = self.format_spec.to_object(space) if self.format_spec is not None else space.w_None  # expr
+        space.setattr(w_node, space.wrap('format_spec'), w_format_spec)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        w_value = get_field(space, w_node, 'value', False)
+        w_conversion = get_field(space, w_node, 'conversion', True)
+        w_format_spec = get_field(space, w_node, 'format_spec', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = expr.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
+        _conversion = space.int_w(w_conversion)
+        _format_spec = expr.from_object(space, w_format_spec)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
+        return FormattedValue(_value, _conversion, _format_spec, _lineno, _col_offset)
+
+State.ast_type('FormattedValue', 'expr', ['value', 'conversion', 'format_spec'])
+
+
+class JoinedStr(expr):
+
+    def __init__(self, values, lineno, col_offset):
+        self.values = values
+        expr.__init__(self, lineno, col_offset)
+
+    def walkabout(self, visitor):
+        visitor.visit_JoinedStr(self)
+
+    def mutate_over(self, visitor):
+        if self.values:
+            for i in range(len(self.values)):
+                if self.values[i] is not None:
+                    self.values[i] = self.values[i].mutate_over(visitor)
+        return visitor.visit_JoinedStr(self)
+
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_JoinedStr)
+        if self.values is None:
+            values_w = []
+        else:
+            values_w = [node.to_object(space) for node in self.values] # expr
+        w_values = space.newlist(values_w)
+        space.setattr(w_node, space.wrap('values'), w_values)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        w_values = get_field(space, w_node, 'values', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        values_w = space.unpackiterable(w_values)
+        _values = [expr.from_object(space, w_item) for w_item in values_w]
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
+        return JoinedStr(_values, _lineno, _col_offset)
+
+State.ast_type('JoinedStr', 'expr', ['values'])
+
+
 class Bytes(expr):
 
     def __init__(self, s, lineno, col_offset):
@@ -4022,6 +4120,10 @@
         return self.default_visitor(node)
     def visit_Str(self, node):
         return self.default_visitor(node)
+    def visit_FormattedValue(self, node):
+        return self.default_visitor(node)
+    def visit_JoinedStr(self, node):
+        return self.default_visitor(node)
     def visit_Bytes(self, node):
         return self.default_visitor(node)
     def visit_NameConstant(self, node):
@@ -4251,6 +4353,14 @@
     def visit_Str(self, node):
         pass
 
+    def visit_FormattedValue(self, node):
+        node.value.walkabout(self)
+        if node.format_spec:
+            node.format_spec.walkabout(self)
+
+    def visit_JoinedStr(self, node):
+        self.visit_sequence(node.values)
+
     def visit_Bytes(self, node):
         pass
 
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1,15 +1,15 @@
 from pypy.interpreter.astcompiler import ast, consts, misc
 from pypy.interpreter.astcompiler import asthelpers # Side effects
+from pypy.interpreter.astcompiler import fstring
 from pypy.interpreter import error
 from pypy.interpreter.pyparser.pygram import syms, tokens
 from pypy.interpreter.pyparser.error import SyntaxError
-from pypy.interpreter.pyparser import parsestring
 from rpython.rlib.objectmodel import always_inline, we_are_translated
 
 
-def ast_from_node(space, node, compile_info):
+def ast_from_node(space, node, compile_info, recursive_parser=None):
     """Turn a parse tree, node, to AST."""
-    ast = ASTBuilder(space, node, compile_info).build_ast()
+    ast = ASTBuilder(space, node, compile_info, recursive_parser).build_ast()
     #
     # When we are not translated, we send this ast to validate_ast.
     # The goal is to check that validate_ast doesn't crash on valid
@@ -54,10 +54,11 @@
 
 class ASTBuilder(object):
 
-    def __init__(self, space, n, compile_info):
+    def __init__(self, space, n, compile_info, recursive_parser=None):
         self.space = space
         self.compile_info = compile_info
         self.root_node = n
+        self.recursive_parser = recursive_parser
 
     def build_ast(self):
         """Convert an top level parse tree node into an AST mod."""
@@ -1189,7 +1190,7 @@
             value = self.handle_expr(node.get_child(i+2))
             i += 3
         return (i,key,value)
-    
+
     def handle_atom(self, atom_node):
         first_child = atom_node.get_child(0)
         first_child_type = first_child.type
@@ -1207,39 +1208,10 @@
                                 first_child.get_column())
             return ast.NameConstant(w_singleton, first_child.get_lineno(),
                                 first_child.get_column())
+        #
         elif first_child_type == tokens.STRING:
-            space = self.space
-            encoding = self.compile_info.encoding
-            try:
-                sub_strings_w = [
-                    parsestring.parsestr(
-                            space, encoding, atom_node.get_child(i).get_value())
-                        for i in range(atom_node.num_children())]
-            except error.OperationError as e:
-                if e.match(space, space.w_UnicodeError):
-                    kind = 'unicode error'
-                elif e.match(space, space.w_ValueError):
-                    kind = 'value error'
-                else:
-                    raise
-                # Unicode/ValueError in literal: turn into SyntaxError
-                e.normalize_exception(space)
-                errmsg = space.str_w(space.str(e.get_w_value(space)))
-                raise self.error('(%s) %s' % (kind, errmsg), atom_node)
-            # Implement implicit string concatenation.
-            w_string = sub_strings_w[0]
-            for i in range(1, len(sub_strings_w)):
-                try:
-                    w_string = space.add(w_string, sub_strings_w[i])
-                except error.OperationError as e:
-                    if not e.match(space, space.w_TypeError):
-                        raise
-                    self.error("cannot mix bytes and nonbytes literals",
-                              atom_node)
-                # UnicodeError in literal: turn into SyntaxError
-            strdata = space.isinstance_w(w_string, space.w_unicode)
-            node = ast.Str if strdata else ast.Bytes
-            return node(w_string, atom_node.get_lineno(), atom_node.get_column())
+            return fstring.string_parse_literal(self, atom_node)
+        #
         elif first_child_type == tokens.NUMBER:
             num_value = self.parse_number(first_child.get_value())
             return ast.Num(num_value, atom_node.get_lineno(), atom_node.get_column())
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1502,6 +1502,23 @@
             sub.value.walkabout(self)
         self._compile_slice(sub.slice, sub.ctx)
 
+    def visit_JoinedStr(self, joinedstr):
+        self.update_position(joinedstr.lineno)
+        for node in joinedstr.values:
+            node.walkabout(self)
+        self.emit_op_arg(ops.BUILD_STRING, len(joinedstr.values))
+
+    def visit_FormattedValue(self, fmt):
+        fmt.value.walkabout(self)
+        arg = 0
+        if fmt.conversion == ord('s'): arg = consts.FVC_STR
+        if fmt.conversion == ord('r'): arg = consts.FVC_REPR
+        if fmt.conversion == ord('a'): arg = consts.FVC_ASCII
+        if fmt.format_spec is not None:
+            arg |= consts.FVS_HAVE_SPEC
+            fmt.format_spec.walkabout(self)
+        self.emit_op_arg(ops.FORMAT_VALUE, arg)
+
 
 class TopLevelCodeGenerator(PythonCodeGenerator):
 
diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -33,3 +33,12 @@
 PyCF_IGNORE_COOKIE = 0x0800
 PyCF_ACCEPT_NULL_BYTES = 0x10000000   # PyPy only, for compile()
 PyCF_FOUND_ENCODING = 0x20000000      # PyPy only, for pytokenizer
+
+# Masks and values used by FORMAT_VALUE opcode
+FVC_MASK      = 0x3
+FVC_NONE      = 0x0
+FVC_STR       = 0x1
+FVC_REPR      = 0x2
+FVC_ASCII     = 0x3
+FVS_MASK      = 0x4
+FVS_HAVE_SPEC = 0x4
diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -0,0 +1,377 @@
+from pypy.interpreter.astcompiler import ast, consts
+from pypy.interpreter.pyparser import parsestring
+from pypy.interpreter import error
+from pypy.interpreter import unicodehelper
+from rpython.rlib.rstring import UnicodeBuilder
+
+
+def add_constant_string(astbuilder, joined_pieces, w_string, atom_node):
+    space = astbuilder.space
+    is_unicode = space.isinstance_w(w_string, space.w_unicode)
+    # Implement implicit string concatenation.
+    if joined_pieces:
+        prev = joined_pieces[-1]
+        if is_unicode and isinstance(prev, ast.Str):
+            w_string = space.add(prev.s, w_string)
+            del joined_pieces[-1]
+        elif not is_unicode and isinstance(prev, ast.Bytes):
+            w_string = space.add(prev.s, w_string)
+            del joined_pieces[-1]
+    node = ast.Str if is_unicode else ast.Bytes
+    joined_pieces.append(node(w_string, atom_node.get_lineno(),
+                                        atom_node.get_column()))
+
+def f_constant_string(astbuilder, joined_pieces, u, atom_node):
+    space = astbuilder.space
+    add_constant_string(astbuilder, joined_pieces, space.newunicode(u),
+                        atom_node)
+
+def f_string_compile(astbuilder, source, atom_node):
+    # Note: a f-string is kept as a single literal up to here.
+    # At this point only, we recursively call the AST compiler
+    # on all the '{expr}' parts.  The 'expr' part is not parsed
+    # or even tokenized together with the rest of the source code!
+    from pypy.interpreter.pyparser import pyparse
+    from pypy.interpreter.astcompiler.astbuilder import ast_from_node
+
+    # complain if 'source' is only whitespace or an empty string
+    for c in source:
+        if c not in ' \t\n\r\v\f':
+            break
+    else:
+        astbuilder.error("f-string: empty expression not allowed", atom_node)
+
+    if astbuilder.recursive_parser is None:
+        astbuilder.error("internal error: parser not available for parsing "
+                   "the expressions inside the f-string", atom_node)
+    source = '(%s)' % source.encode('utf-8')
+
+    info = pyparse.CompileInfo("<fstring>", "eval",
+                               consts.PyCF_SOURCE_IS_UTF8 |
+                               consts.PyCF_IGNORE_COOKIE,
+                               optimize=astbuilder.compile_info.optimize)
+    parser = astbuilder.recursive_parser
+    parse_tree = parser.parse_source(source, info)
+    return ast_from_node(astbuilder.space, parse_tree, info,
+                         recursive_parser=parser)
+
+
+def unexpected_end_of_string(astbuilder, atom_node):
+    astbuilder.error("f-string: expecting '}'", atom_node)
+
+
+def fstring_find_expr(astbuilder, fstr, atom_node, rec):
+    # Parse the f-string at fstr.current_index.  We know it starts an
+    # expression (so it must be at '{'). Returns the FormattedValue node,
+    # which includes the expression, conversion character, and
+    # format_spec expression.
+    conversion = -1      # the conversion char.  -1 if not specified.
+    format_spec = None
+
+    # 0 if we're not in a string, else the quote char we're trying to
+    # match (single or double quote).
+    quote_char = 0
+
+    # If we're inside a string, 1=normal, 3=triple-quoted.
+    string_type = 0
+
+    # Keep track of nesting level for braces/parens/brackets in
+    # expressions.
+    nested_depth = 0
+
+    # Can only nest one level deep.
+    if rec >= 2:
+        astbuilder.error("f-string: expressions nested too deeply", atom_node)
+
+    # The first char must be a left brace, or we wouldn't have gotten
+    # here. Skip over it.
+    u = fstr.unparsed
+    i = fstr.current_index
+    assert u[i] == u'{'
+    i += 1
+
+    expr_start = i
+    while i < len(u):
+
+        # Loop invariants.
+        assert nested_depth >= 0
+        if quote_char:
+            assert string_type == 1 or string_type == 3
+        else:
+            assert string_type == 0
+
+        ch = u[i]
+        # Nowhere inside an expression is a backslash allowed.
+        if ch == u'\\':
+            # Error: can't include a backslash character, inside
+            # parens or strings or not.
+            astbuilder.error("f-string expression part "
+                             "cannot include a backslash", atom_node)
+
+        if quote_char:
+            # We're inside a string. See if we're at the end.
+            # <a long comment goes here about how we're duplicating
+            # some existing logic>
+            if ord(ch) == quote_char:
+                # Does this match the string_type (single or triple
+                # quoted)?
+                if string_type == 3:
+                    if i + 2 < len(u) and u[i + 1] == u[i + 2] == ch:
+                        # We're at the end of a triple quoted string.
+                        i += 3
+                        string_type = 0
+                        quote_char = 0
+                        continue
+                else:
+                    # We're at the end of a normal string.
+                    i += 1
+                    string_type = 0
+                    quote_char = 0
+                    continue
+        elif ch == u"'" or ch == u'"':
+            # Is this a triple quoted string?
+            if i + 2 < len(u) and u[i + 1] == u[i + 2] == ch:
+                string_type = 3
+                i += 2
+            else:
+                # Start of a normal string.
+                string_type = 1
+            # Start looking for the end of the string.
+            quote_char = ord(ch)
+        elif ch in u"[{(":
+            nested_depth += 1
+        elif nested_depth != 0 and ch in u"]})":
+            nested_depth -= 1
+        elif ch == u'#':
+            # Error: can't include a comment character, inside parens
+            # or not.
+            astbuilder.error("f-string expression part cannot include '#'",
+                             atom_node)
+        elif nested_depth == 0 and ch in u"!:}":
+            # First, test for the special case of "!=". Since '=' is
+            # not an allowed conversion character, nothing is lost in
+            # this test.
+            if ch == '!' and i + 1 < len(u) and u[i+1] == u'=':
+                # This isn't a conversion character, just continue.
+                i += 1
+                continue
+            # Normal way out of this loop.
+            break
+        #else:
+        #   This isn't a conversion character, just continue.
+        i += 1
+
+    # If we leave this loop in a string or with mismatched parens, we
+    # don't care. We'll get a syntax error when compiling the
+    # expression. But, we can produce a better error message, so
+    # let's just do that.
+    if quote_char:
+        astbuilder.error("f-string: unterminated string", atom_node)
+
+    if nested_depth:
+        astbuilder.error("f-string: mismatched '(', '{' or '['", atom_node)
+
+    if i >= len(u):
+        unexpected_end_of_string(astbuilder, atom_node)
+
+    # Compile the expression as soon as possible, so we show errors
+    # related to the expression before errors related to the
+    # conversion or format_spec.
+    expr = f_string_compile(astbuilder, u[expr_start:i], atom_node)
+    assert isinstance(expr, ast.Expression)
+
+    # Check for a conversion char, if present.
+    if u[i] == u'!':
+        i += 1
+        if i >= len(u):
+            unexpected_end_of_string(astbuilder, atom_node)
+
+        conversion = ord(u[i])
+        i += 1
+        if conversion not in (ord('s'), ord('r'), ord('a')):
+            astbuilder.error("f-string: invalid conversion character: "
+                             "expected 's', 'r', or 'a'", atom_node)
+
+    # Check for the format spec, if present.
+    if i >= len(u):
+        unexpected_end_of_string(astbuilder, atom_node)
+    if u[i] == u':':
+        i += 1
+        if i >= len(u):
+            unexpected_end_of_string(astbuilder, atom_node)
+        fstr.current_index = i
+        subpieces = []
+        parse_f_string(astbuilder, subpieces, fstr, atom_node, rec + 1)
+        format_spec = f_string_to_ast_node(astbuilder, subpieces, atom_node)
+        i = fstr.current_index
+
+    if i >= len(u) or u[i] != u'}':
+        unexpected_end_of_string(astbuilder, atom_node)
+
+    # We're at a right brace. Consume it.
+    i += 1
+    fstr.current_index = i
+
+    # And now create the FormattedValue node that represents this
+    # entire expression with the conversion and format spec.
+    return ast.FormattedValue(expr.body, conversion, format_spec,
+                              atom_node.get_lineno(),
+                              atom_node.get_column())
+
+
+def fstring_find_literal(astbuilder, fstr, atom_node, rec):
+    # Return the next literal part.  Updates the current index inside 'fstr'.
+    # Differs from CPython: this version handles double-braces on its own.
+    u = fstr.unparsed
+    literal_start = fstr.current_index
+    in_named_escape = False
+
+    # Get any literal string. It ends when we hit an un-doubled left
+    # brace (which isn't part of a unicode name escape such as
+    # "\N{EULER CONSTANT}"), or the end of the string.
+    i = literal_start
+    builder = UnicodeBuilder()
+    while i < len(u):
+        ch = u[i]
+        if (not in_named_escape and ch == u'{' and i - literal_start >= 2
+                and u[i - 2] == u'\\' and u[i - 1] == u'N'):
+            in_named_escape = True
+        elif in_named_escape and ch == u'}':
+            in_named_escape = False
+        elif ch == u'{' or ch == u'}':
+            # Check for doubled braces, but only at the top level. If
+            # we checked at every level, then f'{0:{3}}' would fail
+            # with the two closing braces.
+            if rec == 0 and i + 1 < len(u) and u[i + 1] == ch:
+                i += 1   # skip over the second brace
+            elif rec == 0 and ch == u'}':
+                # Where a single '{' is the start of a new expression, a
+                # single '}' is not allowed.
+                astbuilder.error("f-string: single '}' is not allowed",
+                                 atom_node)
+            else:
+                # We're either at a '{', which means we're starting another
+                # expression; or a '}', which means we're at the end of this
+                # f-string (for a nested format_spec).
+                break
+        builder.append(ch)
+        i += 1
+
+    fstr.current_index = i
+    literal = builder.build()
+    if not fstr.raw_mode and u'\\' in literal:
+        # xxx messy
+        space = astbuilder.space
+        literal = literal.encode('utf-8')
+        literal = parsestring.decode_unicode_utf8(space, literal, 0,
+                                                  len(literal))
+        literal = unicodehelper.decode_unicode_escape(space, literal)
+    return literal
+
+
+def fstring_find_literal_and_expr(astbuilder, fstr, atom_node, rec):
+    # Return a tuple with the next literal part, and optionally the
+    # following expression node.  Updates the current index inside 'fstr'.
+    literal = fstring_find_literal(astbuilder, fstr, atom_node, rec)
+
+    u = fstr.unparsed
+    i = fstr.current_index
+    if i >= len(u) or u[i] == u'}':
+        # We're at the end of the string or the end of a nested
+        # f-string: no expression.
+        expr = None
+    else:
+        # We must now be the start of an expression, on a '{'.
+        assert u[i] == u'{'
+        expr = fstring_find_expr(astbuilder, fstr, atom_node, rec)
+    return literal, expr
+
+
+def parse_f_string(astbuilder, joined_pieces, fstr, atom_node, rec=0):
+    # In our case, parse_f_string() and fstring_find_literal_and_expr()
+    # could be merged into a single function with a clearer logic.  It's
+    # done this way to follow CPython's source code more closely.
+
+    space = astbuilder.space
+    if not space.config.objspace.fstrings:
+        raise oefmt(space.w_SystemError,
+                    "f-strings have been disabled in this version of pypy "
+                    "with the translation option --no-objspace-fstrings.  "
+                    "The PyPy team (and CPython) thinks f-strings don't "
+                    "add any security risks, but we leave it to you to "
+                    "convince whoever translated this pypy that it is "
+                    "really the case")
+
+    while True:
+        literal, expr = fstring_find_literal_and_expr(astbuilder, fstr,
+                                                      atom_node, rec)
+
+        # add the literal part
+        f_constant_string(astbuilder, joined_pieces, literal, atom_node)
+
+        if expr is None:
+            break         # We're done with this f-string.
+
+        joined_pieces.append(expr)
+
+    # If recurse_lvl is zero, then we must be at the end of the
+    # string. Otherwise, we must be at a right brace.
+    if rec == 0 and fstr.current_index < len(fstr.unparsed) - 1:
+        astbuilder.error("f-string: unexpected end of string", atom_node)
+
+    if rec != 0 and (fstr.current_index >= len(fstr.unparsed) or
+                     fstr.unparsed[fstr.current_index] != u'}'):
+        astbuilder.error("f-string: expecting '}'", atom_node)
+
+
+def f_string_to_ast_node(astbuilder, joined_pieces, atom_node):
+    # remove empty Strs
+    values = [node for node in joined_pieces
+                   if not (isinstance(node, ast.Str) and not node.s)]
+    if len(values) > 1:
+        return ast.JoinedStr(values, atom_node.get_lineno(),
+                                     atom_node.get_column())
+    elif len(values) == 1:
+        return values[0]
+    else:
+        assert len(joined_pieces) > 0    # they are all empty strings
+        return joined_pieces[0]
+
+
+def string_parse_literal(astbuilder, atom_node):
+    space = astbuilder.space
+    encoding = astbuilder.compile_info.encoding
+    joined_pieces = []
+    try:
+        for i in range(atom_node.num_children()):
+            w_next = parsestring.parsestr(
+                    space, encoding, atom_node.get_child(i).get_value())
+            if not isinstance(w_next, parsestring.W_FString):
+                add_constant_string(astbuilder, joined_pieces, w_next,
+                                    atom_node)
+            else:
+                parse_f_string(astbuilder, joined_pieces, w_next, atom_node)
+
+    except error.OperationError as e:
+        if e.match(space, space.w_UnicodeError):
+            kind = 'unicode error'
+        elif e.match(space, space.w_ValueError):
+            kind = 'value error'
+        else:
+            raise
+        # Unicode/ValueError in literal: turn into SyntaxError
+        e.normalize_exception(space)
+        errmsg = space.str_w(space.str(e.get_w_value(space)))
+        raise astbuilder.error('(%s) %s' % (kind, errmsg), atom_node)
+
+    if len(joined_pieces) == 1:   # <= the common path
+        return joined_pieces[0]   # ast.Str, Bytes or FormattedValue
+
+    # with more than one piece, it is a combination of Str and
+    # FormattedValue pieces---if there is a Bytes, then we got
+    # an invalid mixture of bytes and unicode literals
+    for node in joined_pieces:
+        if isinstance(node, ast.Bytes):
+            astbuilder.error("cannot mix bytes and nonbytes literals",
+                             atom_node)
+    return f_string_to_ast_node(astbuilder, joined_pieces, atom_node)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -12,7 +12,7 @@
     p = pyparse.PythonParser(space)
     info = pyparse.CompileInfo("<test>", mode)
     cst = p.parse_source(expr, info)
-    ast = astbuilder.ast_from_node(space, cst, info)
+    ast = astbuilder.ast_from_node(space, cst, info, recursive_parser=p)
     return codegen.compile_ast(space, ast, info)
 
 def generate_function_code(expr, space):
@@ -20,7 +20,7 @@
     p = pyparse.PythonParser(space)
     info = pyparse.CompileInfo("<test>", 'exec')
     cst = p.parse_source(expr, info)
-    ast = astbuilder.ast_from_node(space, cst, info)
+    ast = astbuilder.ast_from_node(space, cst, info, recursive_parser=p)
     function_ast = optimize.optimize_ast(space, ast.body[0], info)
     function_ast = ast.body[0]
     assert isinstance(function_ast, FunctionDef)
@@ -1162,6 +1162,38 @@
         """
         yield self.st, source, "f()", 43
 
+    def test_fstring(self):
+        yield self.st, """x = 42; z = f'ab{x}cd'""", 'z', 'ab42cd'
+        yield self.st, """z = f'{{'""", 'z', '{'
+        yield self.st, """z = f'}}'""", 'z', '}'
+        yield self.st, """z = f'x{{y'""", 'z', 'x{y'
+        yield self.st, """z = f'x}}y'""", 'z', 'x}y'
+        yield self.st, """z = f'{{{4*10}}}'""", 'z', '{40}'
+        yield self.st, r"""z = fr'x={4*10}\n'""", 'z', 'x=40\\n'
+
+        yield self.st, """x = 'hi'; z = f'{x}'""", 'z', 'hi'
+        yield self.st, """x = 'hi'; z = f'{x!s}'""", 'z', 'hi'
+        yield self.st, """x = 'hi'; z = f'{x!r}'""", 'z', "'hi'"
+        yield self.st, """x = 'hi'; z = f'{x!a}'""", 'z', "'hi'"
+
+        yield self.st, """x = 'hi'; z = f'''{\nx}'''""", 'z', 'hi'
+
+        yield self.st, """x = 'hi'; z = f'{x:5}'""", 'z', 'hi   '
+        yield self.st, """x = 42;   z = f'{x:5}'""", 'z', '   42'
+        yield self.st, """x = 2; z = f'{5:{x:+1}0}'""", 'z', (' ' * 18 + '+5')
+
+        yield self.st, """z=f'{"}"}'""", 'z', '}'
+
+        yield self.st, """z=f'{f"{0}"*3}'""", 'z', '000'
+
+    def test_fstring_error(self):
+        raises(SyntaxError, self.run, "f'{}'")
+        raises(SyntaxError, self.run, "f'{   \t   }'")
+        raises(SyntaxError, self.run, "f'{5#}'")
+        raises(SyntaxError, self.run, "f'{5)#}'")
+        raises(SyntaxError, self.run, "f'''{5)\n#}'''")
+        raises(SyntaxError, self.run, "f'\\x'")
+
 
 class AppTestCompiler:
 
@@ -1384,3 +1416,9 @@
         code, blocks = generate_function_code(source, self.space)
         # there is a stack computation error
         assert blocks[0].instructions[3].arg == 0
+
+    def test_fstring(self):
+        source = """def f(x):
+            return f'ab{x}cd'
+        """
+        code, blocks = generate_function_code(source, self.space)
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -70,6 +70,8 @@
          | Call(expr func, expr* args, keyword* keywords)
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
+         | FormattedValue(expr value, int? conversion, expr? format_spec)
+         | JoinedStr(expr* values)
          | Bytes(bytes s)
          | NameConstant(singleton value)
          | Ellipsis
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -448,3 +448,11 @@
             node.value is not space.w_True and
             node.value is not space.w_False):
             raise ValidationError("singleton must be True, False, or None")
+
+    def visit_JoinedStr(self, node):
+        self._validate_exprs(node.values)
+
+    def visit_FormattedValue(self, node):
+        self._validate_expr(node.value)
+        if node.format_spec:
+            self._validate_expr(node.format_spec)
diff --git a/pypy/interpreter/pycompiler.py b/pypy/interpreter/pycompiler.py
--- a/pypy/interpreter/pycompiler.py
+++ b/pypy/interpreter/pycompiler.py
@@ -150,7 +150,8 @@
         space = self.space
         try:
             parse_tree = self.parser.parse_source(source, info)
-            mod = astbuilder.ast_from_node(space, parse_tree, info)
+            mod = astbuilder.ast_from_node(space, parse_tree, info,
+                                           recursive_parser=self.parser)
         except parseerror.TabError as e:
             raise OperationError(space.w_TabError,
                                  e.wrap_info(space))
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -434,6 +434,10 @@
                 self.GET_AITER(oparg, next_instr)
             elif opcode == opcodedesc.GET_ANEXT.index:
                 self.GET_ANEXT(oparg, next_instr)
+            elif opcode == opcodedesc.FORMAT_VALUE.index:
+                self.FORMAT_VALUE(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_STRING.index:
+                self.BUILD_STRING(oparg, next_instr)
             else:
                 self.MISSING_OPCODE(oparg, next_instr)
 
@@ -1607,6 +1611,39 @@
                         "from __anext__: %T", w_next_iter)
         self.pushvalue(w_awaitable)
 
+    def FORMAT_VALUE(self, oparg, next_instr):
+        from pypy.interpreter.astcompiler import consts
+        space = self.space
+        #
+        if (oparg & consts.FVS_MASK) == consts.FVS_HAVE_SPEC:
+            w_spec = self.popvalue()
+        else:
+            w_spec = space.newunicode(u'')
+        w_value = self.popvalue()
+        #
+        conversion = oparg & consts.FVC_MASK
+        if conversion == consts.FVC_STR:
+            w_value = space.str(w_value)
+        elif conversion == consts.FVC_REPR:
+            w_value = space.repr(w_value)
+        elif conversion == consts.FVC_ASCII:
+            from pypy.objspace.std.unicodeobject import ascii_from_object
+            w_value = ascii_from_object(space, w_value)
+        #
+        w_res = space.format(w_value, w_spec)
+        self.pushvalue(w_res)
+
+    @jit.unroll_safe
+    def BUILD_STRING(self, itemcount, next_instr):
+        space = self.space
+        lst = []
+        for i in range(itemcount-1, -1, -1):
+            w_item = self.peekvalue(i)
+            lst.append(space.unicode_w(w_item))
+        self.dropvalues(itemcount)
+        w_res = space.newunicode(u''.join(lst))
+        self.pushvalue(w_res)
+
 ### ____________________________________________________________ ###
 
 class ExitFrame(Exception):
diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py
--- a/pypy/interpreter/pyparser/dfa_generated.py
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -23,7 +23,7 @@
      '8': 6, '9': 6, ':': 15, ';': 15,
      '<': 10, '=': 14, '>': 9, '@': 14,
      'A': 1, 'B': 2, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'E': 1, 'F': 2, 'G': 1, 'H': 1,
      'I': 1, 'J': 1, 'K': 1, 'L': 1,
      'M': 1, 'N': 1, 'O': 1, 'P': 1,
      'Q': 1, 'R': 3, 'S': 1, 'T': 1,
@@ -31,7 +31,7 @@
      'Y': 1, 'Z': 1, '[': 15, '\\': 19,
      ']': 15, '^': 14, '_': 1, '`': 15,
      'a': 1, 'b': 2, 'c': 1, 'd': 1,
-     'e': 1, 'f': 1, 'g': 1, 'h': 1,
+     'e': 1, 'f': 2, 'g': 1, 'h': 1,
      'i': 1, 'j': 1, 'k': 1, 'l': 1,
      'm': 1, 'n': 1, 'o': 1, 'p': 1,
      'q': 1, 'r': 3, 's': 1, 't': 1,
@@ -78,14 +78,14 @@
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 4, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'E': 1, 'F': 4, 'G': 1, 'H': 1,
      'I': 1, 'J': 1, 'K': 1, 'L': 1,
      'M': 1, 'N': 1, 'O': 1, 'P': 1,
      'Q': 1, 'R': 1, 'S': 1, 'T': 1,
      'U': 1, 'V': 1, 'W': 1, 'X': 1,
      'Y': 1, 'Z': 1, '_': 1, 'a': 1,
      'b': 4, 'c': 1, 'd': 1, 'e': 1,
-     'f': 1, 'g': 1, 'h': 1, 'i': 1,
+     'f': 4, 'g': 1, 'h': 1, 'i': 1,
      'j': 1, 'k': 1, 'l': 1, 'm': 1,
      'n': 1, 'o': 1, 'p': 1, 'q': 1,
      'r': 1, 's': 1, 't': 1, 'u': 1,
diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -152,9 +152,9 @@
         return group(states,
                      chain(states,
                            maybe(states, groupStr(states, "rR")),
-                           maybe(states, groupStr(states, "bB"))),
+                           maybe(states, groupStr(states, "bBfF"))),
                      chain(states,
-                           maybe(states, groupStr(states, "bB")),
+                           maybe(states, groupStr(states, "bBfF")),
                            maybe(states, groupStr(states, "rR"))),
                      maybe(states, groupStr(states, "uU")))
     # ____________________________________________________________
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -1,11 +1,22 @@
 # coding: utf-8
+from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import unicodehelper
 from rpython.rlib.rstring import StringBuilder
 
 
+class W_FString(W_Root):
+    def __init__(self, unparsed, raw_mode):
+        assert isinstance(unparsed, unicode)
+        self.unparsed = unparsed     # but the quotes are removed
+        self.raw_mode = raw_mode
+        self.current_index = 0       # for astcompiler.fstring
+
+
 def parsestr(space, encoding, s):
-    """Parses a string or unicode literal, and return a wrapped value.
+    """Parses a string or unicode literal, and return usually
+    a wrapped value.  If we get an f-string, then instead return
+    an unparsed but unquoted W_FString instance.
 
     If encoding=None, the source string is ascii only.
     In other cases, the source string is in utf-8 encoding.
@@ -23,6 +34,7 @@
     rawmode = False
     unicode_literal = True
     saw_u = False
+    saw_f = False
 
     # string decoration handling
     if quote == 'b' or quote == 'B':
@@ -37,6 +49,10 @@
         ps += 1
         quote = s[ps]
         rawmode = True
+    elif quote == 'f' or quote == 'F':
+        ps += 1
+        quote = s[ps]
+        saw_f = True
 
     if not saw_u:
         if quote == 'r' or quote == 'R':
@@ -47,6 +63,10 @@
             ps += 1
             quote = s[ps]
             unicode_literal = False
+        elif quote == 'f' or quote == 'F':
+            ps += 1
+            quote = s[ps]
+            saw_f = True
 
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
@@ -70,6 +90,9 @@
             substr = s[ps:q]
         else:
             substr = decode_unicode_utf8(space, s, ps, q)
+        if saw_f:
+            v = unicodehelper.decode_utf8(space, substr)
+            return W_FString(v, rawmode)
         v = unicodehelper.decode_unicode_escape(space, substr)
         return space.wrap(v)
 
@@ -88,6 +111,8 @@
             return space.newbytes(substr)
         else:
             v = unicodehelper.decode_utf8(space, substr)
+            if saw_f:
+                return W_FString(v, rawmode)
             return space.wrap(v)
 
     v = PyString_DecodeEscape(space, substr, 'strict', encoding)
diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py
--- a/pypy/interpreter/pyparser/pytokenize.py
+++ b/pypy/interpreter/pyparser/pytokenize.py
@@ -27,10 +27,12 @@
            'R' : None,
            "u" : None,
            "U" : None,
+           'f' : None,
+           'F' : None,
            'b' : None,
            'B' : None}
 
-for uniPrefix in ("", "b", "B"):
+for uniPrefix in ("", "b", "B", "f", "F"):
     for rawPrefix in ("", "r", "R"):
         prefix_1 = uniPrefix + rawPrefix
         prefix_2 = rawPrefix + uniPrefix
@@ -55,6 +57,11 @@
 for t in ("'''", '"""',
           "r'''", 'r"""', "R'''", 'R"""',
           "u'''", 'u"""', "U'''", 'U"""',
+          "f'''", 'f"""', "F'''", 'F"""',
+          "fr'''", 'fr"""', "Fr'''", 'Fr"""',
+          "fR'''", 'fR"""', "FR'''", 'FR"""',
+          "rf'''", 'rf"""', "rF'''", 'rF"""',
+          "Rf'''", 'Rf"""', "RF'''", 'RF"""',
           "b'''", 'b"""', "B'''", 'B"""',
           "br'''", 'br"""', "Br'''", 'Br"""',
           "bR'''", 'bR"""', "BR'''", 'BR"""',
@@ -65,6 +72,11 @@
 for t in ("'", '"',
           "r'", 'r"', "R'", 'R"',
           "u'", 'u"', "U'", 'U"',
+          "f'", 'f"', "F'", 'F"',
+          "fr'", 'fr"', "Fr'", 'Fr"',
+          "fR'", 'fR"', "FR'", 'FR"',
+          "rf'", 'rf"', "rF'", 'rF"',
+          "Rf'", 'Rf"', "RF'", 'RF"',
           "b'", 'b"', "B'", 'B"',
           "br'", 'br"', "Br'", 'Br"',
           "bR'", 'bR"', "BR'", 'BR"',
diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py
--- a/pypy/module/parser/pyparser.py
+++ b/pypy/module/parser/pyparser.py
@@ -9,9 +9,10 @@
 
 
 class W_STType(W_Root):
-    def __init__(self, tree, mode):
+    def __init__(self, tree, mode, recursive_parser=None):
         self.tree = tree
         self.mode = mode
+        self.recursive_parser = recursive_parser
 
     @specialize.arg(3)
     def _build_app_tree(self, space, node, seq_maker, with_lineno, with_column):
@@ -52,7 +53,7 @@
     def descr_compile(self, space, filename="<syntax-tree>"):
         info = pyparse.CompileInfo(filename, self.mode)
         try:
-            ast = ast_from_node(space, self.tree, info)
+            ast = ast_from_node(space, self.tree, info, self.recursive_parser)
             result = compile_ast(space, ast, info)
         except error.IndentationError as e:
             raise OperationError(space.w_IndentationError,
@@ -82,7 +83,7 @@
     except error.SyntaxError as e:
         raise OperationError(space.w_SyntaxError,
                              e.wrap_info(space))
-    return space.wrap(W_STType(tree, mode))
+    return space.wrap(W_STType(tree, mode, recursive_parser=parser))
 
 
 @unwrap_spec(source=str)

From pypy.commits at gmail.com  Tue Jan 24 11:32:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 08:32:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <588781a1.eb86df0a.537ed.d9fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89750:08f726964553
Date: 2017-01-24 17:31 +0100
http://bitbucket.org/pypy/pypy/changeset/08f726964553/

Log:	merge heads

diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -15,6 +15,10 @@
 #define PyMem_Free  PyMem_FREE
 #define PyMem_Realloc  PyMem_REALLOC
 
+#define PyMem_RawMalloc PyMem_Malloc
+#define PyMem_RawFree PyMem_Free
+#define PyMem_RawRealloc PyMem_Realloc
+
 /*
  * Type-oriented memory interface
  * ==============================

From pypy.commits at gmail.com  Tue Jan 24 11:32:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 08:32:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fstring-pep498: close branch,
 ready for merge
Message-ID: <5887819d.dda1df0a.5795c.d8f2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-fstring-pep498
Changeset: r89748:40322e3c6462
Date: 2017-01-24 17:29 +0100
http://bitbucket.org/pypy/pypy/changeset/40322e3c6462/

Log:	close branch, ready for merge


From pypy.commits at gmail.com  Tue Jan 24 12:11:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 09:11:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Python.asdl gives attributes (lineno,
 col_offset) to the class 'arg', 
Message-ID: <58878ab3.da6d1c0a.c2556.67c2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89751:fe93a6ca25bb
Date: 2017-01-24 18:10 +0100
http://bitbucket.org/pypy/pypy/changeset/fe93a6ca25bb/

Log:	Python.asdl gives attributes (lineno, col_offset) to the class
	'arg', but because this ended up as a Product instead of a Sum the
	attributes were ignored. Fix

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -3879,9 +3879,11 @@
 
 class arg(AST):
 
-    def __init__(self, arg, annotation):
+    def __init__(self, arg, annotation, lineno, col_offset):
         self.arg = arg
         self.annotation = annotation
+        self.lineno = lineno
+        self.col_offset = col_offset
 
     def mutate_over(self, visitor):
         if self.annotation:
@@ -3897,19 +3899,27 @@
         space.setattr(w_node, space.wrap('arg'), w_arg)
         w_annotation = self.annotation.to_object(space) if self.annotation is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('annotation'), w_annotation)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
         w_arg = get_field(space, w_node, 'arg', False)
         w_annotation = get_field(space, w_node, 'annotation', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
         _arg = space.identifier_w(w_arg)
         if _arg is None:
             raise_required_value(space, w_node, 'arg')
         _annotation = expr.from_object(space, w_annotation)
-        return arg(_arg, _annotation)
-
-State.ast_type('arg', 'AST', ['arg', 'annotation'])
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
+        return arg(_arg, _annotation, _lineno, _col_offset)
+
+State.ast_type('arg', 'AST', ['arg', 'annotation'], ['lineno', 'col_offset'])
 
 class keyword(AST):
 
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -665,7 +665,8 @@
                 argname = name_node.get_value()
                 argname = self.new_identifier(argname)
                 self.check_forbidden_name(argname, name_node)
-                kwonly.append(ast.arg(argname, ann))
+                kwonly.append(ast.arg(argname, ann, arg.get_lineno(),
+                                                    arg.get_column()))
                 i += 2
             elif arg_type == tokens.DOUBLESTAR:
                 return i
@@ -678,7 +679,7 @@
         ann = None
         if arg_node.num_children() == 3:
             ann = self.handle_expr(arg_node.get_child(2))
-        return ast.arg(name, ann)
+        return ast.arg(name, ann, arg_node.get_lineno(), arg_node.get_column())
 
     def handle_stmt(self, stmt):
         stmt_type = stmt.type
diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py
--- a/pypy/interpreter/astcompiler/test/test_validate.py
+++ b/pypy/interpreter/astcompiler/test/test_validate.py
@@ -51,18 +51,18 @@
             args = ast.arguments(args, vararg, kwonlyargs,
                                  kw_defaults, kwarg, defaults)
             return fac(args)
-        args = [ast.arg("x", ast.Name("x", ast.Store, 0, 0))]
+        args = [ast.arg("x", ast.Name("x", ast.Store, 0, 0), 0, 0)]
         check(arguments(args=args), "must have Load context")
         check(arguments(kwonlyargs=args), "must have Load context")
         check(arguments(defaults=[ast.Num(self.space.wrap(3), 0, 0)]),
                        "more positional defaults than args")
         check(arguments(kw_defaults=[ast.Num(self.space.wrap(4), 0, 0)]),
                        "length of kwonlyargs is not the same as kw_defaults")
-        args = [ast.arg("x", ast.Name("x", ast.Load, 0, 0))]
+        args = [ast.arg("x", ast.Name("x", ast.Load, 0, 0), 0, 0)]
         check(arguments(args=args, defaults=[ast.Name("x", ast.Store, 0, 0)]),
                        "must have Load context")
-        args = [ast.arg("a", ast.Name("x", ast.Load, 0, 0)),
-                ast.arg("b", ast.Name("y", ast.Load, 0, 0))]
+        args = [ast.arg("a", ast.Name("x", ast.Load, 0, 0), 0, 0),
+                ast.arg("b", ast.Name("y", ast.Load, 0, 0), 0, 0)]
         check(arguments(kwonlyargs=args,
                           kw_defaults=[None, ast.Name("x", ast.Store, 0, 0)]),
                           "must have Load context")
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -56,6 +56,7 @@
 
     def visitSum(self, sum, base):
         if is_simple_sum(sum):
+            assert not sum.attributes
             self.emit("class %s(AST):" % (base,))
             self.emit("@staticmethod", 1)
             self.emit("def from_object(space, w_node):", 1)
@@ -111,15 +112,19 @@
     def visitProduct(self, product, name):
         self.emit("class %s(AST):" % (name,))
         self.emit("")
-        self.make_constructor(product.fields, product)
+        self.make_constructor(product.fields + product.attributes, product)
         self.emit("")
         self.make_mutate_over(product, name)
         self.emit("def walkabout(self, visitor):", 1)
         self.emit("visitor.visit_%s(self)" % (name,), 2)
         self.emit("")
-        self.make_converters(product.fields, name)
-        self.emit("State.ast_type(%r, 'AST', %s)" %
-                  (name, [f.name for f in product.fields]))
+        self.make_converters(product.fields + product.attributes, name)
+        if product.attributes:
+            attr_names = ', %s' % ([a.name for a in product.attributes],)
+        else:
+            attr_names = ''
+        self.emit("State.ast_type(%r, 'AST', %s%s)" %
+                  (name, [f.name for f in product.fields], attr_names))
         self.emit("")
 
     def get_value_converter(self, field, value):

From pypy.commits at gmail.com  Tue Jan 24 12:24:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 09:24:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Attempt to fix lib-python's test_ast.py:
 change the recorded position
Message-ID: <58878dc5.2d88df0a.f10f0.e885@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89752:abf9387139e4
Date: 2017-01-24 18:23 +0100
http://bitbucket.org/pypy/pypy/changeset/abf9387139e4/

Log:	Attempt to fix lib-python's test_ast.py: change the recorded
	position for ast.Set,Dict,SetComp,DictComp

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1249,10 +1249,10 @@
                     (n_maker_children > 1 and
                      maker.get_child(1).type == tokens.COMMA)):
                     # a set display
-                    return self.handle_setdisplay(maker)
+                    return self.handle_setdisplay(maker, atom_node)
                 elif n_maker_children > 1 and maker.get_child(1).type == syms.comp_for:
                     # a set comprehension
-                    return self.handle_setcomp(maker)
+                    return self.handle_setcomp(maker, atom_node)
                 elif (n_maker_children > (3-is_dict) and
                       maker.get_child(3-is_dict).type == syms.comp_for):
                     # a dictionary comprehension
@@ -1260,10 +1260,10 @@
                         raise self.error("dict unpacking cannot be used in "
                                          "dict comprehension", atom_node)
                     
-                    return self.handle_dictcomp(maker)
+                    return self.handle_dictcomp(maker, atom_node)
                 else:
                     # a dictionary display
-                    return self.handle_dictdisplay(maker)
+                    return self.handle_dictdisplay(maker, atom_node)
         else:
             raise AssertionError("unknown atom")
 
@@ -1361,22 +1361,22 @@
         return ast.ListComp(elt, comps, listcomp_node.get_lineno(),
                             listcomp_node.get_column())
 
-    def handle_setcomp(self, set_maker):
+    def handle_setcomp(self, set_maker, atom_node):
         ch = set_maker.get_child(0)
         elt = self.handle_expr(ch)
         if isinstance(elt, ast.Starred):
             self.error("iterable unpacking cannot be used in comprehension", ch)
         comps = self.comprehension_helper(set_maker.get_child(1))
-        return ast.SetComp(elt, comps, set_maker.get_lineno(),
-                           set_maker.get_column())
+        return ast.SetComp(elt, comps, atom_node.get_lineno(),
+                                       atom_node.get_column())
 
-    def handle_dictcomp(self, dict_maker):
+    def handle_dictcomp(self, dict_maker, atom_node):
         i, key, value = self.handle_dictelement(dict_maker, 0)
         comps = self.comprehension_helper(dict_maker.get_child(i))
-        return ast.DictComp(key, value, comps, dict_maker.get_lineno(),
-                            dict_maker.get_column())
+        return ast.DictComp(key, value, comps, atom_node.get_lineno(),
+                                               atom_node.get_column())
     
-    def handle_dictdisplay(self, node):
+    def handle_dictdisplay(self, node, atom_node):
         keys = []
         values = []
         i = 0
@@ -1385,16 +1385,18 @@
             keys.append(key)
             values.append(value)
             i += 1
-        return ast.Dict(keys, values, node.get_lineno(), node.get_column())
+        return ast.Dict(keys, values, atom_node.get_lineno(),
+                                      atom_node.get_column())
     
-    def handle_setdisplay(self, node):
+    def handle_setdisplay(self, node, atom_node):
         elts = []
         i = 0
         while i < node.num_children():
             expr = self.handle_expr(node.get_child(i))
             elts.append(expr)
             i += 2
-        return ast.Set(elts, node.get_lineno(), node.get_column())
+        return ast.Set(elts, atom_node.get_lineno(),
+                             atom_node.get_column())
 
     def handle_exprlist(self, exprlist, context):
         exprs = []

From pypy.commits at gmail.com  Tue Jan 24 12:28:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 09:28:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: raise a regular SyntaxError
Message-ID: <58878ea8.e4361c0a.2f8c9.3d4f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89753:cee6cd3d91ce
Date: 2017-01-24 18:27 +0100
http://bitbucket.org/pypy/pypy/changeset/cee6cd3d91ce/

Log:	raise a regular SyntaxError

diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -294,13 +294,13 @@
 
     space = astbuilder.space
     if not space.config.objspace.fstrings:
-        raise oefmt(space.w_SystemError,
-                    "f-strings have been disabled in this version of pypy "
-                    "with the translation option --no-objspace-fstrings.  "
-                    "The PyPy team (and CPython) thinks f-strings don't "
-                    "add any security risks, but we leave it to you to "
-                    "convince whoever translated this pypy that it is "
-                    "really the case")
+        raise astbuilder.error(
+            "f-strings have been disabled in this version of pypy "
+            "with the translation option '--no-objspace-fstrings'.  "
+            "The PyPy team (and CPython) thinks f-strings don't "
+            "add any security risks, but we leave it to you to "
+            "convince whoever translated this pypy that it is "
+            "really the case", atom_node)
 
     while True:
         literal, expr = fstring_find_literal_and_expr(astbuilder, fstr,

From pypy.commits at gmail.com  Tue Jan 24 18:25:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 15:25:40 -0800 (PST)
Subject: [pypy-commit] cffi default: Improve the error message for
 getattr/setattr
Message-ID: <5887e274.8c7e1c0a.a8214.a4b3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2873:8f9136ac88e6
Date: 2017-01-25 00:25 +0100
http://bitbucket.org/cffi/cffi/changeset/8f9136ac88e6/

Log:	Improve the error message for getattr/setattr

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -455,6 +455,8 @@
 static PyObject *
 get_field_name(CTypeDescrObject *ct, CFieldObject *cf);   /* forward */
 
+/* returns 0 if the struct ctype is opaque, 1 if it is not, or -1 if
+   an exception occurs */
 #define force_lazy_struct(ct)                                           \
     ((ct)->ct_stuff != NULL ? 1 : do_realize_lazy_struct(ct))
 
@@ -2452,11 +2454,26 @@
     return _cdata_add_or_sub(v, w, -1);
 }
 
+static void
+_cdata_attr_errmsg(char *errmsg, CDataObject *cd, PyObject *attr)
+{
+    char *text;
+    if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+        return;
+    PyErr_Clear();
+    text = PyText_AsUTF8(attr);
+    if (text == NULL)
+        return;
+    PyErr_Format(PyExc_AttributeError, errmsg, cd->c_type->ct_name, text);
+}
+
 static PyObject *
 cdata_getattro(CDataObject *cd, PyObject *attr)
 {
     CFieldObject *cf;
     CTypeDescrObject *ct = cd->c_type;
+    char *errmsg = "cdata '%s' has no attribute '%s'";
+    PyObject *x;
 
     if (ct->ct_flags & CT_POINTER)
         ct = ct->ct_itemdescr;
@@ -2488,14 +2505,19 @@
                 return new_simple_cdata(data,
                     (CTypeDescrObject *)cf->cf_type->ct_stuff);
             }
+            errmsg = "cdata '%s' has no field '%s'";
             break;
         case -1:
             return NULL;
         default:
+            errmsg = "cdata '%s' points to an opaque type: cannot read fields";
             break;
         }
     }
-    return PyObject_GenericGetAttr((PyObject *)cd, attr);
+    x = PyObject_GenericGetAttr((PyObject *)cd, attr);
+    if (x == NULL)
+        _cdata_attr_errmsg(errmsg, cd, attr);
+    return x;
 }
 
 static int
@@ -2503,6 +2525,8 @@
 {
     CFieldObject *cf;
     CTypeDescrObject *ct = cd->c_type;
+    char *errmsg = "cdata '%s' has no attribute '%s'";
+    int x;
 
     if (ct->ct_flags & CT_POINTER)
         ct = ct->ct_itemdescr;
@@ -2522,14 +2546,19 @@
                     return -1;
                 }
             }
+            errmsg = "cdata '%s' has no field '%s'";
             break;
         case -1:
             return -1;
         default:
+            errmsg = "cdata '%s' points to an opaque type: cannot write fields";
             break;
         }
     }
-    return PyObject_GenericSetAttr((PyObject *)cd, attr, value);
+    x = PyObject_GenericSetAttr((PyObject *)cd, attr, value);
+    if (x < 0)
+        _cdata_attr_errmsg(errmsg, cd, attr);
+    return x;
 }
 
 static PyObject *
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -748,8 +748,14 @@
     BInt = new_primitive_type("int")
     BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
-    p = cast(BStructPtr, 0)
-    py.test.raises(AttributeError, "p.a1")    # opaque
+    p = cast(BStructPtr, 42)
+    e = py.test.raises(AttributeError, "p.a1")    # opaque
+    assert str(e.value) == ("cdata 'struct foo *' points to an opaque type: "
+                            "cannot read fields")
+    e = py.test.raises(AttributeError, "p.a1 = 10")    # opaque
+    assert str(e.value) == ("cdata 'struct foo *' points to an opaque type: "
+                            "cannot write fields")
+
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
     p = newp(BStructPtr, None)
@@ -760,8 +766,24 @@
     assert s.a2 == 123
     py.test.raises(OverflowError, "s.a1 = sys.maxsize+1")
     assert s.a1 == 0
-    py.test.raises(AttributeError, "p.foobar")
-    py.test.raises(AttributeError, "s.foobar")
+    e = py.test.raises(AttributeError, "p.foobar")
+    assert str(e.value) == "cdata 'struct foo *' has no field 'foobar'"
+    e = py.test.raises(AttributeError, "p.foobar = 42")
+    assert str(e.value) == "cdata 'struct foo *' has no field 'foobar'"
+    e = py.test.raises(AttributeError, "s.foobar")
+    assert str(e.value) == "cdata 'struct foo' has no field 'foobar'"
+    e = py.test.raises(AttributeError, "s.foobar = 42")
+    assert str(e.value) == "cdata 'struct foo' has no field 'foobar'"
+    j = cast(BInt, 42)
+    e = py.test.raises(AttributeError, "j.foobar")
+    assert str(e.value) == "cdata 'int' has no attribute 'foobar'"
+    e = py.test.raises(AttributeError, "j.foobar = 42")
+    assert str(e.value) == "cdata 'int' has no attribute 'foobar'"
+    j = cast(new_pointer_type(BInt), 42)
+    e = py.test.raises(AttributeError, "j.foobar")
+    assert str(e.value) == "cdata 'int *' has no attribute 'foobar'"
+    e = py.test.raises(AttributeError, "j.foobar = 42")
+    assert str(e.value) == "cdata 'int *' has no attribute 'foobar'"
 
 def test_union_instance():
     BInt = new_primitive_type("int")

From pypy.commits at gmail.com  Tue Jan 24 18:53:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 15:53:31 -0800 (PST)
Subject: [pypy-commit] cffi default: extra tests
Message-ID: <5887e8fb.08061c0a.66788.0b3f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2874:ece6f0f2da93
Date: 2017-01-25 00:44 +0100
http://bitbucket.org/cffi/cffi/changeset/ece6f0f2da93/

Log:	extra tests

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -784,6 +784,11 @@
     assert str(e.value) == "cdata 'int *' has no attribute 'foobar'"
     e = py.test.raises(AttributeError, "j.foobar = 42")
     assert str(e.value) == "cdata 'int *' has no attribute 'foobar'"
+    pp = newp(new_pointer_type(BStructPtr), p)
+    e = py.test.raises(AttributeError, "pp.a1")
+    assert str(e.value) == "cdata 'struct foo * *' has no attribute 'a1'"
+    e = py.test.raises(AttributeError, "pp.a1 = 42")
+    assert str(e.value) == "cdata 'struct foo * *' has no attribute 'a1'"
 
 def test_union_instance():
     BInt = new_primitive_type("int")

From pypy.commits at gmail.com  Tue Jan 24 18:54:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 24 Jan 2017 15:54:58 -0800 (PST)
Subject: [pypy-commit] pypy default: update to cffi/ece6f0f2da93
Message-ID: <5887e952.01af1c0a.67eec.9b9b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89754:f6c083857478
Date: 2017-01-25 00:53 +0100
http://bitbucket.org/pypy/pypy/changeset/f6c083857478/

Log:	update to cffi/ece6f0f2da93

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -323,17 +323,28 @@
         #
         return self._add_or_sub(w_other, -1)
 
-    def getcfield(self, w_attr):
-        return self.ctype.getcfield(self.space.str_w(w_attr))
+    def getcfield(self, w_attr, mode):
+        space = self.space
+        attr = space.str_w(w_attr)
+        try:
+            cfield = self.ctype.getcfield(attr)
+        except KeyError:
+            raise oefmt(space.w_AttributeError, "cdata '%s' has no field '%s'",
+                        self.ctype.name, attr)
+        if cfield is None:
+            raise oefmt(space.w_AttributeError,
+                        "cdata '%s' points to an opaque type: cannot %s fields",
+                        self.ctype.name, mode)
+        return cfield
 
     def getattr(self, w_attr):
-        cfield = self.getcfield(w_attr)
+        cfield = self.getcfield(w_attr, mode="read")
         with self as ptr:
             w_res = cfield.read(ptr, self)
         return w_res
 
     def setattr(self, w_attr, w_value):
-        cfield = self.getcfield(w_attr)
+        cfield = self.getcfield(w_attr, mode="write")
         with self as ptr:
             cfield.write(ptr, w_value)
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -345,7 +345,10 @@
         return result
 
     def getcfield(self, attr):
-        return self.ctitem.getcfield(attr)
+        from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
+        if isinstance(self.ctitem, W_CTypeStructOrUnion):
+            return self.ctitem.getcfield(attr)
+        return W_CType.getcfield(self, attr)
 
     def typeoffsetof_field(self, fieldname, following):
         if following == 0:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -161,18 +161,18 @@
         return self._fields_dict[attr]
 
     def getcfield(self, attr):
-        ready = self._fields_dict is not None
-        if not ready and self.size >= 0:
+        # Returns a W_CField.  Error cases: returns None if we are an
+        # opaque struct; or raises KeyError if the particular field
+        # 'attr' does not exist.  The point of not directly building the
+        # error here is to get the exact ctype in the error message: it
+        # might be of the kind 'struct foo' or 'struct foo *'.
+        if self._fields_dict is None:
+            if self.size < 0:
+                return None
             self.force_lazy_struct()
-            ready = True
-        if ready:
-            self = jit.promote(self)
-            attr = jit.promote_string(attr)
-            try:
-                return self._getcfield_const(attr)
-            except KeyError:
-                pass
-        return W_CType.getcfield(self, attr)
+        self = jit.promote(self)
+        attr = jit.promote_string(attr)
+        return self._getcfield_const(attr)    # <= KeyError here
 
     def cdata_dir(self):
         if self.size < 0:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -737,8 +737,14 @@
     BInt = new_primitive_type("int")
     BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
-    p = cast(BStructPtr, 0)
-    py.test.raises(AttributeError, "p.a1")    # opaque
+    p = cast(BStructPtr, 42)
+    e = py.test.raises(AttributeError, "p.a1")    # opaque
+    assert str(e.value) == ("cdata 'struct foo *' points to an opaque type: "
+                            "cannot read fields")
+    e = py.test.raises(AttributeError, "p.a1 = 10")    # opaque
+    assert str(e.value) == ("cdata 'struct foo *' points to an opaque type: "
+                            "cannot write fields")
+
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
     p = newp(BStructPtr, None)
@@ -749,8 +755,29 @@
     assert s.a2 == 123
     py.test.raises(OverflowError, "s.a1 = sys.maxsize+1")
     assert s.a1 == 0
-    py.test.raises(AttributeError, "p.foobar")
-    py.test.raises(AttributeError, "s.foobar")
+    e = py.test.raises(AttributeError, "p.foobar")
+    assert str(e.value) == "cdata 'struct foo *' has no field 'foobar'"
+    e = py.test.raises(AttributeError, "p.foobar = 42")
+    assert str(e.value) == "cdata 'struct foo *' has no field 'foobar'"
+    e = py.test.raises(AttributeError, "s.foobar")
+    assert str(e.value) == "cdata 'struct foo' has no field 'foobar'"
+    e = py.test.raises(AttributeError, "s.foobar = 42")
+    assert str(e.value) == "cdata 'struct foo' has no field 'foobar'"
+    j = cast(BInt, 42)
+    e = py.test.raises(AttributeError, "j.foobar")
+    assert str(e.value) == "cdata 'int' has no attribute 'foobar'"
+    e = py.test.raises(AttributeError, "j.foobar = 42")
+    assert str(e.value) == "cdata 'int' has no attribute 'foobar'"
+    j = cast(new_pointer_type(BInt), 42)
+    e = py.test.raises(AttributeError, "j.foobar")
+    assert str(e.value) == "cdata 'int *' has no attribute 'foobar'"
+    e = py.test.raises(AttributeError, "j.foobar = 42")
+    assert str(e.value) == "cdata 'int *' has no attribute 'foobar'"
+    pp = newp(new_pointer_type(BStructPtr), p)
+    e = py.test.raises(AttributeError, "pp.a1")
+    assert str(e.value) == "cdata 'struct foo * *' has no attribute 'a1'"
+    e = py.test.raises(AttributeError, "pp.a1 = 42")
+    assert str(e.value) == "cdata 'struct foo * *' has no attribute 'a1'"
 
 def test_union_instance():
     BInt = new_primitive_type("int")

From pypy.commits at gmail.com  Tue Jan 24 23:40:56 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Tue, 24 Jan 2017 20:40:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add test_fstring
Message-ID: <58882c58.45aadf0a.26af1.b376@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r89755:62d461b8e4a2
Date: 2017-01-24 20:40 -0800
http://bitbucket.org/pypy/pypy/changeset/62d461b8e4a2/

Log:	add test_fstring

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -225,6 +225,7 @@
     RegrTest('test_format.py', core=True),
     RegrTest('test_fractions.py'),
     RegrTest('test_frame.py'),
+    RegrTest('test_fstring.py'),
     RegrTest('test_ftplib.py'),
     RegrTest('test_funcattrs.py', core=True),
     RegrTest('test_functools.py'),

From pypy.commits at gmail.com  Wed Jan 25 04:45:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 01:45:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix test, with explanation
Message-ID: <588873ce.ccaddf0a.320ea.11b8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89756:a780a04a65e8
Date: 2017-01-25 10:40 +0100
http://bitbucket.org/pypy/pypy/changeset/a780a04a65e8/

Log:	Fix test, with explanation

diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py
--- a/lib-python/3/test/test_fstring.py
+++ b/lib-python/3/test/test_fstring.py
@@ -541,29 +541,43 @@
         self.assertEqual(f'{f"{y}"*3}', '555')
 
     def test_invalid_string_prefixes(self):
-        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
-                            ["fu''",
-                             "uf''",
-                             "Fu''",
-                             "fU''",
-                             "Uf''",
-                             "uF''",
-                             "ufr''",
-                             "urf''",
-                             "fur''",
-                             "fru''",
-                             "rfu''",
-                             "ruf''",
-                             "FUR''",
-                             "Fur''",
-                             "fb''",
-                             "fB''",
-                             "Fb''",
-                             "FB''",
-                             "bf''",
-                             "bF''",
-                             "Bf''",
-                             "BF''",
+        # CPython checks that "fu''" etc. all gives 'unexpected EOF while
+        # parsing'.  Why doesn't it give 'invalid syntax'?  In fact:
+        #   eval("fu''")     => unexpected EOF while parsing
+        #   exec("fu''")     => invalid syntax
+        #   eval("fu'xxx'")  => invalid syntax
+        #   exec("fu'xxx'")  => invalid syntax
+        #   eval("fu''''''") => invalid syntax
+        #   exec("fu''''''") => invalid syntax
+        # so CPython does give an 'invalid syntax' in all cases except
+        # a very corner case, and it happens to be the one tested for
+        # here in CPython's version of the test.
+        #
+        # In light of that, I changed the test to check for a regular
+        # 'invalid syntax' case, which works on both CPython and PyPy.
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["fu' '",
+                             "uf' '",
+                             "Fu' '",
+                             "fU' '",
+                             "Uf' '",
+                             "uF' '",
+                             "ufr' '",
+                             "urf' '",
+                             "fur' '",
+                             "fru' '",
+                             "rfu' '",
+                             "ruf' '",
+                             "FUR' '",
+                             "Fur' '",
+                             "fb' '",
+                             "fB' '",
+                             "Fb' '",
+                             "FB' '",
+                             "bf' '",
+                             "bF' '",
+                             "Bf' '",
+                             "BF' '",
                              ])
 
     def test_leading_trailing_spaces(self):

From pypy.commits at gmail.com  Wed Jan 25 04:45:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 01:45:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Allow both CPython and PyPy error messages
Message-ID: <588873d0.2d88df0a.f10f0.0b65@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89757:7b730843efb4
Date: 2017-01-25 10:44 +0100
http://bitbucket.org/pypy/pypy/changeset/7b730843efb4/

Log:	Allow both CPython and PyPy error messages

diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py
--- a/lib-python/3/test/test_fstring.py
+++ b/lib-python/3/test/test_fstring.py
@@ -150,7 +150,7 @@
         # Inside of strings, don't interpret doubled brackets.
         self.assertEqual(f'{"{{}}"}', '{{}}')
 
-        self.assertAllRaise(TypeError, 'unhashable type',
+        self.assertAllRaise(TypeError, 'unhashable',
                             ["f'{ {{}} }'", # dict in a set
                              ])
 
@@ -413,7 +413,7 @@
 
         # lambda doesn't work without parens, because the colon
         #  makes the parser think it's a format_spec
-        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing|invalid syntax',
                             ["f'{lambda x:x}'",
                              ])
 
@@ -748,11 +748,11 @@
 
     def test_errors(self):
         # see issue 26287
-        self.assertAllRaise(TypeError, 'unsupported',
+        self.assertAllRaise(TypeError, 'unsupported|non-empty format string',
                             [r"f'{(lambda: 0):x}'",
                              r"f'{(0,):x}'",
                              ])
-        self.assertAllRaise(ValueError, 'Unknown format code',
+        self.assertAllRaise(ValueError, 'Unknown format code|unknown presentation',
                             [r"f'{1000:j}'",
                              r"f'{1000:j}'",
                             ])

From pypy.commits at gmail.com  Wed Jan 25 04:45:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 01:45:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <588873d2.84301c0a.f6198.699f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89758:6b430d93c851
Date: 2017-01-25 10:45 +0100
http://bitbucket.org/pypy/pypy/changeset/6b430d93c851/

Log:	merge heads

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -225,6 +225,7 @@
     RegrTest('test_format.py', core=True),
     RegrTest('test_fractions.py'),
     RegrTest('test_frame.py'),
+    RegrTest('test_fstring.py'),
     RegrTest('test_ftplib.py'),
     RegrTest('test_funcattrs.py', core=True),
     RegrTest('test_functools.py'),

From pypy.commits at gmail.com  Wed Jan 25 04:50:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 01:50:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: even better example
Message-ID: <588874e0.d4301c0a.46ee6.609f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89759:92a9e374d458
Date: 2017-01-25 10:49 +0100
http://bitbucket.org/pypy/pypy/changeset/92a9e374d458/

Log:	even better example

diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py
--- a/lib-python/3/test/test_fstring.py
+++ b/lib-python/3/test/test_fstring.py
@@ -545,10 +545,9 @@
         # parsing'.  Why doesn't it give 'invalid syntax'?  In fact:
         #   eval("fu''")     => unexpected EOF while parsing
         #   exec("fu''")     => invalid syntax
+        #   eval("fu'' ")    => invalid syntax
         #   eval("fu'xxx'")  => invalid syntax
-        #   exec("fu'xxx'")  => invalid syntax
         #   eval("fu''''''") => invalid syntax
-        #   exec("fu''''''") => invalid syntax
         # so CPython does give an 'invalid syntax' in all cases except
         # a very corner case, and it happens to be the one tested for
         # here in CPython's version of the test.

From pypy.commits at gmail.com  Wed Jan 25 05:43:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 02:43:07 -0800 (PST)
Subject: [pypy-commit] pypy default: Avoid translating a filled cache (worst
 case, it might keep alive some
Message-ID: <5888813b.0a8edf0a.71ab6.262f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89760:94763a48e468
Date: 2017-01-25 11:41 +0100
http://bitbucket.org/pypy/pypy/changeset/94763a48e468/

Log:	Avoid translating a filled cache (worst case, it might keep alive
	some long-dead objects which ends up as static data)

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -88,6 +88,9 @@
         for i in range(len(self.lookup_where)):
             self.lookup_where[i] = None_None
 
+    def _cleanup_(self):
+        self.clear()
+
 class _Global(object):
     weakref_warning_printed = False
 _global = _Global()

From pypy.commits at gmail.com  Wed Jan 25 05:54:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 02:54:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: issue #2471 fix
Message-ID: <588883e1.0d1a1c0a.7ab00.9264@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89761:965d1e442759
Date: 2017-01-25 11:47 +0100
http://bitbucket.org/pypy/pypy/changeset/965d1e442759/

Log:	issue #2471 fix

diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -31,7 +31,7 @@
             filename2 = os.path.join(os.path.dirname(file), 'typeids.txt')
             if not os.path.exists(filename2):
                 data = zlib.decompress(gc.get_typeids_z())
-                f = open(filename2, 'w')
+                f = open(filename2, 'wb')
                 f.write(data)
                 f.close()
             filename2 = os.path.join(os.path.dirname(file), 'typeids.lst')
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -202,7 +202,7 @@
 def get_typeids_z(space):
     a = rgc.get_typeids_z()
     s = ''.join([a[i] for i in range(len(a))])
-    return space.wrap(s)
+    return space.newbytes(s)
 
 def get_typeids_list(space):
     l = rgc.get_typeids_list()

From pypy.commits at gmail.com  Wed Jan 25 06:31:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 03:31:21 -0800 (PST)
Subject: [pypy-commit] pypy default: A comment,
 and another cache that could be cleared
Message-ID: <58888c89.2687df0a.33952.4622@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89762:64e22023ba60
Date: 2017-01-25 12:30 +0100
http://bitbucket.org/pypy/pypy/changeset/64e22023ba60/

Log:	A comment, and another cache that could be cleared

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -436,6 +436,9 @@
         for i in range(len(self.cached_attrs)):
             self.cached_attrs[i] = None
 
+    def _cleanup_(self):
+        self.clear()
+
 # ____________________________________________________________
 # object implementation
 
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -71,6 +71,10 @@
 class MethodCache(object):
 
     def __init__(self, space):
+        # Note: these attributes never change which object they contain,
+        # so reading 'cache.versions' for example is constant-folded.
+        # The actual list in 'cache.versions' is not a constant, of
+        # course.
         SIZE = 1 << space.config.objspace.std.methodcachesizeexp
         self.versions = [None] * SIZE
         self.names = [None] * SIZE

From pypy.commits at gmail.com  Wed Jan 25 09:11:08 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 25 Jan 2017 06:11:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add failing mmap test
Message-ID: <5888b1fc.2ca9df0a.efe25.772c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89763:ac6a7be4dfd6
Date: 2017-01-25 14:10 +0000
http://bitbucket.org/pypy/pypy/changeset/ac6a7be4dfd6/

Log:	Add failing mmap test

diff --git a/pypy/module/cpyext/test/test_buffer.py b/pypy/module/cpyext/test/test_buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_buffer.py
@@ -0,0 +1,26 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+class AppTestMmap(AppTestCpythonExtensionBase):
+    spaceconfig = {
+        'usemodules': (AppTestCpythonExtensionBase.spaceconfig['usemodules']
+                       + ['mmap'])}
+    def test_mmap_buffer(self):
+        module = self.import_extension('mmap_buffer', [
+            ('isbuffer', 'METH_O',
+             """
+             Py_buffer view;
+
+             if (PyObject_GetBuffer(args, &view,
+                    PyBUF_ANY_CONTIGUOUS|PyBUF_WRITABLE) != 0) {
+                return NULL;
+             }
+             return PyLong_FromLong(1);
+             """)])
+        import os, mmap
+        tmpname = os.path.join(self.udir, 'test_mmap_buffer')
+        print(tmpname)
+        with open(tmpname, 'w+b') as f:
+            f.write(b'123')
+            f.flush()
+            m = mmap.mmap(f.fileno(), 3)
+            assert module.isbuffer(m) == 1

From pypy.commits at gmail.com  Wed Jan 25 09:37:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 25 Jan 2017 06:37:30 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Enable buffer interface for mmap.mmap
Message-ID: <5888b82a.c6d11c0a.c5e15.06e5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89764:0d0968dd57fc
Date: 2017-01-25 14:36 +0000
http://bitbucket.org/pypy/pypy/changeset/0d0968dd57fc/

Log:	Enable buffer interface for mmap.mmap

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -260,7 +260,7 @@
             raise mmap_error(space, e)
         return space.wrap(self)
 
-W_MMap.typedef = TypeDef("mmap.mmap",
+W_MMap.typedef = TypeDef("mmap.mmap", None, None, 'read-write',
     __new__ = interp2app(mmap),
     close = interp2app(W_MMap.close),
     read_byte = interp2app(W_MMap.read_byte),

From pypy.commits at gmail.com  Wed Jan 25 10:21:01 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 25 Jan 2017 07:21:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Enable missing buffer procs for
 _io.BytesIO, array.array and memoryview
Message-ID: <5888c25d.4395df0a.4a123.90a6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89765:32be445aee1d
Date: 2017-01-25 15:20 +0000
http://bitbucket.org/pypy/pypy/changeset/32be445aee1d/

Log:	Enable missing buffer procs for _io.BytesIO, array.array and
	memoryview

diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -203,7 +203,7 @@
             space.call_method(self.getdict(space), "update", w_dict)
 
 W_BytesIO.typedef = TypeDef(
-    '_io.BytesIO', W_BufferedIOBase.typedef,
+    '_io.BytesIO', W_BufferedIOBase.typedef, None, 'read-write',
     __new__  = interp2app(W_BytesIO.descr_new.im_func),
     __init__  = interp2app(W_BytesIO.descr_init),
 
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -539,7 +539,7 @@
             return space.wrap(s)
 
 W_ArrayBase.typedef = TypeDef(
-    'array.array',
+    'array.array', None, None, 'read-write',
     __new__ = interp2app(w_array),
 
     __len__ = interp2app(W_ArrayBase.descr_len),
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -696,7 +696,7 @@
     return flags & (MEMORYVIEW_SCALAR|MEMORYVIEW_C)
 
 W_MemoryView.typedef = TypeDef(
-    "memoryview",
+    "memoryview", None, None, "read-write",
     __doc__ = """\
 Create a new memoryview object which references the given object.
 """,

From pypy.commits at gmail.com  Wed Jan 25 10:48:08 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 25 Jan 2017 07:48:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Creating more than one objspace in cpyext
 breaks the tests
Message-ID: <5888c8b8.cda0df0a.22ee7.6d6a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89766:2dff0146f6fa
Date: 2017-01-25 15:47 +0000
http://bitbucket.org/pypy/pypy/changeset/2dff0146f6fa/

Log:	Creating more than one objspace in cpyext breaks the tests

diff --git a/pypy/module/cpyext/test/test_buffer.py b/pypy/module/cpyext/test/test_buffer.py
--- a/pypy/module/cpyext/test/test_buffer.py
+++ b/pypy/module/cpyext/test/test_buffer.py
@@ -1,9 +1,6 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 class AppTestMmap(AppTestCpythonExtensionBase):
-    spaceconfig = {
-        'usemodules': (AppTestCpythonExtensionBase.spaceconfig['usemodules']
-                       + ['mmap'])}
     def test_mmap_buffer(self):
         module = self.import_extension('mmap_buffer', [
             ('isbuffer', 'METH_O',
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -87,7 +87,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
-                                   'itertools', 'time', 'binascii',
+                                   'itertools', 'time', 'binascii', 'mmap',
                                    ])
 
     enable_leak_checking = True

From pypy.commits at gmail.com  Wed Jan 25 11:05:13 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 25 Jan 2017 08:05:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Resync slot table with CPython 3.5
Message-ID: <5888ccb9.1282df0a.6360e.aa58@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89767:11c714b2a414
Date: 2017-01-25 16:04 +0000
http://bitbucket.org/pypy/pypy/changeset/11c714b2a414/

Log:	Resync slot table with CPython 3.5

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -789,6 +789,8 @@
 
 ETSLOT = TPSLOT
 
+def AMSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC):
+    return ETSLOT(NAME, "tp_as_async.c_" + SLOT, FUNCTION, WRAPPER, DOC)
 def SQSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC):
     return ETSLOT(NAME, "tp_as_sequence.c_" + SLOT, FUNCTION, WRAPPER, DOC)
 def MPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC):
@@ -827,161 +829,186 @@
 # Done.
 slotdefs_str = r"""
 static slotdef slotdefs[] = {
-        SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc,
-               "x.__len__() <==> len(x)"),
-        SQSLOT("__add__", sq_concat, slot_sq_concat, wrap_binaryfunc,
-          "x.__add__(y) <==> x+y"),
-        SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc,
-          "x.__mul__(n) <==> x*n"),
-        SQSLOT("__rmul__", sq_repeat, NULL, wrap_indexargfunc,
-          "x.__rmul__(n) <==> n*x"),
-        SQSLOT("__getitem__", sq_item, slot_sq_item, wrap_sq_item,
-               "x.__getitem__(y) <==> x[y]"),
-        SQSLOT("__setitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_setitem,
-               "x.__setitem__(i, y) <==> x[i]=y"),
-        SQSLOT("__delitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_delitem,
-               "x.__delitem__(y) <==> del x[y]"),
-        SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc,
-               "x.__contains__(y) <==> y in x"),
-        SQSLOT("__iadd__", sq_inplace_concat, NULL,
-          wrap_binaryfunc, "x.__iadd__(y) <==> x+=y"),
-        SQSLOT("__imul__", sq_inplace_repeat, NULL,
-          wrap_indexargfunc, "x.__imul__(y) <==> x*=y"),
+    TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""),
+    TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""),
+    TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""),
+    TPSLOT("__delattr__", tp_setattr, NULL, NULL, ""),
+    TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc,
+           "__repr__($self, /)\n--\n\nReturn repr(self)."),
+    TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc,
+           "__hash__($self, /)\n--\n\nReturn hash(self)."),
+    FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call,
+           "__call__($self, /, *args, **kwargs)\n--\n\nCall self as a function.",
+           PyWrapperFlag_KEYWORDS),
+    TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc,
+           "__str__($self, /)\n--\n\nReturn str(self)."),
+    TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook,
+           wrap_binaryfunc,
+           "__getattribute__($self, name, /)\n--\n\nReturn getattr(self, name)."),
+    TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""),
+    TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr,
+           "__setattr__($self, name, value, /)\n--\n\nImplement setattr(self, name, value)."),
+    TPSLOT("__delattr__", tp_setattro, slot_tp_setattro, wrap_delattr,
+           "__delattr__($self, name, /)\n--\n\nImplement delattr(self, name)."),
+    TPSLOT("__lt__", tp_richcompare, slot_tp_richcompare, richcmp_lt,
+           "__lt__($self, value, /)\n--\n\nReturn self<value."),
+    TPSLOT("__le__", tp_richcompare, slot_tp_richcompare, richcmp_le,
+           "__le__($self, value, /)\n--\n\nReturn self<=value."),
+    TPSLOT("__eq__", tp_richcompare, slot_tp_richcompare, richcmp_eq,
+           "__eq__($self, value, /)\n--\n\nReturn self==value."),
+    TPSLOT("__ne__", tp_richcompare, slot_tp_richcompare, richcmp_ne,
+           "__ne__($self, value, /)\n--\n\nReturn self!=value."),
+    TPSLOT("__gt__", tp_richcompare, slot_tp_richcompare, richcmp_gt,
+           "__gt__($self, value, /)\n--\n\nReturn self>value."),
+    TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge,
+           "__ge__($self, value, /)\n--\n\nReturn self>=value."),
+    TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc,
+           "__iter__($self, /)\n--\n\nImplement iter(self)."),
+    TPSLOT("__next__", tp_iternext, slot_tp_iternext, wrap_next,
+           "__next__($self, /)\n--\n\nImplement next(self)."),
+    TPSLOT("__get__", tp_descr_get, slot_tp_descr_get, wrap_descr_get,
+           "__get__($self, instance, owner, /)\n--\n\nReturn an attribute of instance, which is of type owner."),
+    TPSLOT("__set__", tp_descr_set, slot_tp_descr_set, wrap_descr_set,
+           "__set__($self, instance, value, /)\n--\n\nSet an attribute of instance to value."),
+    TPSLOT("__delete__", tp_descr_set, slot_tp_descr_set,
+           wrap_descr_delete,
+           "__delete__($self, instance, /)\n--\n\nDelete an attribute of instance."),
+    FLSLOT("__init__", tp_init, slot_tp_init, (wrapperfunc)wrap_init,
+           "__init__($self, /, *args, **kwargs)\n--\n\n"
+           "Initialize self.  See help(type(self)) for accurate signature.",
+           PyWrapperFlag_KEYWORDS),
+    TPSLOT("__new__", tp_new, slot_tp_new, NULL,
+           "__new__(type, /, *args, **kwargs)\n--\n\n"
+           "Create and return new object.  See help(type) for accurate signature."),
+    TPSLOT("__del__", tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""),
 
-        MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc,
-               "x.__len__() <==> len(x)"),
-        MPSLOT("__getitem__", mp_subscript, slot_mp_subscript,
-               wrap_binaryfunc,
-               "x.__getitem__(y) <==> x[y]"),
-        MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript,
-               wrap_objobjargproc,
-               "x.__setitem__(i, y) <==> x[i]=y"),
-        MPSLOT("__delitem__", mp_ass_subscript, slot_mp_ass_subscript,
-               wrap_delitem,
-               "x.__delitem__(y) <==> del x[y]"),
+    AMSLOT("__await__", am_await, slot_am_await, wrap_unaryfunc,
+           "__await__($self, /)\n--\n\nReturn an iterator to be used in await expression."),
+    AMSLOT("__aiter__", am_aiter, slot_am_aiter, wrap_unaryfunc,
+           "__aiter__($self, /)\n--\n\nReturn an awaitable, that resolves in asynchronous iterator."),
+    AMSLOT("__anext__", am_anext, slot_am_anext, wrap_unaryfunc,
+           "__anext__($self, /)\n--\n\nReturn a value or raise StopAsyncIteration."),
 
-        BINSLOT("__add__", nb_add, slot_nb_add,
-                "+"),
-        RBINSLOT("__radd__", nb_add, slot_nb_add,
-                 "+"),
-        BINSLOT("__sub__", nb_subtract, slot_nb_subtract,
-                "-"),
-        RBINSLOT("__rsub__", nb_subtract, slot_nb_subtract,
-                 "-"),
-        BINSLOT("__mul__", nb_multiply, slot_nb_multiply,
-                "*"),
-        RBINSLOT("__rmul__", nb_multiply, slot_nb_multiply,
-                 "*"),
-        BINSLOT("__mod__", nb_remainder, slot_nb_remainder,
-                "%"),
-        RBINSLOT("__rmod__", nb_remainder, slot_nb_remainder,
-                 "%"),
-        BINSLOTNOTINFIX("__divmod__", nb_divmod, slot_nb_divmod,
-                        "divmod(x, y)"),
-        RBINSLOTNOTINFIX("__rdivmod__", nb_divmod, slot_nb_divmod,
-                 "divmod(y, x)"),
-        NBSLOT("__pow__", nb_power, slot_nb_power, wrap_ternaryfunc,
-               "x.__pow__(y[, z]) <==> pow(x, y[, z])"),
-        NBSLOT("__rpow__", nb_power, slot_nb_power, wrap_ternaryfunc_r,
-               "y.__rpow__(x[, z]) <==> pow(x, y[, z])"),
-        UNSLOT("__neg__", nb_negative, slot_nb_negative, wrap_unaryfunc, "-x"),
-        UNSLOT("__pos__", nb_positive, slot_nb_positive, wrap_unaryfunc, "+x"),
-        UNSLOT("__abs__", nb_absolute, slot_nb_absolute, wrap_unaryfunc,
-               "abs(x)"),
-        UNSLOT("__bool__", nb_bool, slot_nb_bool, wrap_inquirypred,
-               "x != 0"),
-        UNSLOT("__invert__", nb_invert, slot_nb_invert, wrap_unaryfunc, "~x"),
-        BINSLOT("__lshift__", nb_lshift, slot_nb_lshift, "<<"),
-        RBINSLOT("__rlshift__", nb_lshift, slot_nb_lshift, "<<"),
-        BINSLOT("__rshift__", nb_rshift, slot_nb_rshift, ">>"),
-        RBINSLOT("__rrshift__", nb_rshift, slot_nb_rshift, ">>"),
-        BINSLOT("__and__", nb_and, slot_nb_and, "&"),
-        RBINSLOT("__rand__", nb_and, slot_nb_and, "&"),
-        BINSLOT("__xor__", nb_xor, slot_nb_xor, "^"),
-        RBINSLOT("__rxor__", nb_xor, slot_nb_xor, "^"),
-        BINSLOT("__or__", nb_or, slot_nb_or, "|"),
-        RBINSLOT("__ror__", nb_or, slot_nb_or, "|"),
-        UNSLOT("__int__", nb_int, slot_nb_int, wrap_unaryfunc,
-               "int(x)"),
-        UNSLOT("__float__", nb_float, slot_nb_float, wrap_unaryfunc,
-               "float(x)"),
-        NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc,
-               "x[y:z] <==> x[y.__index__():z.__index__()]"),
-        IBSLOT("__iadd__", nb_inplace_add, slot_nb_inplace_add,
-               wrap_binaryfunc, "+="),
-        IBSLOT("__isub__", nb_inplace_subtract, slot_nb_inplace_subtract,
-               wrap_binaryfunc, "-="),
-        IBSLOT("__imul__", nb_inplace_multiply, slot_nb_inplace_multiply,
-               wrap_binaryfunc, "*="),
-        IBSLOT("__imod__", nb_inplace_remainder, slot_nb_inplace_remainder,
-               wrap_binaryfunc, "%="),
-        IBSLOT("__ipow__", nb_inplace_power, slot_nb_inplace_power,
-               wrap_binaryfunc, "**="),
-        IBSLOT("__ilshift__", nb_inplace_lshift, slot_nb_inplace_lshift,
-               wrap_binaryfunc, "<<="),
-        IBSLOT("__irshift__", nb_inplace_rshift, slot_nb_inplace_rshift,
-               wrap_binaryfunc, ">>="),
-        IBSLOT("__iand__", nb_inplace_and, slot_nb_inplace_and,
-               wrap_binaryfunc, "&="),
-        IBSLOT("__ixor__", nb_inplace_xor, slot_nb_inplace_xor,
-               wrap_binaryfunc, "^="),
-        IBSLOT("__ior__", nb_inplace_or, slot_nb_inplace_or,
-               wrap_binaryfunc, "|="),
-        BINSLOT("__floordiv__", nb_floor_divide, slot_nb_floor_divide, "//"),
-        RBINSLOT("__rfloordiv__", nb_floor_divide, slot_nb_floor_divide, "//"),
-        BINSLOT("__truediv__", nb_true_divide, slot_nb_true_divide, "/"),
-        RBINSLOT("__rtruediv__", nb_true_divide, slot_nb_true_divide, "/"),
-        IBSLOT("__ifloordiv__", nb_inplace_floor_divide,
-               slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"),
-        IBSLOT("__itruediv__", nb_inplace_true_divide,
-               slot_nb_inplace_true_divide, wrap_binaryfunc, "/"),
+    BINSLOT("__add__", nb_add, slot_nb_add,
+           "+"),
+    RBINSLOT("__radd__", nb_add, slot_nb_add,
+           "+"),
+    BINSLOT("__sub__", nb_subtract, slot_nb_subtract,
+           "-"),
+    RBINSLOT("__rsub__", nb_subtract, slot_nb_subtract,
+           "-"),
+    BINSLOT("__mul__", nb_multiply, slot_nb_multiply,
+           "*"),
+    RBINSLOT("__rmul__", nb_multiply, slot_nb_multiply,
+           "*"),
+    BINSLOT("__mod__", nb_remainder, slot_nb_remainder,
+           "%"),
+    RBINSLOT("__rmod__", nb_remainder, slot_nb_remainder,
+           "%"),
+    BINSLOTNOTINFIX("__divmod__", nb_divmod, slot_nb_divmod,
+           "Return divmod(self, value)."),
+    RBINSLOTNOTINFIX("__rdivmod__", nb_divmod, slot_nb_divmod,
+           "Return divmod(value, self)."),
+    NBSLOT("__pow__", nb_power, slot_nb_power, wrap_ternaryfunc,
+           "__pow__($self, value, mod=None, /)\n--\n\nReturn pow(self, value, mod)."),
+    NBSLOT("__rpow__", nb_power, slot_nb_power, wrap_ternaryfunc_r,
+           "__rpow__($self, value, mod=None, /)\n--\n\nReturn pow(value, self, mod)."),
+    UNSLOT("__neg__", nb_negative, slot_nb_negative, wrap_unaryfunc, "-self"),
+    UNSLOT("__pos__", nb_positive, slot_nb_positive, wrap_unaryfunc, "+self"),
+    UNSLOT("__abs__", nb_absolute, slot_nb_absolute, wrap_unaryfunc,
+           "abs(self)"),
+    UNSLOT("__bool__", nb_bool, slot_nb_bool, wrap_inquirypred,
+           "self != 0"),
+    UNSLOT("__invert__", nb_invert, slot_nb_invert, wrap_unaryfunc, "~self"),
+    BINSLOT("__lshift__", nb_lshift, slot_nb_lshift, "<<"),
+    RBINSLOT("__rlshift__", nb_lshift, slot_nb_lshift, "<<"),
+    BINSLOT("__rshift__", nb_rshift, slot_nb_rshift, ">>"),
+    RBINSLOT("__rrshift__", nb_rshift, slot_nb_rshift, ">>"),
+    BINSLOT("__and__", nb_and, slot_nb_and, "&"),
+    RBINSLOT("__rand__", nb_and, slot_nb_and, "&"),
+    BINSLOT("__xor__", nb_xor, slot_nb_xor, "^"),
+    RBINSLOT("__rxor__", nb_xor, slot_nb_xor, "^"),
+    BINSLOT("__or__", nb_or, slot_nb_or, "|"),
+    RBINSLOT("__ror__", nb_or, slot_nb_or, "|"),
+    UNSLOT("__int__", nb_int, slot_nb_int, wrap_unaryfunc,
+           "int(self)"),
+    UNSLOT("__float__", nb_float, slot_nb_float, wrap_unaryfunc,
+           "float(self)"),
+    IBSLOT("__iadd__", nb_inplace_add, slot_nb_inplace_add,
+           wrap_binaryfunc, "+="),
+    IBSLOT("__isub__", nb_inplace_subtract, slot_nb_inplace_subtract,
+           wrap_binaryfunc, "-="),
+    IBSLOT("__imul__", nb_inplace_multiply, slot_nb_inplace_multiply,
+           wrap_binaryfunc, "*="),
+    IBSLOT("__imod__", nb_inplace_remainder, slot_nb_inplace_remainder,
+           wrap_binaryfunc, "%="),
+    IBSLOT("__ipow__", nb_inplace_power, slot_nb_inplace_power,
+           wrap_binaryfunc, "**="),
+    IBSLOT("__ilshift__", nb_inplace_lshift, slot_nb_inplace_lshift,
+           wrap_binaryfunc, "<<="),
+    IBSLOT("__irshift__", nb_inplace_rshift, slot_nb_inplace_rshift,
+           wrap_binaryfunc, ">>="),
+    IBSLOT("__iand__", nb_inplace_and, slot_nb_inplace_and,
+           wrap_binaryfunc, "&="),
+    IBSLOT("__ixor__", nb_inplace_xor, slot_nb_inplace_xor,
+           wrap_binaryfunc, "^="),
+    IBSLOT("__ior__", nb_inplace_or, slot_nb_inplace_or,
+           wrap_binaryfunc, "|="),
+    BINSLOT("__floordiv__", nb_floor_divide, slot_nb_floor_divide, "//"),
+    RBINSLOT("__rfloordiv__", nb_floor_divide, slot_nb_floor_divide, "//"),
+    BINSLOT("__truediv__", nb_true_divide, slot_nb_true_divide, "/"),
+    RBINSLOT("__rtruediv__", nb_true_divide, slot_nb_true_divide, "/"),
+    IBSLOT("__ifloordiv__", nb_inplace_floor_divide,
+           slot_nb_inplace_floor_divide, wrap_binaryfunc, "//="),
+    IBSLOT("__itruediv__", nb_inplace_true_divide,
+           slot_nb_inplace_true_divide, wrap_binaryfunc, "/="),
+    NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc,
+           "__index__($self, /)\n--\n\n"
+           "Return self converted to an integer, if self is suitable "
+           "for use as an index into a list."),
+    BINSLOT("__matmul__", nb_matrix_multiply, slot_nb_matrix_multiply,
+            "@"),
+    RBINSLOT("__rmatmul__", nb_matrix_multiply, slot_nb_matrix_multiply,
+             "@"),
+    IBSLOT("__imatmul__", nb_inplace_matrix_multiply, slot_nb_inplace_matrix_multiply,
+           wrap_binaryfunc, "@="),
+    MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc,
+           "__len__($self, /)\n--\n\nReturn len(self)."),
+    MPSLOT("__getitem__", mp_subscript, slot_mp_subscript,
+           wrap_binaryfunc,
+           "__getitem__($self, key, /)\n--\n\nReturn self[key]."),
+    MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript,
+           wrap_objobjargproc,
+           "__setitem__($self, key, value, /)\n--\n\nSet self[key] to value."),
+    MPSLOT("__delitem__", mp_ass_subscript, slot_mp_ass_subscript,
+           wrap_delitem,
+           "__delitem__($self, key, /)\n--\n\nDelete self[key]."),
 
-        TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc,
-               "x.__str__() <==> str(x)"),
-        TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc,
-               "x.__repr__() <==> repr(x)"),
-        TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc,
-               "x.__hash__() <==> hash(x)"),
-        FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call,
-               "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS),
-        TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook,
-               wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"),
-        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""),
-        TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr,
-               "x.__setattr__('name', value) <==> x.name = value"),
-        TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""),
-        TPSLOT("__delattr__", tp_setattro, slot_tp_setattro, wrap_delattr,
-               "x.__delattr__('name') <==> del x.name"),
-        TPSLOT("__delattr__", tp_setattr, NULL, NULL, ""),
-        TPSLOT("__lt__", tp_richcompare, slot_tp_richcompare, richcmp_lt,
-               "x.__lt__(y) <==> x<y"),
-        TPSLOT("__le__", tp_richcompare, slot_tp_richcompare, richcmp_le,
-               "x.__le__(y) <==> x<=y"),
-        TPSLOT("__eq__", tp_richcompare, slot_tp_richcompare, richcmp_eq,
-               "x.__eq__(y) <==> x==y"),
-        TPSLOT("__ne__", tp_richcompare, slot_tp_richcompare, richcmp_ne,
-               "x.__ne__(y) <==> x!=y"),
-        TPSLOT("__gt__", tp_richcompare, slot_tp_richcompare, richcmp_gt,
-               "x.__gt__(y) <==> x>y"),
-        TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge,
-               "x.__ge__(y) <==> x>=y"),
-        TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc,
-               "x.__iter__() <==> iter(x)"),
-        TPSLOT("__next__", tp_iternext, slot_tp_iternext, wrap_next,
-               "x.__next__() <==> next(x)"),
-        TPSLOT("__get__", tp_descr_get, slot_tp_descr_get, wrap_descr_get,
-               "descr.__get__(obj[, type]) -> value"),
-        TPSLOT("__set__", tp_descr_set, slot_tp_descr_set, wrap_descr_set,
-               "descr.__set__(obj, value)"),
-        TPSLOT("__delete__", tp_descr_set, slot_tp_descr_set,
-               wrap_descr_delete, "descr.__delete__(obj)"),
-        FLSLOT("__init__", tp_init, slot_tp_init, (wrapperfunc)wrap_init,
-               "x.__init__(...) initializes x; "
-               "see help(type(x)) for signature",
-               PyWrapperFlag_KEYWORDS),
-        TPSLOT("__new__", tp_new, slot_tp_new, NULL, ""),
-        TPSLOT("__del__", tp_del, slot_tp_del, NULL, ""),
-        {NULL}
+    SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc,
+           "__len__($self, /)\n--\n\nReturn len(self)."),
+
+    SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc,
+           "__add__($self, value, /)\n--\n\nReturn self+value."),
+    SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc,
+           "__mul__($self, value, /)\n--\n\nReturn self*value.n"),
+    SQSLOT("__rmul__", sq_repeat, NULL, wrap_indexargfunc,
+           "__rmul__($self, value, /)\n--\n\nReturn self*value."),
+    SQSLOT("__getitem__", sq_item, slot_sq_item, wrap_sq_item,
+           "__getitem__($self, key, /)\n--\n\nReturn self[key]."),
+    SQSLOT("__setitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_setitem,
+           "__setitem__($self, key, value, /)\n--\n\nSet self[key] to value."),
+    SQSLOT("__delitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_delitem,
+           "__delitem__($self, key, /)\n--\n\nDelete self[key]."),
+    SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc,
+           "__contains__($self, key, /)\n--\n\nReturn key in self."),
+    SQSLOT("__iadd__", sq_inplace_concat, NULL,
+           wrap_binaryfunc,
+           "__iadd__($self, value, /)\n--\n\nImplement self+=value."),
+    SQSLOT("__imul__", sq_inplace_repeat, NULL,
+           wrap_indexargfunc,
+           "__imul__($self, value, /)\n--\n\nImplement self*=value."),
+
+    {NULL}
 };
 """
 

From pypy.commits at gmail.com  Wed Jan 25 11:30:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 08:30:26 -0800 (PST)
Subject: [pypy-commit] pypy default: A siphash24 implementation based on
 CPython's, itself based on
Message-ID: <5888d2a2.4c861c0a.86e8.de05@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89768:01c2df465b1a
Date: 2017-01-25 17:29 +0100
http://bitbucket.org/pypy/pypy/changeset/01c2df465b1a/

Log:	A siphash24 implementation based on CPython's, itself based on
	https://github.com/majek/csiphash/

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rsiphash.py
@@ -0,0 +1,156 @@
+import sys, os, struct
+from contextlib import contextmanager
+from rpython.rlib import rarithmetic
+from rpython.rlib.objectmodel import not_rpython, always_inline
+from rpython.rlib.rarithmetic import r_uint64
+from rpython.rlib.rawstorage import misaligned_is_fine
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem.lloperation import llop
+
+
+if sys.byteorder == 'little':
+    def _le64toh(x):
+        return x
+else:
+    _le64toh = rarithmetic.byteswap
+
+
+# Initialize the values of the secret seed: two 64-bit constants.
+# CPython picks a new seed every time 'python' starts.  PyPy cannot do
+# that as easily because many details may rely on getting the same hash
+# value before and after translation.  We can, however, pick a random
+# seed once per translation, which should already be quite good.
+
+ at not_rpython
+def select_random_seed():
+    global k0, k1    # note: the globals k0, k1 are already byte-swapped
+    v0, v1 = struct.unpack("QQ", os.urandom(16))
+    k0 = r_uint64(v0)
+    k1 = r_uint64(v1)
+
+select_random_seed()
+
+ at contextmanager
+def choosen_seed(new_k0, new_k1, test_misaligned_path=False):
+    global k0, k1, misaligned_is_fine
+    old = k0, k1, misaligned_is_fine
+    k0 = _le64toh(r_uint64(new_k0))
+    k1 = _le64toh(r_uint64(new_k1))
+    if test_misaligned_path:
+        misaligned_is_fine = False
+    yield
+    k0, k1, misaligned_is_fine = old
+
+def get_current_seed():
+    return _le64toh(k0), _le64toh(k1)
+
+
+magic0 = r_uint64(0x736f6d6570736575)
+magic1 = r_uint64(0x646f72616e646f6d)
+magic2 = r_uint64(0x6c7967656e657261)
+magic3 = r_uint64(0x7465646279746573)
+
+
+ at always_inline
+def _rotate(x, b):
+    return (x << b) | (x >> (64 - b))
+
+ at always_inline
+def _half_round(a, b, c, d, s, t):
+    a += b
+    c += d
+    b = _rotate(b, s) ^ a
+    d = _rotate(d, t) ^ c
+    a = _rotate(a, 32)
+    return a, b, c, d
+
+ at always_inline
+def _double_round(v0, v1, v2, v3):
+    v0,v1,v2,v3 = _half_round(v0,v1,v2,v3,13,16)
+    v2,v1,v0,v3 = _half_round(v2,v1,v0,v3,17,21)
+    v0,v1,v2,v3 = _half_round(v0,v1,v2,v3,13,16)
+    v2,v1,v0,v3 = _half_round(v2,v1,v0,v3,17,21)
+    return v0, v1, v2, v3
+
+
+def siphash24(ptr, size):
+    """Takes a CCHARP pointer and a size.  Returns the hash as a r_uint64,
+    which can then be casted to the expected type."""
+
+    addr_in = llmemory.cast_ptr_to_adr(ptr)
+    direct = (misaligned_is_fine or
+                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
+
+    b = r_uint64(size) << 56
+    v0 = k0 ^ magic0
+    v1 = k1 ^ magic1
+    v2 = k0 ^ magic2
+    v3 = k1 ^ magic3
+
+    index = 0
+    if direct:
+        while size >= 8:
+            mi = llop.raw_load(rffi.ULONGLONG, addr_in, index)
+            mi = _le64toh(mi)
+            size -= 8
+            index += 8
+            v3 ^= mi
+            v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+            v0 ^= mi
+    else:
+        while size >= 8:
+            mi = (
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index)) |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1)) << 8 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2)) << 16 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4)) << 32 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5)) << 40 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 7)) << 56
+            )
+            mi = _le64toh(mi)
+            size -= 8
+            index += 8
+            v3 ^= mi
+            v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+            v0 ^= mi
+
+    t = r_uint64(0)
+    if size == 7:
+        t = r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48
+        size = 6
+    if size == 6:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5)) << 40
+        size = 5
+    if size == 5:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4)) << 32
+        size = 4
+    if size == 4:
+        if direct:
+            t |= r_uint64(llop.raw_load(rffi.UINT, addr_in, index))
+            size = 0
+        else:
+            t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24
+            size = 3
+    if size == 3:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2)) << 16
+        size = 2
+    if size == 2:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1)) << 8
+        size = 1
+    if size == 1:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index))
+        size = 0
+    assert size == 0
+
+    b |= _le64toh(t)
+
+    v3 ^= b
+    v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+    v0 ^= b
+    v2 ^= 0xff
+    v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+
+    return (v0 ^ v1) ^ (v2 ^ v3)
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -0,0 +1,44 @@
+from rpython.rlib.rsiphash import siphash24, choosen_seed
+from rpython.rtyper.lltypesystem import rffi
+
+
+CASES = [
+    (2323638336262702335 , ""),
+    (5150479602681463644 , "h"),
+    (1013213613370725794 , "he"),
+    (7028032310911240238 , "hel"),
+    (9535960132410784494 , "hell"),
+    (3256502711089771242 , "hello"),
+    (2389188832234450176 , "hello "),
+    (13253855839845990393, "hello w"),
+    (7850036019043917323 , "hello wo"),
+    (14283308628425005953, "hello wor"),
+    (9605549962279590084 , "hello worl"),
+    (16371281469632894235, "hello world"),
+    (7298637955795769949 , "hello world\x9a"),
+    (13530878135053370821, "hello world\xf3\x80"),
+    (1643533543579802994 , "\xffhel\x82lo world\xbc"),
+    (14632093238728197380, "hexlylxox rewqw"),
+    (3434253029196696424 , "hexlylxox rewqws"),
+    (9855754545877066788 , "hexlylxox rewqwsv"),
+    (5233065012564472454 , "hexlylxox rewqwkashdw89"),
+    (16768585622569081808, "hexlylxox rewqwkeashdw89"),
+    (17430482483431293463, "HEEExlylxox rewqwkashdw89"),
+    (695783005783737705  , "hello woadwealidewd 3829ez 32ig dxwaebderld"),
+]
+
+def check(s):
+    p = rffi.str2charp(s)
+    q = rffi.str2charp('?' + s)
+    with choosen_seed(0x8a9f065a358479f4, 0x11cb1e9ee7f40e1f,
+                      test_misaligned_path=True):
+        x = siphash24(p, len(s))
+        y = siphash24(rffi.ptradd(q, 1), len(s))
+    rffi.free_charp(p)
+    rffi.free_charp(q)
+    assert x == y
+    return x
+
+def test_siphash24():
+    for expected, string in CASES:
+        assert check(string) == expected

From pypy.commits at gmail.com  Wed Jan 25 14:06:11 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 25 Jan 2017 11:06:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Reimplement PyMemoryView_GET_BUFFER and
 PyMemoryView_GET_BASE as macros
Message-ID: <5888f723.9aa2df0a.4ef06.f96d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89769:25624675b30b
Date: 2017-01-25 19:05 +0000
http://bitbucket.org/pypy/pypy/changeset/25624675b30b/

Log:	Reimplement PyMemoryView_GET_BUFFER and PyMemoryView_GET_BASE as
	macros

diff --git a/pypy/module/cpyext/include/memoryobject.h b/pypy/module/cpyext/include/memoryobject.h
--- a/pypy/module/cpyext/include/memoryobject.h
+++ b/pypy/module/cpyext/include/memoryobject.h
@@ -14,6 +14,10 @@
 } PyMemoryViewObject;
 
 
+/* Get a pointer to the memoryview's private copy of the exporter's buffer. */
+#define PyMemoryView_GET_BUFFER(op) (&((PyMemoryViewObject *)(op))->view)
+/* Get a pointer to the exporting object (this may be NULL!). */
+#define PyMemoryView_GET_BASE(op) (((PyMemoryViewObject *)(op))->view.obj)
 
 
 #ifdef __cplusplus
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -118,15 +118,6 @@
             "a bytes-like object is required, not '%T'", w_exporter)
     return generic_cpy_call(space, pb.c_bf_getbuffer, exporter, view, flags)
 
- at cpython_api([PyObject], Py_bufferP, error=CANNOT_FAIL)
-def PyMemoryView_GET_BUFFER(space, pyobj):
-    """Return a pointer to the buffer-info structure wrapped by the given
-    object.  The object must be a memoryview instance; this macro doesn't
-    check its type, you must do it yourself or you will risk crashes."""
-    # XXX move to a c-macro
-    py_memobj = rffi.cast(PyMemoryViewObject, pyobj)
-    return py_memobj.c_view
-
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
     ndim = buf.getndim()
@@ -259,9 +250,3 @@
         py_mem.c_view.c_shape = view.c_shape
     # XXX ignore suboffsets?
     return py_obj
-
- at cpython_api([PyObject], PyObject)
-def PyMemoryView_GET_BASE(space, w_obj):
-    # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    # XXX needed for numpy on py3k
-    raise NotImplementedError('PyMemoryView_GET_BASE')
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -5,6 +5,7 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
 from pypy.module.cpyext.pyobject import from_ref
+from pypy.module.cpyext.memoryobject import PyMemoryViewObject
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
@@ -20,8 +21,10 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        w_memoryview = from_ref(space, api.PyMemoryView_FromObject(w_buf))
-        view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        c_memoryview = rffi.cast(
+            PyMemoryViewObject, api.PyMemoryView_FromObject(w_buf))
+        w_memoryview = from_ref(space, c_memoryview)
+        view = c_memoryview.c_view
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)
         assert f == 'B'

From pypy.commits at gmail.com  Wed Jan 25 14:45:36 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 25 Jan 2017 11:45:36 -0800 (PST)
Subject: [pypy-commit] pypy default: Reimplement PyMemoryView_GET_BUFFER and
 PyMemoryView_GET_BASE as macros
Message-ID: <58890060.09bb1c0a.e9e8b.5691@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89770:f475e5872e53
Date: 2017-01-25 21:41 +0200
http://bitbucket.org/pypy/pypy/changeset/f475e5872e53/

Log:	Reimplement PyMemoryView_GET_BUFFER and PyMemoryView_GET_BASE as
	macros

diff --git a/pypy/module/cpyext/include/memoryobject.h b/pypy/module/cpyext/include/memoryobject.h
--- a/pypy/module/cpyext/include/memoryobject.h
+++ b/pypy/module/cpyext/include/memoryobject.h
@@ -14,6 +14,10 @@
 } PyMemoryViewObject;
 
 
+/* Get a pointer to the memoryview's private copy of the exporter's buffer. */
+#define PyMemoryView_GET_BUFFER(op) (&((PyMemoryViewObject *)(op))->view)
+/* Get a pointer to the exporting object (this may be NULL!). */
+#define PyMemoryView_GET_BASE(op) (((PyMemoryViewObject *)(op))->view.obj)
 
 
 #ifdef __cplusplus
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -128,15 +128,6 @@
     view.c_obj = make_ref(space, w_obj)
     return ret
 
- at cpython_api([PyObject], Py_bufferP, error=CANNOT_FAIL)
-def PyMemoryView_GET_BUFFER(space, pyobj):
-    """Return a pointer to the buffer-info structure wrapped by the given
-    object.  The object must be a memoryview instance; this macro doesn't
-    check its type, you must do it yourself or you will risk crashes."""
-    # XXX move to a c-macro
-    py_memobj = rffi.cast(PyMemoryViewObject, pyobj)
-    return py_memobj.c_view
-
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
     ndim = buf.getndim()
@@ -269,9 +260,3 @@
     # XXX ignore suboffsets?
     return py_obj
 
- at cpython_api([PyObject], PyObject)
-def PyMemoryView_GET_BASE(space, w_obj):
-    # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    # XXX needed for numpy on py3k
-    raise NotImplementedError('PyMemoryView_GET_BASE')
-
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -5,6 +5,7 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
 from pypy.module.cpyext.pyobject import from_ref
+from pypy.module.cpyext.memoryobject import PyMemoryViewObject
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
@@ -20,8 +21,10 @@
 
     def test_frombuffer(self, space, api):
         w_buf = space.newbuffer(StringBuffer("hello"))
-        w_memoryview = from_ref(space, api.PyMemoryView_FromObject(w_buf))
-        view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        c_memoryview = rffi.cast(
+            PyMemoryViewObject, api.PyMemoryView_FromObject(w_buf))
+        w_memoryview = from_ref(space, c_memoryview)
+        view = c_memoryview.c_view
         assert view.c_ndim == 1
         f = rffi.charp2str(view.c_format)
         assert f == 'B'

From pypy.commits at gmail.com  Wed Jan 25 14:51:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 11:51:46 -0800 (PST)
Subject: [pypy-commit] pypy default: Implement Siphash-2-4,
 the same hashing function as CPython 3.x.
Message-ID: <588901d2.45aadf0a.26af1.1c34@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89771:134351c66421
Date: 2017-01-25 20:51 +0100
http://bitbucket.org/pypy/pypy/changeset/134351c66421/

Log:	Implement Siphash-2-4, the same hashing function as CPython 3.x.
	Disabled by default.

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -520,10 +520,22 @@
 # ----------
 
 HASH_ALGORITHM = "rpython"  # XXX Is there a better name?
+HASH_ALGORITHM_FIXED = False
 
-def _hash_string(s):
-    """The algorithm behind compute_hash() for a string or a unicode."""
+ at not_rpython
+def set_hash_algorithm(algo):
+    """Must be called very early, before any string is hashed with
+    compute_hash()!"""
+    global HASH_ALGORITHM
+    if HASH_ALGORITHM != algo:
+        assert not HASH_ALGORITHM_FIXED, "compute_hash() already called!"
+        assert algo in ("rpython", "siphash24")
+        HASH_ALGORITHM = algo
+
+
+def _hash_string_rpython(s):
     from rpython.rlib.rarithmetic import intmask
+
     length = len(s)
     if length == 0:
         return -1
@@ -535,6 +547,83 @@
     x ^= length
     return intmask(x)
 
+
+ at not_rpython
+def _hash_string_siphash24(s):
+    """This version is called when untranslated only."""
+    import array
+    from rpython.rlib.rsiphash import siphash24
+    from rpython.rtyper.lltypesystem import lltype, rffi
+    from rpython.rlib.rarithmetic import intmask
+
+    if isinstance(s, str):
+        pass
+    elif isinstance(s, unicode):
+        if rffi.sizeof(lltype.UniChar) == 4:
+            kind = "I"
+        else:
+            kind = "H"
+        s = array.array(kind, map(ord, s)).tostring()
+    else:
+        if lltype.typeOf(s).TO.chars.OF == lltype.Char:
+            kind = "B"
+        elif rffi.sizeof(lltype.UniChar) == 4:
+            kind = "I"
+        else:
+            kind = "H"
+        s = array.array(kind, map(ord, s.chars)).tostring()
+    ptr = rffi.str2charp(s)
+    x = siphash24(ptr, len(s))
+    rffi.free_charp(ptr)
+    return intmask(x)
+
+def ll_hash_string_siphash24(ll_s):
+    """Called from lltypesystem/rstr.py.  'll_s' is a rstr.STR or UNICODE."""
+    from rpython.rlib.rsiphash import siphash24
+    from rpython.rtyper.lltypesystem import lltype, rffi, rstr
+    from rpython.rlib.rarithmetic import intmask
+
+    length = len(ll_s.chars)
+    # no GC operation from here!
+    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:
+        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
+    else:
+        addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
+        length *= rffi.sizeof(rstr.UNICODE.chars.OF)
+    x = siphash24(addr, length)
+    keepalive_until_here(ll_s)
+    return intmask(x)
+ll_hash_string_siphash24._jit_look_inside_ = False
+
+
+ at not_rpython
+def _hash_string(s):
+    """The algorithm behind compute_hash() for a string or a unicode.
+    This version is only for untranslated usage, and 's' is a str or unicode.
+    """
+    global HASH_ALGORITHM_FIXED
+    HASH_ALGORITHM_FIXED = True
+    if HASH_ALGORITHM == "rpython":
+        return _hash_string_rpython(s)
+    if HASH_ALGORITHM == "siphash24":
+        return _hash_string_siphash24(s)
+    raise NotImplementedError
+
+def ll_hash_string(ll_s):
+    """The algorithm behind compute_hash() for a string or a unicode.
+    This version is called from lltypesystem/rstr.py, and 'll_s' is a
+    rstr.STR or rstr.UNICODE.
+    """
+    if HASH_ALGORITHM == "rpython":
+        return _hash_string_rpython(ll_s.chars)
+    if HASH_ALGORITHM == "siphash24":
+        if we_are_translated():
+            return ll_hash_string_siphash24(ll_s)
+        else:
+            return _hash_string_siphash24(ll_s)
+    raise NotImplementedError
+
+
 def _hash_float(f):
     """The algorithm behind compute_hash() for a float.
     This implementation is identical to the CPython implementation,
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -2,6 +2,7 @@
 from contextlib import contextmanager
 from rpython.rlib import rarithmetic
 from rpython.rlib.objectmodel import not_rpython, always_inline
+from rpython.rlib.rgc import no_collect
 from rpython.rlib.rarithmetic import r_uint64
 from rpython.rlib.rawstorage import misaligned_is_fine
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
@@ -73,11 +74,11 @@
     return v0, v1, v2, v3
 
 
-def siphash24(ptr, size):
-    """Takes a CCHARP pointer and a size.  Returns the hash as a r_uint64,
+ at no_collect
+def siphash24(addr_in, size):
+    """Takes an address pointer and a size.  Returns the hash as a r_uint64,
     which can then be casted to the expected type."""
 
-    addr_in = llmemory.cast_ptr_to_adr(ptr)
     direct = (misaligned_is_fine or
                  (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
 
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
--- a/rpython/rlib/test/test_rsiphash.py
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -1,5 +1,5 @@
 from rpython.rlib.rsiphash import siphash24, choosen_seed
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import llmemory, rffi
 
 
 CASES = [
@@ -32,8 +32,8 @@
     q = rffi.str2charp('?' + s)
     with choosen_seed(0x8a9f065a358479f4, 0x11cb1e9ee7f40e1f,
                       test_misaligned_path=True):
-        x = siphash24(p, len(s))
-        y = siphash24(rffi.ptradd(q, 1), len(s))
+        x = siphash24(llmemory.cast_ptr_to_adr(p), len(s))
+        y = siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
     rffi.free_charp(p)
     rffi.free_charp(q)
     assert x == y
diff --git a/rpython/rtyper/lltypesystem/rbytearray.py b/rpython/rtyper/lltypesystem/rbytearray.py
--- a/rpython/rtyper/lltypesystem/rbytearray.py
+++ b/rpython/rtyper/lltypesystem/rbytearray.py
@@ -8,10 +8,10 @@
 def mallocbytearray(size):
     return lltype.malloc(BYTEARRAY, size)
 
-_, _, copy_bytearray_contents = rstr._new_copy_contents_fun(BYTEARRAY, BYTEARRAY,
+_, _, copy_bytearray_contents, _ = rstr._new_copy_contents_fun(BYTEARRAY, BYTEARRAY,
                                                          lltype.Char,
                                                          'bytearray')
-_, _, copy_bytearray_contents_from_str = rstr._new_copy_contents_fun(rstr.STR,
+_, _, copy_bytearray_contents_from_str, _ = rstr._new_copy_contents_fun(rstr.STR,
                                                                   BYTEARRAY,
                                                                   lltype.Char,
                                                                   'bytearray_from_str')
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -3,7 +3,7 @@
 from rpython.annotator import model as annmodel
 from rpython.rlib import jit, types
 from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated,
-    _hash_string, keepalive_until_here, specialize, enforceargs)
+    ll_hash_string, keepalive_until_here, specialize, enforceargs)
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.error import TyperError
@@ -136,15 +136,19 @@
     copy_raw_to_string = func_with_new_name(copy_raw_to_string,
                                               'copy_raw_to_%s' % name)
 
-    return copy_string_to_raw, copy_raw_to_string, copy_string_contents
+    return (copy_string_to_raw, copy_raw_to_string, copy_string_contents,
+            _get_raw_buf)
 
 (copy_string_to_raw,
  copy_raw_to_string,
- copy_string_contents) = _new_copy_contents_fun(STR, STR, Char, 'string')
+ copy_string_contents,
+ _get_raw_buf_string) = _new_copy_contents_fun(STR, STR, Char, 'string')
 
 (copy_unicode_to_raw,
  copy_raw_to_unicode,
- copy_unicode_contents) = _new_copy_contents_fun(UNICODE, UNICODE, UniChar, 'unicode')
+ copy_unicode_contents,
+ _get_raw_buf_unicode) = _new_copy_contents_fun(UNICODE, UNICODE, UniChar,
+                                                'unicode')
 
 CONST_STR_CACHE = WeakValueDictionary()
 CONST_UNICODE_CACHE = WeakValueDictionary()
@@ -382,7 +386,7 @@
         # but our malloc initializes the memory to zero, so we use zero as the
         # special non-computed-yet value.  Also, jit.conditional_call_elidable
         # always checks for zero, for now.
-        x = _hash_string(s.chars)
+        x = ll_hash_string(s)
         if x == 0:
             x = 29872897
         s.hash = x
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -1,8 +1,12 @@
 from __future__ import with_statement
 
 import math
-import sys
+import sys, os
 
+if __name__ == '__main__':
+    # hack for test_hash_string_siphash24()
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__),
+                                    '..', '..', '..', '..'))
 import py
 
 from rpython.rlib.rstackovf import StackOverflow
@@ -597,6 +601,40 @@
         assert res[3] == compute_hash(d)
         assert res[4] == compute_hash(("Hi", None, (7.5, 2, d)))
 
+    def _test_hash_string(self, algo):
+        from rpython.rlib import objectmodel
+        objectmodel.set_hash_algorithm(algo)
+        s = "hello"
+        u = u"world"
+        hash_s = compute_hash(s)
+        hash_u = compute_hash(u)
+        #
+        def fn(length):
+            assert length >= 1
+            return str((compute_hash(s),
+                        compute_hash(u),
+                        compute_hash(s[0] + s[1:length]),
+                        compute_hash(u[0] + u[1:length])))
+
+        assert fn(5) == str((hash_s, hash_u, hash_s, hash_u))
+
+        f = self.getcompiled(fn, [int])
+        res = f(5)
+        res = [int(a) for a in res[1:-1].split(",")]
+        assert res[0] == hash_s
+        assert res[1] == hash_u
+        assert res[2] == hash_s
+        assert res[3] == hash_u
+
+    def test_hash_string_rpython(self):
+        self._test_hash_string("rpython")
+
+    def test_hash_string_siphash24(self):
+        import subprocess
+        subprocess.check_call([sys.executable, __file__, "siphash24",
+                               self.__class__.__module__,
+                               self.__class__.__name__])
+
     def test_list_basic_ops(self):
         def list_basic_ops(i, j):
             l = [1, 2, 3]
@@ -896,3 +934,11 @@
         f = self.getcompiled(func, [int])
         res = f(2)
         assert res == 1     # and not 2
+
+
+if __name__ == '__main__':
+    # for test_hash_string_siphash24()
+    algo, clsmodule, clsname = sys.argv[1:]
+    mod = __import__(clsmodule, None, None, [clsname])
+    cls = getattr(mod, clsname)
+    cls()._test_hash_string(algo)

From pypy.commits at gmail.com  Wed Jan 25 15:37:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 12:37:21 -0800 (PST)
Subject: [pypy-commit] pypy default: Fixes, trying to use siphash24 on pypy
Message-ID: <58890c81.b296df0a.ebe67.1e39@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89772:db5482cc94a2
Date: 2017-01-25 20:36 +0000
http://bitbucket.org/pypy/pypy/changeset/db5482cc94a2/

Log:	Fixes, trying to use siphash24 on pypy

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -614,6 +614,9 @@
     This version is called from lltypesystem/rstr.py, and 'll_s' is a
     rstr.STR or rstr.UNICODE.
     """
+    if not we_are_translated():
+        global HASH_ALGORITHM_FIXED
+        HASH_ALGORITHM_FIXED = True
     if HASH_ALGORITHM == "rpython":
         return _hash_string_rpython(ll_s.chars)
     if HASH_ALGORITHM == "siphash24":
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -44,11 +44,13 @@
 mallocstr = new_malloc(STR, 'mallocstr')
 mallocunicode = new_malloc(UNICODE, 'mallocunicode')
 
+ at specialize.memo()
 def emptystrfun():
-    return emptystr
+    return string_repr.convert_const("")
 
+ at specialize.memo()
 def emptyunicodefun():
-    return emptyunicode
+    return unicode_repr.convert_const(u'')
 
 def _new_copy_contents_fun(SRC_TP, DST_TP, CHAR_TP, name):
     @specialize.arg(0)
@@ -1280,8 +1282,6 @@
 char_repr.ll = LLHelpers
 unichar_repr.ll = LLHelpers
 unicode_repr = UnicodeRepr()
-emptystr = string_repr.convert_const("")
-emptyunicode = unicode_repr.convert_const(u'')
 
 StringRepr.repr = string_repr
 UnicodeRepr.repr = unicode_repr
@@ -1340,14 +1340,6 @@
 string_repr.iterator_repr = StringIteratorRepr()
 unicode_repr.iterator_repr = UnicodeIteratorRepr()
 
-# these should be in rclass, but circular imports prevent (also it's
-# not that insane that a string constant is built in this file).
-
-instance_str_prefix = string_repr.convert_const("<")
-instance_str_infix  = string_repr.convert_const(" object at 0x")
-instance_str_suffix = string_repr.convert_const(">")
-
-null_str = string_repr.convert_const("NULL")
-
-unboxed_instance_str_prefix = string_repr.convert_const("<unboxed ")
-unboxed_instance_str_suffix = string_repr.convert_const(">")
+ at specialize.memo()
+def conststr(s):
+    return string_repr.convert_const(s)
diff --git a/rpython/rtyper/lltypesystem/rtagged.py b/rpython/rtyper/lltypesystem/rtagged.py
--- a/rpython/rtyper/lltypesystem/rtagged.py
+++ b/rpython/rtyper/lltypesystem/rtagged.py
@@ -117,9 +117,9 @@
             from rpython.rtyper.lltypesystem import rstr
             from rpython.rtyper.rint import signed_repr
             llstr1 = signed_repr.ll_str(ll_unboxed_to_int(i))
-            return rstr.ll_strconcat(rstr.unboxed_instance_str_prefix,
+            return rstr.ll_strconcat(rstr.conststr("<unboxed "),
                       rstr.ll_strconcat(llstr1,
-                                        rstr.unboxed_instance_str_suffix))
+                                        rstr.conststr(">")))
         else:
             return InstanceRepr.ll_str(self, i)
 
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -840,18 +840,18 @@
         from rpython.rtyper.lltypesystem.ll_str import ll_int2hex
         from rpython.rlib.rarithmetic import r_uint
         if not i:
-            return rstr.null_str
+            return rstr.conststr("NULL")
         instance = cast_pointer(OBJECTPTR, i)
         # Two choices: the first gives a fast answer but it can change
         # (typically only once) during the life of the object.
         #uid = r_uint(cast_ptr_to_int(i))
         uid = r_uint(llop.gc_id(lltype.Signed, i))
         #
-        res = rstr.instance_str_prefix
+        res = rstr.conststr("<")
         res = rstr.ll_strconcat(res, instance.typeptr.name)
-        res = rstr.ll_strconcat(res, rstr.instance_str_infix)
+        res = rstr.ll_strconcat(res, rstr.conststr(" object at 0x"))
         res = rstr.ll_strconcat(res, ll_int2hex(uid, False))
-        res = rstr.ll_strconcat(res, rstr.instance_str_suffix)
+        res = rstr.ll_strconcat(res, rstr.conststr(">"))
         return res
 
     def get_ll_eq_function(self):
@@ -1092,7 +1092,6 @@
     except StandardError:
         return None
 
-
 # ____________________________________________________________
 #
 #  Low-level implementation of operations on classes and instances

From pypy.commits at gmail.com  Wed Jan 25 15:43:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 12:43:09 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix (obscure reason,
 can be reproduced with "py.test rtyper/")
Message-ID: <58890ddd.28a8df0a.8397e.a4cf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89773:93614a596712
Date: 2017-01-25 21:42 +0100
http://bitbucket.org/pypy/pypy/changeset/93614a596712/

Log:	Fix (obscure reason, can be reproduced with "py.test rtyper/")

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1073,8 +1073,9 @@
         if size is None:
             size = llmemory.sizeof(tp)    # a symbolic result in this case
         return size
-    if isinstance(tp, lltype.Ptr) or tp is llmemory.Address:
-        return globals()['r_void*'].BITS/8
+    if (tp is lltype.Signed or isinstance(tp, lltype.Ptr) 
+                            or tp is llmemory.Address):
+        return LONG_BIT/8
     if tp is lltype.Char or tp is lltype.Bool:
         return 1
     if tp is lltype.UniChar:
@@ -1087,8 +1088,6 @@
         # :-/
         return sizeof_c_type("long double")
     assert isinstance(tp, lltype.Number)
-    if tp is lltype.Signed:
-        return LONG_BIT/8
     return tp._type.BITS/8
 sizeof._annspecialcase_ = 'specialize:memo'
 

From pypy.commits at gmail.com  Wed Jan 25 15:56:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 12:56:56 -0800 (PST)
Subject: [pypy-commit] pypy default: fix error message
Message-ID: <58891118.2687df0a.33952.2d14@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89774:20567a253f17
Date: 2017-01-25 21:56 +0100
http://bitbucket.org/pypy/pypy/changeset/20567a253f17/

Log:	fix error message

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -307,8 +307,8 @@
         if config.objspace.usemodules.cpyext:
             if config.translation.gc not in ('incminimark', 'boehm'):
                 raise Exception("The 'cpyext' module requires the 'incminimark'"
-                                " 'boehm' GC.  You need either 'targetpypystandalone.py"
-                                " --withoutmod-cpyext' or '--gc=incminimark'")
+                    " or 'boehm' GC.  You need either 'targetpypystandalone.py"
+                    " --withoutmod-cpyext', or use one of these two GCs.")
 
         config.translating = True
 

From pypy.commits at gmail.com  Wed Jan 25 17:14:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 25 Jan 2017 14:14:29 -0800 (PST)
Subject: [pypy-commit] pypy default: A latin-1 unicode string needs to have
 the same hash as the byte string.
Message-ID: <58892345.5190df0a.16278.41c3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89775:d8c036e10b26
Date: 2017-01-25 23:13 +0100
http://bitbucket.org/pypy/pypy/changeset/d8c036e10b26/

Log:	A latin-1 unicode string needs to have the same hash as the byte
	string.

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -556,22 +556,20 @@
     from rpython.rtyper.lltypesystem import lltype, rffi
     from rpython.rlib.rarithmetic import intmask
 
-    if isinstance(s, str):
-        pass
-    elif isinstance(s, unicode):
-        if rffi.sizeof(lltype.UniChar) == 4:
-            kind = "I"
+    if not isinstance(s, str):
+        if isinstance(s, unicode):
+            lst = map(ord, s)
         else:
-            kind = "H"
-        s = array.array(kind, map(ord, s)).tostring()
-    else:
-        if lltype.typeOf(s).TO.chars.OF == lltype.Char:
+            lst = map(ord, s.chars)    # for rstr.STR or UNICODE
+        # NOTE: a latin-1 unicode string must have the same hash as the
+        # corresponding byte string.
+        if all(n <= 0xFF for n in lst):
             kind = "B"
         elif rffi.sizeof(lltype.UniChar) == 4:
             kind = "I"
         else:
             kind = "H"
-        s = array.array(kind, map(ord, s.chars)).tostring()
+        s = array.array(kind, lst).tostring()
     ptr = rffi.str2charp(s)
     x = siphash24(ptr, len(s))
     rffi.free_charp(ptr)
@@ -580,16 +578,33 @@
 def ll_hash_string_siphash24(ll_s):
     """Called from lltypesystem/rstr.py.  'll_s' is a rstr.STR or UNICODE."""
     from rpython.rlib.rsiphash import siphash24
-    from rpython.rtyper.lltypesystem import lltype, rffi, rstr
+    from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
     from rpython.rlib.rarithmetic import intmask
 
     length = len(ll_s.chars)
-    # no GC operation from here!
     if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:
+        # no GC operation from here!
         addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
     else:
-        addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
-        length *= rffi.sizeof(rstr.UNICODE.chars.OF)
+        # NOTE: a latin-1 unicode string must have the same hash as the
+        # corresponding byte string.  If the unicode is all within
+        # 0-255, then we need to allocate a byte buffer and copy the
+        # latin-1 encoding in it manually.
+        for i in range(length):
+            if ord(ll_s.chars[i]) > 0xFF:
+                # no GC operation from here!
+                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
+                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
+                break
+        else:
+            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
+            i = 0
+            while i < length:
+                p[i] = chr(ord(ll_s.chars[i]))
+                i += 1
+            x = siphash24(llmemory.cast_ptr_to_adr(p), length)
+            lltype.free(p, flavor='raw')
+            return intmask(x)
     x = siphash24(addr, length)
     keepalive_until_here(ll_s)
     return intmask(x)
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -606,25 +606,34 @@
         objectmodel.set_hash_algorithm(algo)
         s = "hello"
         u = u"world"
+        v = u"\u1234\u2318+\u2bcd\u2102"
         hash_s = compute_hash(s)
         hash_u = compute_hash(u)
+        hash_v = compute_hash(v)
+        assert hash_s == compute_hash(u"hello")   # same hash because it's
+        assert hash_u == compute_hash("world")    #    a latin-1 unicode
         #
         def fn(length):
             assert length >= 1
             return str((compute_hash(s),
                         compute_hash(u),
+                        compute_hash(v),
                         compute_hash(s[0] + s[1:length]),
-                        compute_hash(u[0] + u[1:length])))
+                        compute_hash(u[0] + u[1:length]),
+                        compute_hash(v[0] + v[1:length]),
+                        ))
 
-        assert fn(5) == str((hash_s, hash_u, hash_s, hash_u))
+        assert fn(5) == str((hash_s, hash_u, hash_v, hash_s, hash_u, hash_v))
 
         f = self.getcompiled(fn, [int])
         res = f(5)
         res = [int(a) for a in res[1:-1].split(",")]
         assert res[0] == hash_s
         assert res[1] == hash_u
-        assert res[2] == hash_s
-        assert res[3] == hash_u
+        assert res[2] == hash_v
+        assert res[3] == hash_s
+        assert res[4] == hash_u
+        assert res[5] == hash_v
 
     def test_hash_string_rpython(self):
         self._test_hash_string("rpython")

From pypy.commits at gmail.com  Thu Jan 26 03:30:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 00:30:50 -0800 (PST)
Subject: [pypy-commit] pypy default: Add translation option --hash=siphash24
Message-ID: <5889b3ba.484a1c0a.6cfc5.6402@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89776:ccc88590dc02
Date: 2017-01-26 09:30 +0100
http://bitbucket.org/pypy/pypy/changeset/ccc88590dc02/

Log:	Add translation option --hash=siphash24

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -201,6 +201,10 @@
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
+    ChoiceOption("hash",
+                 "The hash to use for strings",
+                 ["rpython", "siphash24"],
+                 default="rpython", cmdline="--hash"),
 
     OptionDescription("backendopt", "Backend Optimization Options", [
         # control inlining
@@ -390,6 +394,12 @@
         if sys.platform == "darwin" or sys.platform =="win32":
             raise ConfigError("'asmgcc' not supported on this platform")
 
+def apply_extra_settings(config):
+    # make the setting of config.hash definitive
+    from rpython.rlib.objectmodel import set_hash_algorithm
+    config.translation.hash = config.translation.hash
+    set_hash_algorithm(config.translation.hash)
+
 # ----------------------------------------------------------------
 
 def set_platform(config):
diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -11,7 +11,8 @@
 from rpython.config.config import (to_optparse, OptionDescription, BoolOption,
     ArbitraryOption, StrOption, IntOption, Config, ChoiceOption, OptHelpFormatter)
 from rpython.config.translationoption import (get_combined_translation_config,
-    set_opt_level, OPT_LEVELS, DEFAULT_OPT_LEVEL, set_platform, CACHE_DIR)
+    set_opt_level, OPT_LEVELS, DEFAULT_OPT_LEVEL, set_platform, CACHE_DIR,
+    apply_extra_settings)
 
 # clean up early rpython/_cache
 try:
@@ -177,6 +178,9 @@
     if 'handle_config' in targetspec_dic:
         targetspec_dic['handle_config'](config, translateconfig)
 
+    # apply extra settings
+    apply_extra_settings(config)
+
     return targetspec_dic, translateconfig, config, args
 
 def show_help(translateconfig, opt_parser, targetspec_dic, config):

From pypy.commits at gmail.com  Thu Jan 26 03:35:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 00:35:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5889b4df.08061c0a.c3931.988e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89777:ad1beddd3f43
Date: 2017-01-26 09:34 +0100
http://bitbucket.org/pypy/pypy/changeset/ad1beddd3f43/

Log:	hg merge default

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -311,8 +311,8 @@
         if config.objspace.usemodules.cpyext:
             if config.translation.gc not in ('incminimark', 'boehm'):
                 raise Exception("The 'cpyext' module requires the 'incminimark'"
-                                " 'boehm' GC.  You need either 'targetpypystandalone.py"
-                                " --withoutmod-cpyext' or '--gc=incminimark'")
+                    " or 'boehm' GC.  You need either 'targetpypystandalone.py"
+                    " --withoutmod-cpyext', or use one of these two GCs.")
 
         config.translating = True
 
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -323,17 +323,28 @@
         #
         return self._add_or_sub(w_other, -1)
 
-    def getcfield(self, w_attr):
-        return self.ctype.getcfield(self.space.str_w(w_attr))
+    def getcfield(self, w_attr, mode):
+        space = self.space
+        attr = space.str_w(w_attr)
+        try:
+            cfield = self.ctype.getcfield(attr)
+        except KeyError:
+            raise oefmt(space.w_AttributeError, "cdata '%s' has no field '%s'",
+                        self.ctype.name, attr)
+        if cfield is None:
+            raise oefmt(space.w_AttributeError,
+                        "cdata '%s' points to an opaque type: cannot %s fields",
+                        self.ctype.name, mode)
+        return cfield
 
     def getattr(self, w_attr):
-        cfield = self.getcfield(w_attr)
+        cfield = self.getcfield(w_attr, mode="read")
         with self as ptr:
             w_res = cfield.read(ptr, self)
         return w_res
 
     def setattr(self, w_attr, w_value):
-        cfield = self.getcfield(w_attr)
+        cfield = self.getcfield(w_attr, mode="write")
         with self as ptr:
             cfield.write(ptr, w_value)
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -348,7 +348,10 @@
         return result
 
     def getcfield(self, attr):
-        return self.ctitem.getcfield(attr)
+        from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
+        if isinstance(self.ctitem, W_CTypeStructOrUnion):
+            return self.ctitem.getcfield(attr)
+        return W_CType.getcfield(self, attr)
 
     def typeoffsetof_field(self, fieldname, following):
         if following == 0:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -161,18 +161,18 @@
         return self._fields_dict[attr]
 
     def getcfield(self, attr):
-        ready = self._fields_dict is not None
-        if not ready and self.size >= 0:
+        # Returns a W_CField.  Error cases: returns None if we are an
+        # opaque struct; or raises KeyError if the particular field
+        # 'attr' does not exist.  The point of not directly building the
+        # error here is to get the exact ctype in the error message: it
+        # might be of the kind 'struct foo' or 'struct foo *'.
+        if self._fields_dict is None:
+            if self.size < 0:
+                return None
             self.force_lazy_struct()
-            ready = True
-        if ready:
-            self = jit.promote(self)
-            attr = jit.promote_string(attr)
-            try:
-                return self._getcfield_const(attr)
-            except KeyError:
-                pass
-        return W_CType.getcfield(self, attr)
+        self = jit.promote(self)
+        attr = jit.promote_string(attr)
+        return self._getcfield_const(attr)    # <= KeyError here
 
     def cdata_dir(self):
         if self.size < 0:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -737,8 +737,14 @@
     BInt = new_primitive_type("int")
     BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
-    p = cast(BStructPtr, 0)
-    py.test.raises(AttributeError, "p.a1")    # opaque
+    p = cast(BStructPtr, 42)
+    e = py.test.raises(AttributeError, "p.a1")    # opaque
+    assert str(e.value) == ("cdata 'struct foo *' points to an opaque type: "
+                            "cannot read fields")
+    e = py.test.raises(AttributeError, "p.a1 = 10")    # opaque
+    assert str(e.value) == ("cdata 'struct foo *' points to an opaque type: "
+                            "cannot write fields")
+
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
     p = newp(BStructPtr, None)
@@ -749,8 +755,29 @@
     assert s.a2 == 123
     py.test.raises(OverflowError, "s.a1 = sys.maxsize+1")
     assert s.a1 == 0
-    py.test.raises(AttributeError, "p.foobar")
-    py.test.raises(AttributeError, "s.foobar")
+    e = py.test.raises(AttributeError, "p.foobar")
+    assert str(e.value) == "cdata 'struct foo *' has no field 'foobar'"
+    e = py.test.raises(AttributeError, "p.foobar = 42")
+    assert str(e.value) == "cdata 'struct foo *' has no field 'foobar'"
+    e = py.test.raises(AttributeError, "s.foobar")
+    assert str(e.value) == "cdata 'struct foo' has no field 'foobar'"
+    e = py.test.raises(AttributeError, "s.foobar = 42")
+    assert str(e.value) == "cdata 'struct foo' has no field 'foobar'"
+    j = cast(BInt, 42)
+    e = py.test.raises(AttributeError, "j.foobar")
+    assert str(e.value) == "cdata 'int' has no attribute 'foobar'"
+    e = py.test.raises(AttributeError, "j.foobar = 42")
+    assert str(e.value) == "cdata 'int' has no attribute 'foobar'"
+    j = cast(new_pointer_type(BInt), 42)
+    e = py.test.raises(AttributeError, "j.foobar")
+    assert str(e.value) == "cdata 'int *' has no attribute 'foobar'"
+    e = py.test.raises(AttributeError, "j.foobar = 42")
+    assert str(e.value) == "cdata 'int *' has no attribute 'foobar'"
+    pp = newp(new_pointer_type(BStructPtr), p)
+    e = py.test.raises(AttributeError, "pp.a1")
+    assert str(e.value) == "cdata 'struct foo * *' has no attribute 'a1'"
+    e = py.test.raises(AttributeError, "pp.a1 = 42")
+    assert str(e.value) == "cdata 'struct foo * *' has no attribute 'a1'"
 
 def test_union_instance():
     BInt = new_primitive_type("int")
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -437,6 +437,9 @@
         for i in range(len(self.cached_attrs)):
             self.cached_attrs[i] = None
 
+    def _cleanup_(self):
+        self.clear()
+
 # ____________________________________________________________
 # object implementation
 
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -72,6 +72,10 @@
 class MethodCache(object):
 
     def __init__(self, space):
+        # Note: these attributes never change which object they contain,
+        # so reading 'cache.versions' for example is constant-folded.
+        # The actual list in 'cache.versions' is not a constant, of
+        # course.
         SIZE = 1 << space.config.objspace.std.methodcachesizeexp
         self.versions = [None] * SIZE
         self.names = [None] * SIZE
@@ -89,6 +93,9 @@
         for i in range(len(self.lookup_where)):
             self.lookup_where[i] = None_None
 
+    def _cleanup_(self):
+        self.clear()
+
 class _Global(object):
     weakref_warning_printed = False
 _global = _Global()
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -201,6 +201,10 @@
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
+    ChoiceOption("hash",
+                 "The hash to use for strings",
+                 ["rpython", "siphash24"],
+                 default="rpython", cmdline="--hash"),
 
     OptionDescription("backendopt", "Backend Optimization Options", [
         # control inlining
@@ -390,6 +394,12 @@
         if sys.platform == "darwin" or sys.platform =="win32":
             raise ConfigError("'asmgcc' not supported on this platform")
 
+def apply_extra_settings(config):
+    # make the setting of config.hash definitive
+    from rpython.rlib.objectmodel import set_hash_algorithm
+    config.translation.hash = config.translation.hash
+    set_hash_algorithm(config.translation.hash)
+
 # ----------------------------------------------------------------
 
 def set_platform(config):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -520,10 +520,22 @@
 # ----------
 
 HASH_ALGORITHM = "rpython"  # XXX Is there a better name?
+HASH_ALGORITHM_FIXED = False
 
-def _hash_string(s):
-    """The algorithm behind compute_hash() for a string or a unicode."""
+ at not_rpython
+def set_hash_algorithm(algo):
+    """Must be called very early, before any string is hashed with
+    compute_hash()!"""
+    global HASH_ALGORITHM
+    if HASH_ALGORITHM != algo:
+        assert not HASH_ALGORITHM_FIXED, "compute_hash() already called!"
+        assert algo in ("rpython", "siphash24")
+        HASH_ALGORITHM = algo
+
+
+def _hash_string_rpython(s):
     from rpython.rlib.rarithmetic import intmask
+
     length = len(s)
     if length == 0:
         return -1
@@ -535,6 +547,101 @@
     x ^= length
     return intmask(x)
 
+
+ at not_rpython
+def _hash_string_siphash24(s):
+    """This version is called when untranslated only."""
+    import array
+    from rpython.rlib.rsiphash import siphash24
+    from rpython.rtyper.lltypesystem import lltype, rffi
+    from rpython.rlib.rarithmetic import intmask
+
+    if not isinstance(s, str):
+        if isinstance(s, unicode):
+            lst = map(ord, s)
+        else:
+            lst = map(ord, s.chars)    # for rstr.STR or UNICODE
+        # NOTE: a latin-1 unicode string must have the same hash as the
+        # corresponding byte string.
+        if all(n <= 0xFF for n in lst):
+            kind = "B"
+        elif rffi.sizeof(lltype.UniChar) == 4:
+            kind = "I"
+        else:
+            kind = "H"
+        s = array.array(kind, lst).tostring()
+    ptr = rffi.str2charp(s)
+    x = siphash24(ptr, len(s))
+    rffi.free_charp(ptr)
+    return intmask(x)
+
+def ll_hash_string_siphash24(ll_s):
+    """Called from lltypesystem/rstr.py.  'll_s' is a rstr.STR or UNICODE."""
+    from rpython.rlib.rsiphash import siphash24
+    from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
+    from rpython.rlib.rarithmetic import intmask
+
+    length = len(ll_s.chars)
+    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:
+        # no GC operation from here!
+        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
+    else:
+        # NOTE: a latin-1 unicode string must have the same hash as the
+        # corresponding byte string.  If the unicode is all within
+        # 0-255, then we need to allocate a byte buffer and copy the
+        # latin-1 encoding in it manually.
+        for i in range(length):
+            if ord(ll_s.chars[i]) > 0xFF:
+                # no GC operation from here!
+                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
+                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
+                break
+        else:
+            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
+            i = 0
+            while i < length:
+                p[i] = chr(ord(ll_s.chars[i]))
+                i += 1
+            x = siphash24(llmemory.cast_ptr_to_adr(p), length)
+            lltype.free(p, flavor='raw')
+            return intmask(x)
+    x = siphash24(addr, length)
+    keepalive_until_here(ll_s)
+    return intmask(x)
+ll_hash_string_siphash24._jit_look_inside_ = False
+
+
+ at not_rpython
+def _hash_string(s):
+    """The algorithm behind compute_hash() for a string or a unicode.
+    This version is only for untranslated usage, and 's' is a str or unicode.
+    """
+    global HASH_ALGORITHM_FIXED
+    HASH_ALGORITHM_FIXED = True
+    if HASH_ALGORITHM == "rpython":
+        return _hash_string_rpython(s)
+    if HASH_ALGORITHM == "siphash24":
+        return _hash_string_siphash24(s)
+    raise NotImplementedError
+
+def ll_hash_string(ll_s):
+    """The algorithm behind compute_hash() for a string or a unicode.
+    This version is called from lltypesystem/rstr.py, and 'll_s' is a
+    rstr.STR or rstr.UNICODE.
+    """
+    if not we_are_translated():
+        global HASH_ALGORITHM_FIXED
+        HASH_ALGORITHM_FIXED = True
+    if HASH_ALGORITHM == "rpython":
+        return _hash_string_rpython(ll_s.chars)
+    if HASH_ALGORITHM == "siphash24":
+        if we_are_translated():
+            return ll_hash_string_siphash24(ll_s)
+        else:
+            return _hash_string_siphash24(ll_s)
+    raise NotImplementedError
+
+
 def _hash_float(f):
     """The algorithm behind compute_hash() for a float.
     This implementation is identical to the CPython implementation,
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rsiphash.py
@@ -0,0 +1,157 @@
+import sys, os, struct
+from contextlib import contextmanager
+from rpython.rlib import rarithmetic
+from rpython.rlib.objectmodel import not_rpython, always_inline
+from rpython.rlib.rgc import no_collect
+from rpython.rlib.rarithmetic import r_uint64
+from rpython.rlib.rawstorage import misaligned_is_fine
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem.lloperation import llop
+
+
+if sys.byteorder == 'little':
+    def _le64toh(x):
+        return x
+else:
+    _le64toh = rarithmetic.byteswap
+
+
+# Initialize the values of the secret seed: two 64-bit constants.
+# CPython picks a new seed every time 'python' starts.  PyPy cannot do
+# that as easily because many details may rely on getting the same hash
+# value before and after translation.  We can, however, pick a random
+# seed once per translation, which should already be quite good.
+
+ at not_rpython
+def select_random_seed():
+    global k0, k1    # note: the globals k0, k1 are already byte-swapped
+    v0, v1 = struct.unpack("QQ", os.urandom(16))
+    k0 = r_uint64(v0)
+    k1 = r_uint64(v1)
+
+select_random_seed()
+
+ at contextmanager
+def choosen_seed(new_k0, new_k1, test_misaligned_path=False):
+    global k0, k1, misaligned_is_fine
+    old = k0, k1, misaligned_is_fine
+    k0 = _le64toh(r_uint64(new_k0))
+    k1 = _le64toh(r_uint64(new_k1))
+    if test_misaligned_path:
+        misaligned_is_fine = False
+    yield
+    k0, k1, misaligned_is_fine = old
+
+def get_current_seed():
+    return _le64toh(k0), _le64toh(k1)
+
+
+magic0 = r_uint64(0x736f6d6570736575)
+magic1 = r_uint64(0x646f72616e646f6d)
+magic2 = r_uint64(0x6c7967656e657261)
+magic3 = r_uint64(0x7465646279746573)
+
+
+ at always_inline
+def _rotate(x, b):
+    return (x << b) | (x >> (64 - b))
+
+ at always_inline
+def _half_round(a, b, c, d, s, t):
+    a += b
+    c += d
+    b = _rotate(b, s) ^ a
+    d = _rotate(d, t) ^ c
+    a = _rotate(a, 32)
+    return a, b, c, d
+
+ at always_inline
+def _double_round(v0, v1, v2, v3):
+    v0,v1,v2,v3 = _half_round(v0,v1,v2,v3,13,16)
+    v2,v1,v0,v3 = _half_round(v2,v1,v0,v3,17,21)
+    v0,v1,v2,v3 = _half_round(v0,v1,v2,v3,13,16)
+    v2,v1,v0,v3 = _half_round(v2,v1,v0,v3,17,21)
+    return v0, v1, v2, v3
+
+
+ at no_collect
+def siphash24(addr_in, size):
+    """Takes an address pointer and a size.  Returns the hash as a r_uint64,
+    which can then be casted to the expected type."""
+
+    direct = (misaligned_is_fine or
+                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
+
+    b = r_uint64(size) << 56
+    v0 = k0 ^ magic0
+    v1 = k1 ^ magic1
+    v2 = k0 ^ magic2
+    v3 = k1 ^ magic3
+
+    index = 0
+    if direct:
+        while size >= 8:
+            mi = llop.raw_load(rffi.ULONGLONG, addr_in, index)
+            mi = _le64toh(mi)
+            size -= 8
+            index += 8
+            v3 ^= mi
+            v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+            v0 ^= mi
+    else:
+        while size >= 8:
+            mi = (
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index)) |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1)) << 8 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2)) << 16 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4)) << 32 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5)) << 40 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48 |
+                r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 7)) << 56
+            )
+            mi = _le64toh(mi)
+            size -= 8
+            index += 8
+            v3 ^= mi
+            v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+            v0 ^= mi
+
+    t = r_uint64(0)
+    if size == 7:
+        t = r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48
+        size = 6
+    if size == 6:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5)) << 40
+        size = 5
+    if size == 5:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4)) << 32
+        size = 4
+    if size == 4:
+        if direct:
+            t |= r_uint64(llop.raw_load(rffi.UINT, addr_in, index))
+            size = 0
+        else:
+            t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24
+            size = 3
+    if size == 3:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2)) << 16
+        size = 2
+    if size == 2:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1)) << 8
+        size = 1
+    if size == 1:
+        t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index))
+        size = 0
+    assert size == 0
+
+    b |= _le64toh(t)
+
+    v3 ^= b
+    v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+    v0 ^= b
+    v2 ^= 0xff
+    v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+
+    return (v0 ^ v1) ^ (v2 ^ v3)
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -0,0 +1,44 @@
+from rpython.rlib.rsiphash import siphash24, choosen_seed
+from rpython.rtyper.lltypesystem import llmemory, rffi
+
+
+CASES = [
+    (2323638336262702335 , ""),
+    (5150479602681463644 , "h"),
+    (1013213613370725794 , "he"),
+    (7028032310911240238 , "hel"),
+    (9535960132410784494 , "hell"),
+    (3256502711089771242 , "hello"),
+    (2389188832234450176 , "hello "),
+    (13253855839845990393, "hello w"),
+    (7850036019043917323 , "hello wo"),
+    (14283308628425005953, "hello wor"),
+    (9605549962279590084 , "hello worl"),
+    (16371281469632894235, "hello world"),
+    (7298637955795769949 , "hello world\x9a"),
+    (13530878135053370821, "hello world\xf3\x80"),
+    (1643533543579802994 , "\xffhel\x82lo world\xbc"),
+    (14632093238728197380, "hexlylxox rewqw"),
+    (3434253029196696424 , "hexlylxox rewqws"),
+    (9855754545877066788 , "hexlylxox rewqwsv"),
+    (5233065012564472454 , "hexlylxox rewqwkashdw89"),
+    (16768585622569081808, "hexlylxox rewqwkeashdw89"),
+    (17430482483431293463, "HEEExlylxox rewqwkashdw89"),
+    (695783005783737705  , "hello woadwealidewd 3829ez 32ig dxwaebderld"),
+]
+
+def check(s):
+    p = rffi.str2charp(s)
+    q = rffi.str2charp('?' + s)
+    with choosen_seed(0x8a9f065a358479f4, 0x11cb1e9ee7f40e1f,
+                      test_misaligned_path=True):
+        x = siphash24(llmemory.cast_ptr_to_adr(p), len(s))
+        y = siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
+    rffi.free_charp(p)
+    rffi.free_charp(q)
+    assert x == y
+    return x
+
+def test_siphash24():
+    for expected, string in CASES:
+        assert check(string) == expected
diff --git a/rpython/rtyper/lltypesystem/rbytearray.py b/rpython/rtyper/lltypesystem/rbytearray.py
--- a/rpython/rtyper/lltypesystem/rbytearray.py
+++ b/rpython/rtyper/lltypesystem/rbytearray.py
@@ -8,10 +8,10 @@
 def mallocbytearray(size):
     return lltype.malloc(BYTEARRAY, size)
 
-_, _, copy_bytearray_contents = rstr._new_copy_contents_fun(BYTEARRAY, BYTEARRAY,
+_, _, copy_bytearray_contents, _ = rstr._new_copy_contents_fun(BYTEARRAY, BYTEARRAY,
                                                          lltype.Char,
                                                          'bytearray')
-_, _, copy_bytearray_contents_from_str = rstr._new_copy_contents_fun(rstr.STR,
+_, _, copy_bytearray_contents_from_str, _ = rstr._new_copy_contents_fun(rstr.STR,
                                                                   BYTEARRAY,
                                                                   lltype.Char,
                                                                   'bytearray_from_str')
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1073,8 +1073,9 @@
         if size is None:
             size = llmemory.sizeof(tp)    # a symbolic result in this case
         return size
-    if isinstance(tp, lltype.Ptr) or tp is llmemory.Address:
-        return globals()['r_void*'].BITS/8
+    if (tp is lltype.Signed or isinstance(tp, lltype.Ptr) 
+                            or tp is llmemory.Address):
+        return LONG_BIT/8
     if tp is lltype.Char or tp is lltype.Bool:
         return 1
     if tp is lltype.UniChar:
@@ -1087,8 +1088,6 @@
         # :-/
         return sizeof_c_type("long double")
     assert isinstance(tp, lltype.Number)
-    if tp is lltype.Signed:
-        return LONG_BIT/8
     return tp._type.BITS/8
 sizeof._annspecialcase_ = 'specialize:memo'
 
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -3,7 +3,7 @@
 from rpython.annotator import model as annmodel
 from rpython.rlib import jit, types
 from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated,
-    _hash_string, keepalive_until_here, specialize, enforceargs)
+    ll_hash_string, keepalive_until_here, specialize, enforceargs)
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.error import TyperError
@@ -44,11 +44,13 @@
 mallocstr = new_malloc(STR, 'mallocstr')
 mallocunicode = new_malloc(UNICODE, 'mallocunicode')
 
+ at specialize.memo()
 def emptystrfun():
-    return emptystr
+    return string_repr.convert_const("")
 
+ at specialize.memo()
 def emptyunicodefun():
-    return emptyunicode
+    return unicode_repr.convert_const(u'')
 
 def _new_copy_contents_fun(SRC_TP, DST_TP, CHAR_TP, name):
     @specialize.arg(0)
@@ -136,15 +138,19 @@
     copy_raw_to_string = func_with_new_name(copy_raw_to_string,
                                               'copy_raw_to_%s' % name)
 
-    return copy_string_to_raw, copy_raw_to_string, copy_string_contents
+    return (copy_string_to_raw, copy_raw_to_string, copy_string_contents,
+            _get_raw_buf)
 
 (copy_string_to_raw,
  copy_raw_to_string,
- copy_string_contents) = _new_copy_contents_fun(STR, STR, Char, 'string')
+ copy_string_contents,
+ _get_raw_buf_string) = _new_copy_contents_fun(STR, STR, Char, 'string')
 
 (copy_unicode_to_raw,
  copy_raw_to_unicode,
- copy_unicode_contents) = _new_copy_contents_fun(UNICODE, UNICODE, UniChar, 'unicode')
+ copy_unicode_contents,
+ _get_raw_buf_unicode) = _new_copy_contents_fun(UNICODE, UNICODE, UniChar,
+                                                'unicode')
 
 CONST_STR_CACHE = WeakValueDictionary()
 CONST_UNICODE_CACHE = WeakValueDictionary()
@@ -382,7 +388,7 @@
         # but our malloc initializes the memory to zero, so we use zero as the
         # special non-computed-yet value.  Also, jit.conditional_call_elidable
         # always checks for zero, for now.
-        x = _hash_string(s.chars)
+        x = ll_hash_string(s)
         if x == 0:
             x = 29872897
         s.hash = x
@@ -1276,8 +1282,6 @@
 char_repr.ll = LLHelpers
 unichar_repr.ll = LLHelpers
 unicode_repr = UnicodeRepr()
-emptystr = string_repr.convert_const("")
-emptyunicode = unicode_repr.convert_const(u'')
 
 StringRepr.repr = string_repr
 UnicodeRepr.repr = unicode_repr
@@ -1336,14 +1340,6 @@
 string_repr.iterator_repr = StringIteratorRepr()
 unicode_repr.iterator_repr = UnicodeIteratorRepr()
 
-# these should be in rclass, but circular imports prevent (also it's
-# not that insane that a string constant is built in this file).
-
-instance_str_prefix = string_repr.convert_const("<")
-instance_str_infix  = string_repr.convert_const(" object at 0x")
-instance_str_suffix = string_repr.convert_const(">")
-
-null_str = string_repr.convert_const("NULL")
-
-unboxed_instance_str_prefix = string_repr.convert_const("<unboxed ")
-unboxed_instance_str_suffix = string_repr.convert_const(">")
+ at specialize.memo()
+def conststr(s):
+    return string_repr.convert_const(s)
diff --git a/rpython/rtyper/lltypesystem/rtagged.py b/rpython/rtyper/lltypesystem/rtagged.py
--- a/rpython/rtyper/lltypesystem/rtagged.py
+++ b/rpython/rtyper/lltypesystem/rtagged.py
@@ -117,9 +117,9 @@
             from rpython.rtyper.lltypesystem import rstr
             from rpython.rtyper.rint import signed_repr
             llstr1 = signed_repr.ll_str(ll_unboxed_to_int(i))
-            return rstr.ll_strconcat(rstr.unboxed_instance_str_prefix,
+            return rstr.ll_strconcat(rstr.conststr("<unboxed "),
                       rstr.ll_strconcat(llstr1,
-                                        rstr.unboxed_instance_str_suffix))
+                                        rstr.conststr(">")))
         else:
             return InstanceRepr.ll_str(self, i)
 
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -840,18 +840,18 @@
         from rpython.rtyper.lltypesystem.ll_str import ll_int2hex
         from rpython.rlib.rarithmetic import r_uint
         if not i:
-            return rstr.null_str
+            return rstr.conststr("NULL")
         instance = cast_pointer(OBJECTPTR, i)
         # Two choices: the first gives a fast answer but it can change
         # (typically only once) during the life of the object.
         #uid = r_uint(cast_ptr_to_int(i))
         uid = r_uint(llop.gc_id(lltype.Signed, i))
         #
-        res = rstr.instance_str_prefix
+        res = rstr.conststr("<")
         res = rstr.ll_strconcat(res, instance.typeptr.name)
-        res = rstr.ll_strconcat(res, rstr.instance_str_infix)
+        res = rstr.ll_strconcat(res, rstr.conststr(" object at 0x"))
         res = rstr.ll_strconcat(res, ll_int2hex(uid, False))
-        res = rstr.ll_strconcat(res, rstr.instance_str_suffix)
+        res = rstr.ll_strconcat(res, rstr.conststr(">"))
         return res
 
     def get_ll_eq_function(self):
@@ -1092,7 +1092,6 @@
     except StandardError:
         return None
 
-
 # ____________________________________________________________
 #
 #  Low-level implementation of operations on classes and instances
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -1,8 +1,12 @@
 from __future__ import with_statement
 
 import math
-import sys
+import sys, os
 
+if __name__ == '__main__':
+    # hack for test_hash_string_siphash24()
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__),
+                                    '..', '..', '..', '..'))
 import py
 
 from rpython.rlib.rstackovf import StackOverflow
@@ -597,6 +601,49 @@
         assert res[3] == compute_hash(d)
         assert res[4] == compute_hash(("Hi", None, (7.5, 2, d)))
 
+    def _test_hash_string(self, algo):
+        from rpython.rlib import objectmodel
+        objectmodel.set_hash_algorithm(algo)
+        s = "hello"
+        u = u"world"
+        v = u"\u1234\u2318+\u2bcd\u2102"
+        hash_s = compute_hash(s)
+        hash_u = compute_hash(u)
+        hash_v = compute_hash(v)
+        assert hash_s == compute_hash(u"hello")   # same hash because it's
+        assert hash_u == compute_hash("world")    #    a latin-1 unicode
+        #
+        def fn(length):
+            assert length >= 1
+            return str((compute_hash(s),
+                        compute_hash(u),
+                        compute_hash(v),
+                        compute_hash(s[0] + s[1:length]),
+                        compute_hash(u[0] + u[1:length]),
+                        compute_hash(v[0] + v[1:length]),
+                        ))
+
+        assert fn(5) == str((hash_s, hash_u, hash_v, hash_s, hash_u, hash_v))
+
+        f = self.getcompiled(fn, [int])
+        res = f(5)
+        res = [int(a) for a in res[1:-1].split(",")]
+        assert res[0] == hash_s
+        assert res[1] == hash_u
+        assert res[2] == hash_v
+        assert res[3] == hash_s
+        assert res[4] == hash_u
+        assert res[5] == hash_v
+
+    def test_hash_string_rpython(self):
+        self._test_hash_string("rpython")
+
+    def test_hash_string_siphash24(self):
+        import subprocess
+        subprocess.check_call([sys.executable, __file__, "siphash24",
+                               self.__class__.__module__,
+                               self.__class__.__name__])
+
     def test_list_basic_ops(self):
         def list_basic_ops(i, j):
             l = [1, 2, 3]
@@ -896,3 +943,11 @@
         f = self.getcompiled(func, [int])
         res = f(2)
         assert res == 1     # and not 2
+
+
+if __name__ == '__main__':
+    # for test_hash_string_siphash24()
+    algo, clsmodule, clsname = sys.argv[1:]
+    mod = __import__(clsmodule, None, None, [clsname])
+    cls = getattr(mod, clsname)
+    cls()._test_hash_string(algo)
diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -11,7 +11,8 @@
 from rpython.config.config import (to_optparse, OptionDescription, BoolOption,
     ArbitraryOption, StrOption, IntOption, Config, ChoiceOption, OptHelpFormatter)
 from rpython.config.translationoption import (get_combined_translation_config,
-    set_opt_level, OPT_LEVELS, DEFAULT_OPT_LEVEL, set_platform, CACHE_DIR)
+    set_opt_level, OPT_LEVELS, DEFAULT_OPT_LEVEL, set_platform, CACHE_DIR,
+    apply_extra_settings)
 
 # clean up early rpython/_cache
 try:
@@ -177,6 +178,9 @@
     if 'handle_config' in targetspec_dic:
         targetspec_dic['handle_config'](config, translateconfig)
 
+    # apply extra settings
+    apply_extra_settings(config)
+
     return targetspec_dic, translateconfig, config, args
 
 def show_help(translateconfig, opt_parser, targetspec_dic, config):

From pypy.commits at gmail.com  Thu Jan 26 03:42:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 00:42:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: change the default for the --hash option
 in py3.5
Message-ID: <5889b688.d30f1c0a.10f1.c8dd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89778:45239e9d6953
Date: 2017-01-26 09:42 +0100
http://bitbucket.org/pypy/pypy/changeset/45239e9d6953/

Log:	change the default for the --hash option in py3.5

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -229,6 +229,10 @@
             raise Exception("You have to specify the --opt level.\n"
                     "Try --opt=2 or --opt=jit, or equivalently -O2 or -Ojit .")
         self.translateconfig = translateconfig
+
+        # change the default for this option
+        config.translation.suggest(hash="siphash24")
+
         # set up the objspace optimizations based on the --opt argument
         from pypy.config.pypyoption import set_pypy_opt_level
         set_pypy_opt_level(config, translateconfig.opt)

From pypy.commits at gmail.com  Thu Jan 26 03:52:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 00:52:49 -0800 (PST)
Subject: [pypy-commit] pypy default: Document the problem (thanks njs)
Message-ID: <5889b8e1.1282df0a.bb340.70b3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89779:1042ceb32763
Date: 2017-01-26 09:51 +0100
http://bitbucket.org/pypy/pypy/changeset/1042ceb32763/

Log:	Document the problem (thanks njs)

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -21,6 +21,9 @@
 # that as easily because many details may rely on getting the same hash
 # value before and after translation.  We can, however, pick a random
 # seed once per translation, which should already be quite good.
+#
+# XXX no, it is not: e.g. all Ubuntu installations of the same Ubuntu
+# would get the same seed.  That's not good enough.
 
 @not_rpython
 def select_random_seed():

From pypy.commits at gmail.com  Thu Jan 26 03:52:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 00:52:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: revert to the default of --hash=rpython
Message-ID: <5889b8e3.c69bdf0a.b3cdd.6eb7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89780:1921e9b65a21
Date: 2017-01-26 09:52 +0100
http://bitbucket.org/pypy/pypy/changeset/1921e9b65a21/

Log:	revert to the default of --hash=rpython

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -231,7 +231,9 @@
         self.translateconfig = translateconfig
 
         # change the default for this option
-        config.translation.suggest(hash="siphash24")
+        # XXX disabled until we fix the real problem: a per-translation
+        # seed for siphash is bad
+        #config.translation.suggest(hash="siphash24")
 
         # set up the objspace optimizations based on the --opt argument
         from pypy.config.pypyoption import set_pypy_opt_level

From pypy.commits at gmail.com  Thu Jan 26 04:02:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 01:02:59 -0800 (PST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <5889bb43.8dae1c0a.52735.cf38@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89781:8ecc4591e7ec
Date: 2017-01-26 10:02 +0100
http://bitbucket.org/pypy/pypy/changeset/8ecc4591e7ec/

Log:	fix test

diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py
--- a/pypy/interpreter/test/test_appinterp.py
+++ b/pypy/interpreter/test/test_appinterp.py
@@ -23,7 +23,9 @@
     (): 
         y y 
     """)
-    assert str(excinfo.value.errorstr(space)).find('y y') != -1 
+    # NOTE: the following test only works because excinfo.value is not
+    # normalized so far
+    assert str(excinfo.value.get_w_value(space)).find('y y') != -1 
 
 def test_simple_applevel(space):
     app = appdef("""app(x,y): 

From pypy.commits at gmail.com  Thu Jan 26 04:14:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 01:14:12 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix
Message-ID: <5889bde4.da6d1c0a.c6601.dbf6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89782:9eb7c2912413
Date: 2017-01-26 10:13 +0100
http://bitbucket.org/pypy/pypy/changeset/9eb7c2912413/

Log:	Fix

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -71,7 +71,8 @@
 
     def errorstr(self, space, use_repr=False):
         "The exception class and value, as a string."
-        self.normalize_exception(space)
+        if not use_repr:    # see write_unraisable()
+            self.normalize_exception(space)
         w_value = self.get_w_value(space)
         if space is None:
             # this part NOT_RPYTHON
@@ -262,6 +263,11 @@
                     traceback.print_exception(t, v, tb)
                 """)
             else:
+                # Note that like CPython, we don't normalize the
+                # exception here.  So from `'foo'.index('bar')` you get
+                # "Exception ValueError: 'substring not found' in x ignored"
+                # but from `raise ValueError('foo')` you get
+                # "Exception ValueError: ValueError('foo',) in x ignored"
                 msg = 'Exception %s in %s%s ignored\n' % (
                     self.errorstr(space, use_repr=True), where, objrepr)
                 space.call_method(space.sys.get('stderr'), 'write',

From pypy.commits at gmail.com  Thu Jan 26 05:16:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 02:16:33 -0800 (PST)
Subject: [pypy-commit] cffi default: complain clearly if set_source() is
 given a /-separated name
Message-ID: <5889cc81.46bb1c0a.93437.f1c3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2875:510280ce4cc4
Date: 2017-01-26 11:14 +0100
http://bitbucket.org/cffi/cffi/changeset/510280ce4cc4/

Log:	complain clearly if set_source() is given a /-separated name

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -593,11 +593,15 @@
             ensure('extra_link_args', '/MANIFEST')
 
     def set_source(self, module_name, source, source_extension='.c', **kwds):
+        import os
         if hasattr(self, '_assigned_source'):
             raise ValueError("set_source() cannot be called several times "
                              "per ffi object")
         if not isinstance(module_name, basestring):
             raise TypeError("'module_name' must be a string")
+        if os.sep in module_name or (os.altsep and os.altsep in module_name):
+            raise ValueError("'module_name' must not contain '/': use a dotted "
+                             "name to make a 'package.module' location")
         self._assigned_source = (str(module_name), source,
                                  source_extension, kwds)
 
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -36,6 +36,11 @@
                                       ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
+def test_set_source_no_slashes():
+    ffi = FFI()
+    py.test.raises(ValueError, ffi.set_source, "abc/def", None)
+    py.test.raises(ValueError, ffi.set_source, "abc/def", "C code")
+
 
 def test_type_table_func():
     check_type_table("double sin(double);",

From pypy.commits at gmail.com  Thu Jan 26 06:27:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 03:27:05 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: A branch to try and remove the
 guarantee: "the rpython hash of any object doesn't change before and after
 translation"
Message-ID: <5889dd09.8c7e1c0a.a8214.49ee@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89783:533fd2120ac6
Date: 2017-01-26 12:12 +0100
http://bitbucket.org/pypy/pypy/changeset/533fd2120ac6/

Log:	A branch to try and remove the guarantee: "the rpython hash of any
	object doesn't change before and after translation"


From pypy.commits at gmail.com  Thu Jan 26 06:27:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 03:27:07 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: A quick disable of the
 hash-preservation logic, fix a few tests
Message-ID: <5889dd0b.85b2df0a.1da10.b227@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89784:11355355dff6
Date: 2017-01-26 12:26 +0100
http://bitbucket.org/pypy/pypy/changeset/11355355dff6/

Log:	A quick disable of the hash-preservation logic, fix a few tests

diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -11,7 +11,7 @@
 class BoehmGCTransformer(GCTransformer):
     malloc_zero_filled = True
     FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
-    HDR = lltype.Struct("header", ("hash", lltype.Signed))
+    HDR = lltype.Struct("header")
 
     def __init__(self, translator, inline=False):
         super(BoehmGCTransformer, self).__init__(translator, inline=inline)
@@ -33,9 +33,7 @@
 
         def ll_identityhash(addr):
             obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
-            h = obj.hash
-            if h == 0:
-                obj.hash = h = ~llmemory.cast_adr_to_int(addr)
+            h = ~llmemory.cast_adr_to_int(addr)
             return h
 
         if self.translator:
@@ -196,7 +194,7 @@
 
     def gcheader_initdata(self, obj):
         hdr = lltype.malloc(self.HDR, immortal=True)
-        hdr.hash = lltype.identityhash_nocache(obj._as_ptr())
+        #hdr.hash = lltype.identityhash_nocache(obj._as_ptr())
         return hdr._obj
 
 
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1518,7 +1518,7 @@
         hdr = self.gc_header_for(o, needs_hash)
         return hdr._obj
 
-    def get_prebuilt_hash(self, obj):
+    def DISABLED_get_prebuilt_hash(self, obj):
         # for prebuilt objects that need to have their hash stored and
         # restored.  Note that only structures that are StructNodes all
         # the way have their hash stored (and not e.g. structs with var-
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -663,6 +663,7 @@
 def gcstructnode_factory(db, T, obj):
     if (db.gctransformer and
             db.gctransformer.get_prebuilt_hash(obj) is not None):
+        DISABLED
         cls = GcStructNodeWithHash
     else:
         cls = StructNode
diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py
--- a/rpython/translator/c/test/test_boehm.py
+++ b/rpython/translator/c/test/test_boehm.py
@@ -381,7 +381,7 @@
                         current_object_addr_as_int(d2),
                         compute_hash(c),
                         compute_hash(d),
-                        compute_hash(("Hi", None, (7.5, 2, d)))))
+                        compute_hash(("Hi", None, (7.5, 2)))))
 
         f = self.getcompiled(fn)
         res = f()
@@ -390,8 +390,10 @@
         # xxx the next line is too precise, checking the exact implementation
         assert res[0] == ~res[1]
         assert res[2] != compute_hash(c)     # likely
-        assert res[3] == compute_hash(d)
-        assert res[4] == compute_hash(("Hi", None, (7.5, 2, d)))
+        assert res[3] != compute_hash(d)     # likely *not* preserved
+        assert res[4] == compute_hash(("Hi", None, (7.5, 2)))
+        # ^^ true as long as we're using the 'rpython' hash for strings
+        #    and not e.g. siphash24
 
     def test_finalizer_queue(self):
         class A(object):
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -780,7 +780,7 @@
         c = C()
         d = D()
         h_d = compute_hash(d)     # force to be cached on 'd', but not on 'c'
-        h_t = compute_hash(("Hi", None, (7.5, 2, d)))
+        h_t = compute_hash(("Hi", None, (7.5, 2)))
         S = lltype.GcStruct('S', ('x', lltype.Signed),
                                  ('a', lltype.Array(lltype.Signed)))
         s = lltype.malloc(S, 15, zero=True)
@@ -789,10 +789,10 @@
         def f():
             if compute_hash(c) != compute_identity_hash(c):
                 return 12
-            if compute_hash(d) != h_d:
+            if compute_hash(d) == h_d:     # likely *not* preserved
                 return 13
-            if compute_hash(("Hi", None, (7.5, 2, d))) != h_t:
-                return 14
+            if compute_hash(("Hi", None, (7.5, 2))) != h_t:
+                return 14   # ^^ true as long as we're not using e.g. siphash24
             c2 = C()
             h_c2 = compute_hash(c2)
             if compute_hash(c2) != h_c2:
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -588,7 +588,7 @@
                         current_object_addr_as_int(d2),
                         compute_hash(c),
                         compute_hash(d),
-                        compute_hash(("Hi", None, (7.5, 2, d)))))
+                        compute_hash(("Hi", None, (7.5, 2)))))
 
         f = self.getcompiled(fn, [])
         res = f()
@@ -598,8 +598,10 @@
         if res[0] != res[1]:
             assert res[0] == -res[1] - 1
         assert res[2] != compute_hash(c)     # likely
-        assert res[3] == compute_hash(d)
-        assert res[4] == compute_hash(("Hi", None, (7.5, 2, d)))
+        assert res[3] != compute_hash(d)     # likely *not* preserved
+        assert res[4] == compute_hash(("Hi", None, (7.5, 2)))
+        # ^^ true as long as we're using the 'rpython' hash for strings
+        #    and not e.g. siphash24
 
     def _test_hash_string(self, algo):
         from rpython.rlib import objectmodel

From pypy.commits at gmail.com  Thu Jan 26 06:36:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 03:36:12 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Disable support for explicitly
 non-ordered, non-empty prebuilt dicts
Message-ID: <5889df2c.545a1c0a.3e24d.101c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89785:ab8402f0c677
Date: 2017-01-26 12:35 +0100
http://bitbucket.org/pypy/pypy/changeset/ab8402f0c677/

Log:	Disable support for explicitly non-ordered, non-empty prebuilt dicts

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -236,21 +236,14 @@
                 if self.r_rdict_hashfn.lowleveltype != lltype.Void:
                     l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash)
                     l_dict.fnkeyhash = l_fn
-
-                for dictkeycontainer, dictvalue in dictobj._dict.items():
-                    llkey = r_key.convert_const(dictkeycontainer.key)
-                    llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        dictkeycontainer.hash)
-                return l_dict
-
+                any_items = dictobj._dict.items()
             else:
-                for dictkey, dictvalue in dictobj.items():
-                    llkey = r_key.convert_const(dictkey)
-                    llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        l_dict.keyhash(llkey))
-                return l_dict
+                any_items = dictobj.items()
+            if any_items:
+                raise TyperError("found a prebuilt, explicitly non-ordered, "
+                                 "non-empty dict.  it would require additional"
+                                 " support to rehash it at program start-up")
+            return l_dict
 
     def rtype_len(self, hop):
         v_dict, = hop.inputargs(self)

From pypy.commits at gmail.com  Thu Jan 26 08:20:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 05:20:25 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: in-progress: support dicts with no
 indexes
Message-ID: <5889f799.4395df0a.74e8d.da6f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89786:73f39f5b582e
Date: 2017-01-26 14:19 +0100
http://bitbucket.org/pypy/pypy/changeset/73f39f5b582e/

Log:	in-progress: support dicts with no indexes

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -46,20 +46,25 @@
 @jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
 @jit.oopspec('ordereddict.lookup(d, key, hash, flag)')
 def ll_call_lookup_function(d, key, hash, flag):
-    fun = d.lookup_function_no & FUNC_MASK
-    # This likely() here forces gcc to compile the check for fun == FUNC_BYTE
-    # first.  Otherwise, this is a regular switch and gcc (at least 4.7)
-    # compiles this as a series of checks, with the FUNC_BYTE case last.
-    # It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
-    if likely(fun == FUNC_BYTE):
-        return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
-    elif fun == FUNC_SHORT:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
-    elif IS_64BIT and fun == FUNC_INT:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
-    elif fun == FUNC_LONG:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
-    assert False
+    while True:
+        fun = d.lookup_function_no & FUNC_MASK
+        # This likely() here forces gcc to compile the check for fun==FUNC_BYTE
+        # first.  Otherwise, this is a regular switch and gcc (at least 4.7)
+        # compiles this as a series of checks, with the FUNC_BYTE case last.
+        # It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
+        if likely(fun == FUNC_BYTE):
+            return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
+        elif fun == FUNC_SHORT:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
+        elif IS_64BIT and fun == FUNC_INT:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
+        elif fun == FUNC_LONG:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
+        elif fun == FUNC_NO_INDEX:
+            ll_dict_create_index(d)
+            # then, retry
+        else:
+            assert False
 
 def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
                 ll_fasthash_function=None, ll_hash_function=None,
@@ -254,7 +259,6 @@
                     llvalue = r_value.convert_const(dictvalue)
                     _ll_dict_insertclean(l_dict, llkey, llvalue,
                                          dictkeycontainer.hash)
-                return l_dict
 
             else:
                 for dictkey, dictvalue in dictobj.items():
@@ -262,7 +266,9 @@
                     llvalue = r_value.convert_const(dictvalue)
                     _ll_dict_insertclean(l_dict, llkey, llvalue,
                                          l_dict.keyhash(llkey))
-                return l_dict
+
+            ll_remove_index(l_dict)
+            return l_dict
 
     def rtype_len(self, hop):
         v_dict, = hop.inputargs(self)
@@ -458,17 +464,27 @@
 
 IS_64BIT = sys.maxint != 2 ** 31 - 1
 
-FUNC_SHIFT = 2
-FUNC_MASK  = 0x03  # two bits
 if IS_64BIT:
-    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(4)
+    FUNC_SHIFT = 3
+    FUNC_MASK  = 0x07  # three bits
+    FUNC_NO_INDEX, FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(5)
 else:
-    FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(3)
+    FUNC_SHIFT = 2
+    FUNC_MASK  = 0x03  # two bits
+    FUNC_NO_INDEX, FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(4)
 TYPE_BYTE  = rffi.UCHAR
 TYPE_SHORT = rffi.USHORT
 TYPE_INT   = rffi.UINT
 TYPE_LONG  = lltype.Unsigned
 
+NULL = lltype.nullptr(llmemory.GCREF.TO)
+
+def ll_remove_index(d):
+    # Used in MemoryError situation, or when translating prebuilt dicts.
+    # Remove the index completely.
+    d.indexes = NULL
+    d.lookup_function_no = FUNC_NO_INDEX
+
 def ll_malloc_indexes_and_choose_lookup(d, n):
     # keep in sync with ll_clear_indexes() below
     if n <= 256:
@@ -503,22 +519,30 @@
         rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_INT, d.indexes))
     elif fun == FUNC_LONG:
         rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_LONG, d.indexes))
+    elif fun == FUNC_NO_INDEX:
+        # nothing to clear in this case
+        ll_assert(not d.indexes,
+                  "ll_clear_indexes: FUNC_NO_INDEX but d.indexes != NULL")
     else:
         assert False
 
 @jit.dont_look_inside
 def ll_call_insert_clean_function(d, hash, i):
-    fun = d.lookup_function_no & FUNC_MASK
-    if fun == FUNC_BYTE:
-        ll_dict_store_clean(d, hash, i, TYPE_BYTE)
-    elif fun == FUNC_SHORT:
-        ll_dict_store_clean(d, hash, i, TYPE_SHORT)
-    elif IS_64BIT and fun == FUNC_INT:
-        ll_dict_store_clean(d, hash, i, TYPE_INT)
-    elif fun == FUNC_LONG:
-        ll_dict_store_clean(d, hash, i, TYPE_LONG)
-    else:
-        assert False
+    while True:
+        fun = d.lookup_function_no & FUNC_MASK
+        if fun == FUNC_BYTE:
+            return ll_dict_store_clean(d, hash, i, TYPE_BYTE)
+        elif fun == FUNC_SHORT:
+            return ll_dict_store_clean(d, hash, i, TYPE_SHORT)
+        elif IS_64BIT and fun == FUNC_INT:
+            return ll_dict_store_clean(d, hash, i, TYPE_INT)
+        elif fun == FUNC_LONG:
+            return ll_dict_store_clean(d, hash, i, TYPE_LONG)
+        elif fun == FUNC_NO_INDEX:
+            ll_dict_create_index(d)
+            # then, retry
+        else:
+            assert False
 
 def ll_call_delete_by_entry_index(d, hash, i):
     fun = d.lookup_function_no & FUNC_MASK
@@ -530,6 +554,8 @@
         ll_dict_delete_by_entry_index(d, hash, i, TYPE_INT)
     elif fun == FUNC_LONG:
         ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
+    elif fun == FUNC_NO_INDEX:
+        pass   # there is no 'indexes', so nothing to delete
     else:
         assert False
 
@@ -614,7 +640,7 @@
             try:
                 reindexed = ll_dict_grow(d)
             except:
-                _ll_dict_rescue(d)
+                ll_remove_index(d)
                 raise
         rc = d.resize_counter - 3
         if rc <= 0:
@@ -622,7 +648,7 @@
                 ll_dict_resize(d)
                 reindexed = True
             except:
-                _ll_dict_rescue(d)
+                ll_remove_index(d)
                 raise
             rc = d.resize_counter - 3
             ll_assert(rc > 0, "ll_dict_resize failed?")
@@ -640,14 +666,6 @@
         d.num_ever_used_items += 1
         d.num_live_items += 1
 
- at jit.dont_look_inside
-def _ll_dict_rescue(d):
-    # MemoryError situation!  The 'indexes' contains an invalid entry
-    # at this point.  But we can call ll_dict_reindex() with the
-    # following arguments, ensuring no further malloc occurs.
-    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
-_ll_dict_rescue._dont_inline_ = True
-
 def _ll_dict_insertclean(d, key, value, hash):
     # never translated
     ENTRY = lltype.typeOf(d.entries).TO.OF
@@ -775,7 +793,20 @@
     else:
         d.entries = newitems
 
-    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+    # Estimate the index size we would like to have now.  If the new
+    # estimate changed from before, we need to throw away the old index
+    # anyway, so simply remove it for now.  If the size is the same,
+    # then maybe it is a better idea to keep it instead of reallocating
+    # an identical index very soon.  In this case we update the index now.
+    if bool(d.indexes):
+        new_estimate = d.num_live_items  * 2
+        new_size = DICT_INITSIZE
+        while new_size <= new_estimate:
+            new_size *= 2
+        if _ll_len_of_d_indexes(d) == new_size:
+            ll_dict_reindex(d, new_size)
+        else:
+            ll_remove_index(d)
 
 
 def ll_dict_delitem(d, key):
@@ -818,10 +849,10 @@
         assert j >= 0
         d.num_ever_used_items = j
 
-    # If the dictionary is at least 87.5% dead items, then consider shrinking
-    # it.
-    if d.num_live_items + DICT_INITSIZE <= len(d.entries) / 8:
-        ll_dict_resize(d)
+    # If the dictionary is more than 75% dead items, then clean up the
+    # deleted items and remove the index.
+    if d.num_live_items + DICT_INITSIZE < len(d.entries) / 4:
+        ll_dict_remove_deleted_items(d)
 
 def ll_dict_resize(d):
     # make a 'new_size' estimate and shrink it if there are many
@@ -839,11 +870,21 @@
     while new_size <= new_estimate:
         new_size *= 2
 
-    if new_size < _ll_len_of_d_indexes(d):
+    if bool(d.indexes) and new_size < _ll_len_of_d_indexes(d):
         ll_dict_remove_deleted_items(d)
     else:
         ll_dict_reindex(d, new_size)
 
+def ll_dict_create_index(d):
+    if d.num_live_items == 0:
+        new_size = DICT_INITSIZE     # fast path
+    else:
+        new_estimate = d.num_live_items  * 2
+        new_size = DICT_INITSIZE
+        while new_size <= new_estimate:
+            new_size *= 2
+    ll_dict_reindex(d, new_size)
+
 def ll_dict_reindex(d, new_size):
     if bool(d.indexes) and _ll_len_of_d_indexes(d) == new_size:
         ll_clear_indexes(d, new_size)   # hack: we can reuse the same array
@@ -1013,7 +1054,9 @@
 def ll_newdict(DICT):
     d = DICT.allocate()
     d.entries = _ll_empty_array(DICT)
-    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    # Don't allocate an 'indexes' for empty dict.  It seems a typical
+    # program contains tons of empty dicts, so this might be a memory win.
+    ll_remove_index(d)
     d.num_live_items = 0
     d.num_ever_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
@@ -1170,7 +1213,7 @@
             d_entry.f_hash = entry.f_hash
         i += 1
 
-    ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
+    ll_remove_index(newdict)
     return newdict
 ll_dict_copy.oopspec = 'odict.copy(dict)'
 
@@ -1180,7 +1223,7 @@
     DICT = lltype.typeOf(d).TO
     old_entries = d.entries
     d.entries = _ll_empty_array(DICT)
-    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    ll_remove_index(d)
     d.num_live_items = 0
     d.num_ever_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -196,10 +196,11 @@
         num = rordereddict._ll_dictnext(ll_iter)
         ll_key = ll_d.entries[num].key
         assert hlstr(ll_key) == "j"
-        assert ll_d.lookup_function_no == 4    # 1 free item found at the start
+        assert ll_d.lookup_function_no == (   # 1 free item found at the start
+            (1 << rordereddict.FUNC_SHIFT) | rordereddict.FUNC_BYTE)
         rordereddict.ll_dict_delitem(ll_d, llstr("j"))
         assert ll_d.num_ever_used_items == 0
-        assert ll_d.lookup_function_no == 0    # reset
+        assert ll_d.lookup_function_no == rordereddict.FUNC_BYTE   # reset
 
     def test_direct_enter_and_del(self):
         def eq(a, b):

From pypy.commits at gmail.com  Thu Jan 26 08:55:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 05:55:20 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Fix test_memoryerror by using the
 most tight estimate in
Message-ID: <5889ffc8.2da9df0a.7f654.eb3a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89787:9a1928019d6b
Date: 2017-01-26 14:54 +0100
http://bitbucket.org/pypy/pypy/changeset/9a1928019d6b/

Log:	Fix test_memoryerror by using the most tight estimate in
	ll_dict_create_index. This value should have the same guarantee in
	real program: if we get out of memory inserting a new item, then the
	recreated index is still of the smaller size instead of being bigger
	(and then likely not fitting in memory either)

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -879,9 +879,15 @@
     if d.num_live_items == 0:
         new_size = DICT_INITSIZE     # fast path
     else:
-        new_estimate = d.num_live_items  * 2
+        # Use a more conservative estimate than _ll_dict_resize_to() here.
+        # This function is called when initially creating the index (so,
+        # for prebuilt dicts, the chance is that it will never grow);
+        # after we got a MemoryError; and when throwing the index away
+        # because of heavy shrinking of the dict.  The minimum value
+        # here is such that 'new_estimate * 2 - num_live_items * 3 > 0'.
+        new_estimate = (d.num_live_items * 3) // 2 + 1
         new_size = DICT_INITSIZE
-        while new_size <= new_estimate:
+        while new_size < new_estimate:
             new_size *= 2
     ll_dict_reindex(d, new_size)
 

From pypy.commits at gmail.com  Thu Jan 26 09:00:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 06:00:55 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: better fast path
Message-ID: <588a0117.49acdf0a.9bc36.ef5f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89788:b24fe8a08009
Date: 2017-01-26 15:00 +0100
http://bitbucket.org/pypy/pypy/changeset/b24fe8a08009/

Log:	better fast path

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -876,7 +876,7 @@
         ll_dict_reindex(d, new_size)
 
 def ll_dict_create_index(d):
-    if d.num_live_items == 0:
+    if d.num_live_items < DICT_INITSIZE * 2 // 3:
         new_size = DICT_INITSIZE     # fast path
     else:
         # Use a more conservative estimate than _ll_dict_resize_to() here.

From pypy.commits at gmail.com  Thu Jan 26 10:55:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 07:55:44 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Check popitem() in the hypothesis
 test
Message-ID: <588a1c00.e4361c0a.29fd5.77cc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89789:90dc3abfc8f2
Date: 2017-01-26 16:55 +0100
http://bitbucket.org/pypy/pypy/changeset/90dc3abfc8f2/

Log:	Check popitem() in the hypothesis test

diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1155,6 +1155,8 @@
 
 class MappingSpace(object):
     def __init__(self, s_key, s_value):
+        from rpython.rtyper.rtuple import TupleRepr
+
         self.s_key = s_key
         self.s_value = s_value
         rtyper = PseudoRTyper()
@@ -1168,6 +1170,8 @@
         self.reference = self.new_reference()
         self.ll_key = r_key.convert_const
         self.ll_value = r_value.convert_const
+        r_tuple = TupleRepr(rtyper, [r_key, r_value])
+        self.TUPLE = r_tuple.lowleveltype
 
     def setitem(self, key, value):
         ll_key = self.ll_key(key)
@@ -1191,6 +1195,22 @@
         self.reference.clear()
         assert self.ll_len(self.l_dict) == 0
 
+    def popitem(self):
+        try:
+            ll_tuple = self.ll_popitem(self.TUPLE, self.l_dict)
+        except KeyError:
+            assert len(self.reference) == 0
+        else:
+            ll_key = ll_tuple.item0
+            ll_value = ll_tuple.item1
+            for key, value in self.reference.iteritems():
+                if self.ll_key(key) == ll_key:
+                    assert self.ll_value(value) == ll_value
+                    del self.reference[key]
+                    break
+            else:
+                raise AssertionError("popitem() returned unexpected key")
+
     def fullcheck(self):
         assert self.ll_len(self.l_dict) == len(self.reference)
         for key, value in self.reference.iteritems():
@@ -1217,7 +1237,8 @@
     def steps(self):
         if not self.space:
             return builds(Action, just('setup'), tuples(st_keys, st_values))
-        global_actions = [Action('copydict', ()), Action('cleardict', ())]
+        global_actions = [Action('copydict', ()), Action('cleardict', ()),
+                          Action('popitem', ())]
         if self.space.reference:
             return (
                 self.st_setitem() | sampled_from(global_actions) |
@@ -1249,6 +1270,7 @@
     ll_contains = staticmethod(rdict.ll_contains)
     ll_copy = staticmethod(rdict.ll_copy)
     ll_clear = staticmethod(rdict.ll_clear)
+    ll_popitem = staticmethod(rdict.ll_popitem)
 
     def newdict(self, repr):
         return rdict.ll_newdict(repr.DICT)
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -345,6 +345,7 @@
     ll_contains = staticmethod(rodct.ll_dict_contains)
     ll_copy = staticmethod(rodct.ll_dict_copy)
     ll_clear = staticmethod(rodct.ll_dict_clear)
+    ll_popitem = staticmethod(rodct.ll_dict_popitem)
 
     def newdict(self, repr):
         return rodct.ll_newdict(repr.DICT)
@@ -363,6 +364,20 @@
                 break
         return keys_ll
 
+    def popitem(self):
+        # overridden to check that we're getting the most recent key,
+        # not a random one
+        try:
+            ll_tuple = self.ll_popitem(self.TUPLE, self.l_dict)
+        except KeyError:
+            assert len(self.reference) == 0
+        else:
+            ll_key = ll_tuple.item0
+            ll_value = ll_tuple.item1
+            key, value = self.reference.popitem()
+            assert self.ll_key(key) == ll_key
+            assert self.ll_value(value) == ll_value
+
     def fullcheck(self):
         # overridden to also check key order
         assert self.ll_len(self.l_dict) == len(self.reference)
@@ -379,4 +394,4 @@
 
 def test_hypothesis():
     run_state_machine_as_test(
-        ODictSM, settings(max_examples=500, stateful_step_count=100))
+        ODictSM, settings(max_examples=50000, stateful_step_count=100))

From pypy.commits at gmail.com  Thu Jan 26 12:02:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 26 Jan 2017 09:02:25 -0800 (PST)
Subject: [pypy-commit] pypy default: Cleanup: don't pass the whole CConfig
 to places that only need an ECI
Message-ID: <588a2ba1.46bb1c0a.93437.980f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89790:ae071d58c03a
Date: 2017-01-26 17:01 +0000
http://bitbucket.org/pypy/pypy/changeset/ae071d58c03a/

Log:	Cleanup: don't pass the whole CConfig to places that only need an
	ECI

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -131,8 +131,8 @@
 # General interface
 
 class ConfigResult:
-    def __init__(self, CConfig, info, entries):
-        self.CConfig = CConfig
+    def __init__(self, eci, info, entries):
+        self.eci = eci
         self.result = {}
         self.info = info
         self.entries = entries
@@ -154,15 +154,14 @@
 class _CWriter(object):
     """ A simple class which aggregates config parts
     """
-    def __init__(self, CConfig):
+    def __init__(self, eci):
         self.path = uniquefilepath()
         self.f = self.path.open("w")
-        self.config = CConfig
+        self.eci = eci
 
     def write_header(self):
         f = self.f
-        CConfig = self.config
-        CConfig._compilation_info_.write_c_header(f)
+        self.eci.write_c_header(f)
         print >> f, C_HEADER
         print >> f
 
@@ -194,8 +193,7 @@
         self.start_main()
         self.f.write(question + "\n")
         self.close()
-        eci = self.config._compilation_info_
-        try_compile_cache([self.path], eci)
+        try_compile_cache([self.path], self.eci)
 
 def configure(CConfig, ignore_errors=False):
     """Examine the local system by running the C compiler.
@@ -208,6 +206,7 @@
         assert not hasattr(CConfig, attr), \
             "Found legacy attribute %s on CConfig" % attr
 
+    eci = CConfig._compilation_info_
     entries = []
     for key in dir(CConfig):
         value = getattr(CConfig, key)
@@ -215,7 +214,7 @@
             entries.append((key, value))
 
     if entries:   # can be empty if there are only CConfigSingleEntries
-        writer = _CWriter(CConfig)
+        writer = _CWriter(eci)
         writer.write_header()
         for key, entry in entries:
             writer.write_entry(key, entry)
@@ -225,7 +224,6 @@
             writer.write_entry_main(key)
         writer.close()
 
-        eci = CConfig._compilation_info_
         infolist = list(run_example_code(writer.path, eci,
                                          ignore_errors=ignore_errors))
         assert len(infolist) == len(entries)
@@ -236,7 +234,7 @@
             resultinfo[key] = info
             resultentries[entry] = key
 
-        result = ConfigResult(CConfig, resultinfo, resultentries)
+        result = ConfigResult(eci, resultinfo, resultentries)
         for name, entry in entries:
             result.get_entry_result(entry)
         res = result.get_result()
@@ -246,7 +244,7 @@
     for key in dir(CConfig):
         value = getattr(CConfig, key)
         if isinstance(value, CConfigSingleEntry):
-            writer = _CWriter(CConfig)
+            writer = _CWriter(eci)
             writer.write_header()
             res[key] = value.question(writer.ask_gcc)
 
@@ -344,7 +342,7 @@
         allfields = tuple(['c_' + name for name, _ in fields])
         padfields = tuple(padfields)
         name = self.name
-        eci = config_result.CConfig._compilation_info_
+        eci = config_result.eci
         padding_drop = PaddingDrop(name, allfields, padfields, eci)
         hints = {'align': info['align'],
                  'size': info['size'],

From pypy.commits at gmail.com  Thu Jan 26 12:29:48 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 26 Jan 2017 09:29:48 -0800 (PST)
Subject: [pypy-commit] pypy default: Extract configure_entries() from
 configure()
Message-ID: <588a320c.977d1c0a.4b5c2.f9cd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89791:5cf849b8cd32
Date: 2017-01-26 17:29 +0000
http://bitbucket.org/pypy/pypy/changeset/5cf849b8cd32/

Log:	Extract configure_entries() from configure()

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -214,30 +214,7 @@
             entries.append((key, value))
 
     if entries:   # can be empty if there are only CConfigSingleEntries
-        writer = _CWriter(eci)
-        writer.write_header()
-        for key, entry in entries:
-            writer.write_entry(key, entry)
-
-        writer.start_main()
-        for key, entry in entries:
-            writer.write_entry_main(key)
-        writer.close()
-
-        infolist = list(run_example_code(writer.path, eci,
-                                         ignore_errors=ignore_errors))
-        assert len(infolist) == len(entries)
-
-        resultinfo = {}
-        resultentries = {}
-        for info, (key, entry) in zip(infolist, entries):
-            resultinfo[key] = info
-            resultentries[entry] = key
-
-        result = ConfigResult(eci, resultinfo, resultentries)
-        for name, entry in entries:
-            result.get_entry_result(entry)
-        res = result.get_result()
+        res = configure_entries(entries, eci, ignore_errors=ignore_errors)
     else:
         res = {}
 
@@ -250,6 +227,33 @@
 
     return res
 
+
+def configure_entries(entries, eci, ignore_errors=False):
+    writer = _CWriter(eci)
+    writer.write_header()
+    for key, entry in entries:
+        writer.write_entry(key, entry)
+
+    writer.start_main()
+    for key, entry in entries:
+        writer.write_entry_main(key)
+    writer.close()
+
+    infolist = list(run_example_code(
+        writer.path, eci, ignore_errors=ignore_errors))
+    assert len(infolist) == len(entries)
+
+    resultinfo = {}
+    resultentries = {}
+    for info, (key, entry) in zip(infolist, entries):
+        resultinfo[key] = info
+        resultentries[entry] = key
+
+    result = ConfigResult(eci, resultinfo, resultentries)
+    for name, entry in entries:
+        result.get_entry_result(entry)
+    return result.get_result()
+
 # ____________________________________________________________
 
 

From pypy.commits at gmail.com  Thu Jan 26 12:47:52 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 26 Jan 2017 09:47:52 -0800 (PST)
Subject: [pypy-commit] pypy default: Use rffi_platform.configure_entries()
 in cparser
Message-ID: <588a3648.2a99df0a.43838.4e58@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89792:75fb54ed4989
Date: 2017-01-26 17:46 +0000
http://bitbucket.org/pypy/pypy/changeset/75fb54ed4989/

Log:	Use rffi_platform.configure_entries() in cparser

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -699,7 +699,7 @@
         self.headers = headers if headers is not None else ['sys/types.h']
         self.parsed_headers = []
         self.sources = []
-        self._Config = type('Config', (object,), {})
+        self._config_entries = []
         self._TYPES = {}
         self.includes = []
         self.struct_typedefs = {}
@@ -759,8 +759,9 @@
     def realize_struct(self, struct):
         type_name = struct.get_type_name()
         configname = type_name.replace(' ', '__')
-        setattr(self._Config, configname,
-            rffi_platform.Struct(type_name, struct.fields))
+        assert not any(x[0] == configname for x in self._config_entries)
+        self._config_entries.append(
+            (configname, rffi_platform.Struct(type_name, struct.fields)))
         self._TYPES[configname] = struct.TYPE
         return struct.TYPE
 
@@ -795,13 +796,17 @@
             elif name.startswith('macro '):
                 name = name[6:]
                 self.add_macro(name, obj)
-        self._Config._compilation_info_ = self.build_eci()
-        for name, TYPE in rffi_platform.configure(self._Config).iteritems():
+        if not self._config_entries:
+            return
+        eci = self.build_eci()
+        result = rffi_platform.configure_entries(self._config_entries, eci)
+        for name, TYPE in result.iteritems():
             # hack: prevent the source from being pasted into common_header.h
             del TYPE._hints['eci']
             if name in self._TYPES:
                 self._TYPES[name].become(TYPE)
                 del self._TYPES[name]
+        self._config_entries[:] = []
 
     def convert_type(self, obj, quals=0):
         if isinstance(obj, model.DefinedType):

From pypy.commits at gmail.com  Thu Jan 26 14:55:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 26 Jan 2017 11:55:25 -0800 (PST)
Subject: [pypy-commit] pypy default: Preserve the order of entries in
 configure_entries() which now takes a list instead of a dict
Message-ID: <588a542d.810b1c0a.d9417.0e69@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89793:e484cf6b06b6
Date: 2017-01-26 19:54 +0000
http://bitbucket.org/pypy/pypy/changeset/e484cf6b06b6/

Log:	Preserve the order of entries in configure_entries() which now takes
	a list instead of a dict

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -699,8 +699,7 @@
         self.headers = headers if headers is not None else ['sys/types.h']
         self.parsed_headers = []
         self.sources = []
-        self._config_entries = []
-        self._TYPES = {}
+        self._config_entries = OrderedDict()
         self.includes = []
         self.struct_typedefs = {}
         self._handled = set()
@@ -758,11 +757,8 @@
 
     def realize_struct(self, struct):
         type_name = struct.get_type_name()
-        configname = type_name.replace(' ', '__')
-        assert not any(x[0] == configname for x in self._config_entries)
-        self._config_entries.append(
-            (configname, rffi_platform.Struct(type_name, struct.fields)))
-        self._TYPES[configname] = struct.TYPE
+        entry = rffi_platform.Struct(type_name, struct.fields)
+        self._config_entries[entry] = struct.TYPE
         return struct.TYPE
 
     def build_eci(self):
@@ -799,14 +795,12 @@
         if not self._config_entries:
             return
         eci = self.build_eci()
-        result = rffi_platform.configure_entries(self._config_entries, eci)
-        for name, TYPE in result.iteritems():
+        result = rffi_platform.configure_entries(list(self._config_entries), eci)
+        for entry, TYPE in zip(self._config_entries, result):
             # hack: prevent the source from being pasted into common_header.h
             del TYPE._hints['eci']
-            if name in self._TYPES:
-                self._TYPES[name].become(TYPE)
-                del self._TYPES[name]
-        self._config_entries[:] = []
+            self._config_entries[entry].become(TYPE)
+        self._config_entries.clear()
 
     def convert_type(self, obj, quals=0):
         if isinstance(obj, model.DefinedType):
diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -131,25 +131,20 @@
 # General interface
 
 class ConfigResult:
-    def __init__(self, eci, info, entries):
+    def __init__(self, eci, info):
         self.eci = eci
+        self.info = info
         self.result = {}
-        self.info = info
-        self.entries = entries
 
     def get_entry_result(self, entry):
         try:
             return self.result[entry]
         except KeyError:
             pass
-        name = self.entries[entry]
-        info = self.info[name]
+        info = self.info[entry]
         self.result[entry] = entry.build_result(info, self)
         return self.result[entry]
 
-    def get_result(self):
-        return dict([(name, self.result[entry])
-                     for entry, name in self.entries.iteritems()])
 
 class _CWriter(object):
     """ A simple class which aggregates config parts
@@ -207,16 +202,18 @@
             "Found legacy attribute %s on CConfig" % attr
 
     eci = CConfig._compilation_info_
-    entries = []
+    entries = {}
     for key in dir(CConfig):
         value = getattr(CConfig, key)
         if isinstance(value, CConfigEntry):
-            entries.append((key, value))
+            entries[key] = value
 
+    res = {}
     if entries:   # can be empty if there are only CConfigSingleEntries
-        res = configure_entries(entries, eci, ignore_errors=ignore_errors)
-    else:
-        res = {}
+        results = configure_entries(
+            entries.values(), eci, ignore_errors=ignore_errors)
+        for name, result in zip(entries, results):
+            res[name] = result
 
     for key in dir(CConfig):
         value = getattr(CConfig, key)
@@ -231,12 +228,12 @@
 def configure_entries(entries, eci, ignore_errors=False):
     writer = _CWriter(eci)
     writer.write_header()
-    for key, entry in entries:
-        writer.write_entry(key, entry)
+    for i, entry in enumerate(entries):
+        writer.write_entry(str(i), entry)
 
     writer.start_main()
-    for key, entry in entries:
-        writer.write_entry_main(key)
+    for i, entry in enumerate(entries):
+        writer.write_entry_main(str(i))
     writer.close()
 
     infolist = list(run_example_code(
@@ -244,15 +241,11 @@
     assert len(infolist) == len(entries)
 
     resultinfo = {}
-    resultentries = {}
-    for info, (key, entry) in zip(infolist, entries):
-        resultinfo[key] = info
-        resultentries[entry] = key
+    for info, entry in zip(infolist, entries):
+        resultinfo[entry] = info
 
-    result = ConfigResult(eci, resultinfo, resultentries)
-    for name, entry in entries:
-        result.get_entry_result(entry)
-    return result.get_result()
+    result = ConfigResult(eci, resultinfo)
+    return [result.get_entry_result(entry) for entry in entries]
 
 # ____________________________________________________________
 

From pypy.commits at gmail.com  Thu Jan 26 16:38:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 26 Jan 2017 13:38:12 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Tweaks, possibly fix a bug
Message-ID: <588a6c44.8887df0a.945be.a3e4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89794:eaa33d328249
Date: 2017-01-26 22:05 +0100
http://bitbucket.org/pypy/pypy/changeset/eaa33d328249/

Log:	Tweaks, possibly fix a bug

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -642,19 +642,17 @@
             except:
                 ll_remove_index(d)
                 raise
-        rc = d.resize_counter - 3
-        if rc <= 0:
+        if d.resize_counter <= 3:
             try:
                 ll_dict_resize(d)
                 reindexed = True
             except:
                 ll_remove_index(d)
                 raise
-            rc = d.resize_counter - 3
-            ll_assert(rc > 0, "ll_dict_resize failed?")
         if reindexed:
             ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
-        #
+        rc = d.resize_counter - 3
+        ll_assert(rc > 0, "ll_dict_resize failed?")
         d.resize_counter = rc
         entry = d.entries[d.num_ever_used_items]
         entry.key = key
@@ -799,7 +797,7 @@
     # then maybe it is a better idea to keep it instead of reallocating
     # an identical index very soon.  In this case we update the index now.
     if bool(d.indexes):
-        new_estimate = d.num_live_items  * 2
+        new_estimate = d.num_live_items * 2
         new_size = DICT_INITSIZE
         while new_size <= new_estimate:
             new_size *= 2
@@ -842,12 +840,13 @@
         # also possible that there are more dead items immediately behind the
         # last one, we reclaim all the dead items at the end of the ordereditem
         # at the same point.
-        i = d.num_ever_used_items - 2
-        while i >= 0 and not d.entries.valid(i):
+        i = index
+        while True:
             i -= 1
-        j = i + 1
-        assert j >= 0
-        d.num_ever_used_items = j
+            assert i >= 0
+            if d.entries.valid(i):    # must be at least one
+                break
+        d.num_ever_used_items = i + 1
 
     # If the dictionary is more than 75% dead items, then clean up the
     # deleted items and remove the index.
@@ -1065,7 +1064,6 @@
     ll_remove_index(d)
     d.num_live_items = 0
     d.num_ever_used_items = 0
-    d.resize_counter = DICT_INITSIZE * 2
     return d
 OrderedDictRepr.ll_newdict = staticmethod(ll_newdict)
 
@@ -1232,7 +1230,6 @@
     ll_remove_index(d)
     d.num_live_items = 0
     d.num_ever_used_items = 0
-    d.resize_counter = DICT_INITSIZE * 2
     # old_entries.delete() XXX
 ll_dict_clear.oopspec = 'odict.clear(d)'
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1170,6 +1170,7 @@
         self.reference = self.new_reference()
         self.ll_key = r_key.convert_const
         self.ll_value = r_value.convert_const
+        self.removed_keys = []
         r_tuple = TupleRepr(rtyper, [r_key, r_value])
         self.TUPLE = r_tuple.lowleveltype
 
@@ -1185,6 +1186,7 @@
         self.ll_delitem(self.l_dict, ll_key)
         del self.reference[key]
         assert not self.ll_contains(self.l_dict, ll_key)
+        self.removed_keys.append(key)
 
     def copydict(self):
         self.l_dict = self.ll_copy(self.l_dict)
@@ -1192,6 +1194,8 @@
 
     def cleardict(self):
         self.ll_clear(self.l_dict)
+        for key in self.reference:
+            self.removed_keys.append(key)
         self.reference.clear()
         assert self.ll_len(self.l_dict) == 0
 
@@ -1207,15 +1211,27 @@
                 if self.ll_key(key) == ll_key:
                     assert self.ll_value(value) == ll_value
                     del self.reference[key]
+                    self.removed_keys.append(key)
                     break
             else:
                 raise AssertionError("popitem() returned unexpected key")
 
+    def removeindex(self):
+        pass     # overridden in test_rordereddict
+
     def fullcheck(self):
         assert self.ll_len(self.l_dict) == len(self.reference)
         for key, value in self.reference.iteritems():
             assert (self.ll_getitem(self.l_dict, self.ll_key(key)) ==
                 self.ll_value(value))
+        for key in self.removed_keys:
+            if key not in self.reference:
+                try:
+                    self.ll_getitem(self.l_dict, self.ll_key(key))
+                except KeyError:
+                    pass
+                else:
+                    raise AssertionError("removed key still shows up")
 
 class MappingSM(GenericStateMachine):
     def __init__(self):
@@ -1238,7 +1254,7 @@
         if not self.space:
             return builds(Action, just('setup'), tuples(st_keys, st_values))
         global_actions = [Action('copydict', ()), Action('cleardict', ()),
-                          Action('popitem', ())]
+                          Action('popitem', ()), Action('removeindex', ())]
         if self.space.reference:
             return (
                 self.st_setitem() | sampled_from(global_actions) |
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -202,14 +202,17 @@
         assert ll_d.num_ever_used_items == 0
         assert ll_d.lookup_function_no == rordereddict.FUNC_BYTE   # reset
 
-    def test_direct_enter_and_del(self):
+    def _get_int_dict(self):
         def eq(a, b):
             return a == b
 
-        DICT = rordereddict.get_ll_dict(lltype.Signed, lltype.Signed,
+        return rordereddict.get_ll_dict(lltype.Signed, lltype.Signed,
                                  ll_fasthash_function=intmask,
                                  ll_hash_function=intmask,
                                  ll_eq_function=eq)
+
+    def test_direct_enter_and_del(self):
+        DICT = self._get_int_dict()
         ll_d = rordereddict.ll_newdict(DICT)
         numbers = [i * rordereddict.DICT_INITSIZE + 1 for i in range(8)]
         for num in numbers:
@@ -303,6 +306,38 @@
         rordereddict.ll_prepare_dict_update(ll_d, 7)
         # used to get UninitializedMemoryAccess
 
+    def test_bug_resize_counter(self):
+        DICT = self._get_int_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, 0, 0)
+        rordereddict.ll_dict_delitem(ll_d, 0)
+        rordereddict.ll_dict_setitem(ll_d, 0, 0)
+        rordereddict.ll_dict_delitem(ll_d, 0)
+        rordereddict.ll_dict_setitem(ll_d, 0, 0)
+        rordereddict.ll_dict_delitem(ll_d, 0)
+        rordereddict.ll_dict_setitem(ll_d, 0, 0)
+        rordereddict.ll_dict_delitem(ll_d, 0)
+        rordereddict.ll_dict_setitem(ll_d, 1, 0)
+        rordereddict.ll_dict_setitem(ll_d, 0, 0)
+        rordereddict.ll_dict_setitem(ll_d, 2, 0)
+        rordereddict.ll_dict_delitem(ll_d, 1)
+        rordereddict.ll_dict_delitem(ll_d, 0)
+        rordereddict.ll_dict_delitem(ll_d, 2)
+        rordereddict.ll_dict_setitem(ll_d, 0, 0)
+        rordereddict.ll_dict_delitem(ll_d, 0)
+        rordereddict.ll_dict_setitem(ll_d, 0, 0)
+        rordereddict.ll_dict_delitem(ll_d, 0)
+        rordereddict.ll_dict_setitem(ll_d, 0, 0)
+        rordereddict.ll_dict_setitem(ll_d, 1, 0)
+        d = ll_d
+        idx = d.indexes._obj.container
+        num_nonfrees = 0
+        for i in range(idx.getlength()):
+            got = idx.getitem(i)   # 0: unused; 1: deleted
+            num_nonfrees += (got > 0)
+        assert d.resize_counter <= idx.getlength() * 2 - num_nonfrees * 3
+
+
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
         ll_dummy_value = llstr("dupa")
@@ -377,6 +412,12 @@
             key, value = self.reference.popitem()
             assert self.ll_key(key) == ll_key
             assert self.ll_value(value) == ll_value
+            self.removed_keys.append(key)
+
+    def removeindex(self):
+        # remove the index, as done e.g. during translation for prebuilt
+        # dicts
+        rodct.ll_remove_index(self.l_dict)
 
     def fullcheck(self):
         # overridden to also check key order
@@ -387,6 +428,35 @@
             assert self.ll_key(key) == ll_key
             assert (self.ll_getitem(self.l_dict, self.ll_key(key)) ==
                 self.ll_value(self.reference[key]))
+        for key in self.removed_keys:
+            if key not in self.reference:
+                try:
+                    self.ll_getitem(self.l_dict, self.ll_key(key))
+                except KeyError:
+                    pass
+                else:
+                    raise AssertionError("removed key still shows up")
+        # check some internal invariants
+        d = self.l_dict
+        num_lives = 0
+        for i in range(d.num_ever_used_items):
+            if d.entries.valid(i):
+                num_lives += 1
+        assert num_lives == d.num_live_items
+        fun = d.lookup_function_no & rordereddict.FUNC_MASK
+        if fun == rordereddict.FUNC_NO_INDEX:
+            assert not d.indexes
+        else:
+            assert d.indexes
+            idx = d.indexes._obj.container
+            num_lives = 0
+            num_nonfrees = 0
+            for i in range(idx.getlength()):
+                got = idx.getitem(i)   # 0: unused; 1: deleted
+                num_nonfrees += (got > 0)
+                num_lives += (got > 1)
+            assert num_lives == d.num_live_items
+            assert 0 < d.resize_counter <= idx.getlength()*2 - num_nonfrees*3
 
 
 class ODictSM(MappingSM):
@@ -394,4 +464,4 @@
 
 def test_hypothesis():
     run_state_machine_as_test(
-        ODictSM, settings(max_examples=50000, stateful_step_count=100))
+        ODictSM, settings(max_examples=500, stateful_step_count=100))

From pypy.commits at gmail.com  Fri Jan 27 03:00:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 27 Jan 2017 00:00:21 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Tweaks & comments
Message-ID: <588afe15.8887df0a.945be.31e0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89795:40b971b7a0c7
Date: 2017-01-27 08:59 +0100
http://bitbucket.org/pypy/pypy/changeset/40b971b7a0c7/

Log:	Tweaks & comments

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -539,6 +539,10 @@
         elif fun == FUNC_LONG:
             return ll_dict_store_clean(d, hash, i, TYPE_LONG)
         elif fun == FUNC_NO_INDEX:
+            # NB. this case might be reachable or not, but I didn't manage
+            # to get here.  Maybe it is not actually reachable right now
+            # but it depends on details.  Better keep it written down
+            # just in case the details change later.
             ll_dict_create_index(d)
             # then, retry
         else:
@@ -903,12 +907,35 @@
     entries = d.entries
     i = 0
     ibound = d.num_ever_used_items
-    while i < ibound:
-        if entries.valid(i):
-            hash = entries.hash(i)
-            ll_call_insert_clean_function(d, hash, i)
-        i += 1
-    #old_entries.delete() XXXX!
+    #
+    # Write four loops, moving the check for the value of 'fun' out of
+    # the loops.  A small speed-up, it also avoids the (unreachable)
+    # recursive call from here to ll_call_insert_clean_function() to
+    # ll_dict_create_index() back to here.
+    fun = d.lookup_function_no     # == lookup_function_no & FUNC_MASK
+    if fun == FUNC_BYTE:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_BYTE)
+            i += 1
+    elif fun == FUNC_SHORT:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_SHORT)
+            i += 1
+    elif IS_64BIT and fun == FUNC_INT:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_INT)
+            i += 1
+    elif fun == FUNC_LONG:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_LONG)
+            i += 1
+    else:
+        assert False
+
 
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5

From pypy.commits at gmail.com  Fri Jan 27 05:28:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 27 Jan 2017 02:28:36 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: First place to fix inside pypy
Message-ID: <588b20d4.6f98df0a.bc1d7.6b77@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89796:f6f8a968971c
Date: 2017-01-27 09:05 +0100
http://bitbucket.org/pypy/pypy/changeset/f6f8a968971c/

Log:	First place to fix inside pypy

diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -193,6 +193,15 @@
         W_WeakrefBase.__init__(self, space, w_obj, w_callable)
         self.w_hash = None
 
+    def _cleanup_(self):
+        # When a prebuilt weakref is frozen inside a translation, if
+        # this weakref has got an already-cached w_hash, then throw it
+        # away.  That's because the hash value will change after
+        # translation.  It will be recomputed the first time we ask for
+        # it.  Note that such a frozen weakref, if not dead, will point
+        # to a frozen object, so it will never die.
+        self.w_hash = None
+
     def descr__init__weakref(self, space, w_obj, w_callable=None,
                              __args__=None):
         if __args__.arguments_w:

From pypy.commits at gmail.com  Fri Jan 27 05:28:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 27 Jan 2017 02:28:38 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Revert some changes: now only
 dicts initially created and dicts frozen
Message-ID: <588b20d6.2a99df0a.43838.6b9f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89797:a0cc818768af
Date: 2017-01-27 11:27 +0100
http://bitbucket.org/pypy/pypy/changeset/a0cc818768af/

Log:	Revert some changes: now only dicts initially created and dicts
	frozen by translation have no index. Simplifies a bit the diff with
	default---I could find and fix more bugs.

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib import objectmodel, jit, rgc, types
 from rpython.rlib.signature import signature
-from rpython.rlib.objectmodel import specialize, likely
+from rpython.rlib.objectmodel import specialize, likely, not_rpython
 from rpython.rtyper.debug import ll_assert
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rtyper import rmodel
@@ -60,11 +60,9 @@
             return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
         elif fun == FUNC_LONG:
             return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
-        elif fun == FUNC_NO_INDEX:
-            ll_dict_create_index(d)
+        else:
+            ll_dict_create_initial_index(d)
             # then, retry
-        else:
-            assert False
 
 def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
                 ll_fasthash_function=None, ll_hash_function=None,
@@ -240,6 +238,7 @@
             self.setup()
             self.setup_final()
             l_dict = ll_newdict_size(self.DICT, len(dictobj))
+            ll_no_initial_index(l_dict)
             self.dict_cache[key] = l_dict
             r_key = self.key_repr
             if r_key.lowleveltype == llmemory.Address:
@@ -257,18 +256,15 @@
                 for dictkeycontainer, dictvalue in dictobj._dict.items():
                     llkey = r_key.convert_const(dictkeycontainer.key)
                     llvalue = r_value.convert_const(dictvalue)
-                    _ll_dict_insertclean(l_dict, llkey, llvalue,
-                                         dictkeycontainer.hash)
+                    _ll_dict_insert_no_index(l_dict, llkey, llvalue)
+                return l_dict
 
             else:
                 for dictkey, dictvalue in dictobj.items():
                     llkey = r_key.convert_const(dictkey)
                     llvalue = r_value.convert_const(dictvalue)
-                    _ll_dict_insertclean(l_dict, llkey, llvalue,
-                                         l_dict.keyhash(llkey))
-
-            ll_remove_index(l_dict)
-            return l_dict
+                    _ll_dict_insert_no_index(l_dict, llkey, llvalue)
+                return l_dict
 
     def rtype_len(self, hop):
         v_dict, = hop.inputargs(self)
@@ -467,23 +463,26 @@
 if IS_64BIT:
     FUNC_SHIFT = 3
     FUNC_MASK  = 0x07  # three bits
-    FUNC_NO_INDEX, FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(5)
+    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG, FUNC_MUST_REINDEX = range(5)
 else:
     FUNC_SHIFT = 2
     FUNC_MASK  = 0x03  # two bits
-    FUNC_NO_INDEX, FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(4)
+    FUNC_BYTE, FUNC_SHORT, FUNC_LONG, FUNC_MUST_REINDEX = range(4)
 TYPE_BYTE  = rffi.UCHAR
 TYPE_SHORT = rffi.USHORT
 TYPE_INT   = rffi.UINT
 TYPE_LONG  = lltype.Unsigned
 
-NULL = lltype.nullptr(llmemory.GCREF.TO)
-
-def ll_remove_index(d):
-    # Used in MemoryError situation, or when translating prebuilt dicts.
-    # Remove the index completely.
-    d.indexes = NULL
-    d.lookup_function_no = FUNC_NO_INDEX
+def ll_no_initial_index(d):
+    # Used when making new empty dicts, and when translating prebuilt dicts.
+    # Remove the index completely.  A dictionary must always have an
+    # index unless it is freshly created or freshly translated.  Most
+    # dict operations start with ll_call_lookup_function(), which will
+    # recompute the hashes and create the index.
+    ll_assert(d.num_live_items == d.num_ever_used_items, 
+         "ll_no_initial_index(): dict already in use")
+    d.lookup_function_no = FUNC_MUST_REINDEX
+    d.indexes = lltype.nullptr(llmemory.GCREF.TO)
 
 def ll_malloc_indexes_and_choose_lookup(d, n):
     # keep in sync with ll_clear_indexes() below
@@ -519,34 +518,24 @@
         rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_INT, d.indexes))
     elif fun == FUNC_LONG:
         rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_LONG, d.indexes))
-    elif fun == FUNC_NO_INDEX:
-        # nothing to clear in this case
-        ll_assert(not d.indexes,
-                  "ll_clear_indexes: FUNC_NO_INDEX but d.indexes != NULL")
     else:
         assert False
 
 @jit.dont_look_inside
 def ll_call_insert_clean_function(d, hash, i):
-    while True:
-        fun = d.lookup_function_no & FUNC_MASK
-        if fun == FUNC_BYTE:
-            return ll_dict_store_clean(d, hash, i, TYPE_BYTE)
-        elif fun == FUNC_SHORT:
-            return ll_dict_store_clean(d, hash, i, TYPE_SHORT)
-        elif IS_64BIT and fun == FUNC_INT:
-            return ll_dict_store_clean(d, hash, i, TYPE_INT)
-        elif fun == FUNC_LONG:
-            return ll_dict_store_clean(d, hash, i, TYPE_LONG)
-        elif fun == FUNC_NO_INDEX:
-            # NB. this case might be reachable or not, but I didn't manage
-            # to get here.  Maybe it is not actually reachable right now
-            # but it depends on details.  Better keep it written down
-            # just in case the details change later.
-            ll_dict_create_index(d)
-            # then, retry
-        else:
-            assert False
+    fun = d.lookup_function_no & FUNC_MASK
+    if fun == FUNC_BYTE:
+        ll_dict_store_clean(d, hash, i, TYPE_BYTE)
+    elif fun == FUNC_SHORT:
+        ll_dict_store_clean(d, hash, i, TYPE_SHORT)
+    elif IS_64BIT and fun == FUNC_INT:
+        ll_dict_store_clean(d, hash, i, TYPE_INT)
+    elif fun == FUNC_LONG:
+        ll_dict_store_clean(d, hash, i, TYPE_LONG)
+    else:
+        # can't be still FUNC_MUST_REINDEX here
+        ll_assert(False, "ll_call_insert_clean_function(): invalid lookup_fun")
+        assert False
 
 def ll_call_delete_by_entry_index(d, hash, i):
     fun = d.lookup_function_no & FUNC_MASK
@@ -558,9 +547,9 @@
         ll_dict_delete_by_entry_index(d, hash, i, TYPE_INT)
     elif fun == FUNC_LONG:
         ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
-    elif fun == FUNC_NO_INDEX:
-        pass   # there is no 'indexes', so nothing to delete
     else:
+        # can't be still FUNC_MUST_REINDEX here
+        ll_assert(False, "ll_call_delete_by_entry_index(): invalid lookup_fun")
         assert False
 
 def ll_valid_from_flag(entries, i):
@@ -644,19 +633,21 @@
             try:
                 reindexed = ll_dict_grow(d)
             except:
-                ll_remove_index(d)
+                _ll_dict_rescue(d)
                 raise
-        if d.resize_counter <= 3:
+        rc = d.resize_counter - 3
+        if rc <= 0:
             try:
                 ll_dict_resize(d)
                 reindexed = True
             except:
-                ll_remove_index(d)
+                _ll_dict_rescue(d)
                 raise
+            rc = d.resize_counter - 3
+            ll_assert(rc > 0, "ll_dict_resize failed?")
         if reindexed:
             ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
-        rc = d.resize_counter - 3
-        ll_assert(rc > 0, "ll_dict_resize failed?")
+        #
         d.resize_counter = rc
         entry = d.entries[d.num_ever_used_items]
         entry.key = key
@@ -668,15 +659,22 @@
         d.num_ever_used_items += 1
         d.num_live_items += 1
 
-def _ll_dict_insertclean(d, key, value, hash):
+ at jit.dont_look_inside
+def _ll_dict_rescue(d):
+    # MemoryError situation!  The 'indexes' contains an invalid entry
+    # at this point.  But we can call ll_dict_reindex() with the
+    # following arguments, ensuring no further malloc occurs.
+    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+_ll_dict_rescue._dont_inline_ = True
+
+ at not_rpython
+def _ll_dict_insert_no_index(d, key, value):
     # never translated
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
     entry = d.entries[d.num_ever_used_items]
     entry.key = key
     entry.value = value
-    if hasattr(ENTRY, 'f_hash'):
-        entry.f_hash = hash
+    # note that f_hash is left uninitialized in prebuilt dicts
     if hasattr(ENTRY, 'f_valid'):
         entry.f_valid = True
     d.num_ever_used_items += 1
@@ -795,20 +793,7 @@
     else:
         d.entries = newitems
 
-    # Estimate the index size we would like to have now.  If the new
-    # estimate changed from before, we need to throw away the old index
-    # anyway, so simply remove it for now.  If the size is the same,
-    # then maybe it is a better idea to keep it instead of reallocating
-    # an identical index very soon.  In this case we update the index now.
-    if bool(d.indexes):
-        new_estimate = d.num_live_items * 2
-        new_size = DICT_INITSIZE
-        while new_size <= new_estimate:
-            new_size *= 2
-        if _ll_len_of_d_indexes(d) == new_size:
-            ll_dict_reindex(d, new_size)
-        else:
-            ll_remove_index(d)
+    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
 
 
 def ll_dict_delitem(d, key):
@@ -852,10 +837,10 @@
                 break
         d.num_ever_used_items = i + 1
 
-    # If the dictionary is more than 75% dead items, then clean up the
-    # deleted items and remove the index.
-    if d.num_live_items + DICT_INITSIZE < len(d.entries) / 4:
-        ll_dict_remove_deleted_items(d)
+    # If the dictionary is at least 87.5% dead items, then consider shrinking
+    # it.
+    if d.num_live_items + DICT_INITSIZE <= len(d.entries) / 8:
+        ll_dict_resize(d)
 
 def ll_dict_resize(d):
     # make a 'new_size' estimate and shrink it if there are many
@@ -873,25 +858,48 @@
     while new_size <= new_estimate:
         new_size *= 2
 
-    if bool(d.indexes) and new_size < _ll_len_of_d_indexes(d):
+    if new_size < _ll_len_of_d_indexes(d):
         ll_dict_remove_deleted_items(d)
     else:
         ll_dict_reindex(d, new_size)
 
-def ll_dict_create_index(d):
-    if d.num_live_items < DICT_INITSIZE * 2 // 3:
-        new_size = DICT_INITSIZE     # fast path
+def ll_ensure_indexes(d):
+    num = d.lookup_function_no
+    if num == FUNC_MUST_REINDEX:
+        ll_dict_create_initial_index(d)
     else:
-        # Use a more conservative estimate than _ll_dict_resize_to() here.
-        # This function is called when initially creating the index (so,
-        # for prebuilt dicts, the chance is that it will never grow);
-        # after we got a MemoryError; and when throwing the index away
-        # because of heavy shrinking of the dict.  The minimum value
-        # here is such that 'new_estimate * 2 - num_live_items * 3 > 0'.
-        new_estimate = (d.num_live_items * 3) // 2 + 1
-        new_size = DICT_INITSIZE
-        while new_size < new_estimate:
-            new_size *= 2
+        ll_assert((num & FUNC_MASK) != FUNC_MUST_REINDEX,
+                  "bad combination in lookup_function_no")
+
+def ll_dict_create_initial_index(d):
+    """Create the initial index for a dictionary.  The common case is
+    that 'd' is empty.  The uncommon case is that it is a prebuilt
+    dictionary frozen by translation, in which case we must rehash all
+    entries.  The common case must be seen by the JIT.
+    """
+    if d.num_live_items == 0:
+        ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+        d.resize_counter = DICT_INITSIZE * 2
+    else:
+        ll_dict_rehash_after_translation(d)
+
+ at jit.dont_look_inside
+def ll_dict_rehash_after_translation(d):
+    assert d.num_live_items == d.num_ever_used_items
+    assert not d.indexes
+    #
+    # recompute all hashes, if they are stored in d.entries
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    if hasattr(ENTRY, 'f_hash'):
+        for i in range(d.num_ever_used_items):
+            assert d.entries.valid(i)
+            d_entry = d.entries[i]
+            d_entry.f_hash = d.keyhash(d_entry.key)
+    #
+    # Use the smallest acceptable size for ll_dict_reindex
+    new_size = DICT_INITSIZE
+    while new_size * 2 - d.num_live_items * 3 <= 0:
+        new_size *= 2
     ll_dict_reindex(d, new_size)
 
 def ll_dict_reindex(d, new_size):
@@ -909,9 +917,7 @@
     ibound = d.num_ever_used_items
     #
     # Write four loops, moving the check for the value of 'fun' out of
-    # the loops.  A small speed-up, it also avoids the (unreachable)
-    # recursive call from here to ll_call_insert_clean_function() to
-    # ll_dict_create_index() back to here.
+    # the loops.  A small speed-up over ll_call_insert_clean_function().
     fun = d.lookup_function_no     # == lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
         while i < ibound:
@@ -1088,9 +1094,9 @@
     d.entries = _ll_empty_array(DICT)
     # Don't allocate an 'indexes' for empty dict.  It seems a typical
     # program contains tons of empty dicts, so this might be a memory win.
-    ll_remove_index(d)
     d.num_live_items = 0
     d.num_ever_used_items = 0
+    ll_no_initial_index(d)
     return d
 OrderedDictRepr.ll_newdict = staticmethod(ll_newdict)
 
@@ -1175,6 +1181,10 @@
                 # as soon as we do something like ll_dict_reindex().
                 if index == (dict.lookup_function_no >> FUNC_SHIFT):
                     dict.lookup_function_no += (1 << FUNC_SHIFT)
+                # note that we can't have modified a FUNC_MUST_REINDEX
+                # dict here because such dicts have no invalid entries
+                ll_assert((dict.lookup_function_no & FUNC_MASK) !=
+                      FUNC_MUST_REINDEX, "bad combination in _ll_dictnext")
             index = nextindex
         # clear the reference to the dict and prevent restarts
         iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
@@ -1220,6 +1230,8 @@
         return dict.entries[index].value
 
 def ll_dict_copy(dict):
+    ll_ensure_indexes(dict)
+
     DICT = lltype.typeOf(dict).TO
     newdict = DICT.allocate()
     newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
@@ -1244,7 +1256,7 @@
             d_entry.f_hash = entry.f_hash
         i += 1
 
-    ll_remove_index(newdict)
+    ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
     return newdict
 ll_dict_copy.oopspec = 'odict.copy(dict)'
 
@@ -1254,15 +1266,17 @@
     DICT = lltype.typeOf(d).TO
     old_entries = d.entries
     d.entries = _ll_empty_array(DICT)
-    ll_remove_index(d)
     d.num_live_items = 0
     d.num_ever_used_items = 0
+    ll_no_initial_index(d)
+    d.resize_counter = 0
     # old_entries.delete() XXX
 ll_dict_clear.oopspec = 'odict.clear(d)'
 
 def ll_dict_update(dic1, dic2):
     if dic1 == dic2:
         return
+    ll_ensure_indexes(dic2)
     ll_prepare_dict_update(dic1, dic2.num_live_items)
     i = 0
     while i < dic2.num_ever_used_items:
@@ -1289,6 +1303,7 @@
     # the case where dict.update() actually has a lot of collisions.
     # If num_extra is much greater than d.num_live_items the conditional_call
     # will trigger anyway, which is really the goal.
+    ll_ensure_indexes(d)
     x = num_extra - d.num_live_items
     jit.conditional_call(d.resize_counter <= x * 3,
                          _ll_dict_resize_to, d, num_extra)
@@ -1348,6 +1363,7 @@
     if dic.num_live_items == 0:
         raise KeyError
 
+    ll_ensure_indexes(dic)
     entries = dic.entries
 
     # find the last entry.  It's unclear if the loop below is still
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -415,9 +415,10 @@
             self.removed_keys.append(key)
 
     def removeindex(self):
-        # remove the index, as done e.g. during translation for prebuilt
-        # dicts
-        rodct.ll_remove_index(self.l_dict)
+        # remove the index, as done during translation for prebuilt dicts
+        # (but cannot be done if we already removed a key)
+        if not self.removed_keys:
+            rodct.ll_no_initial_index(self.l_dict)
 
     def fullcheck(self):
         # overridden to also check key order
@@ -444,7 +445,7 @@
                 num_lives += 1
         assert num_lives == d.num_live_items
         fun = d.lookup_function_no & rordereddict.FUNC_MASK
-        if fun == rordereddict.FUNC_NO_INDEX:
+        if fun == rordereddict.FUNC_MUST_REINDEX:
             assert not d.indexes
         else:
             assert d.indexes
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -9,7 +9,7 @@
 
 from rpython.conftest import option
 from rpython.rlib import rgc
-from rpython.rlib.objectmodel import keepalive_until_here, compute_hash, compute_identity_hash
+from rpython.rlib.objectmodel import keepalive_until_here, compute_hash, compute_identity_hash, r_dict
 from rpython.rlib.rstring import StringBuilder
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -562,6 +562,41 @@
         res = self.run('prebuilt_weakref')
         assert res == self.run_orig('prebuilt_weakref')
 
+    def define_prebuilt_dicts_of_all_sizes(self):
+        class X:
+            pass
+        dlist = []
+        keyslist = []
+        def keq(x, y):
+            return x is y
+        def khash(x):
+            return compute_hash(x)
+        for size in ([random.randrange(0, 260) for i in range(10)] +
+                     [random.randrange(260, 60000)]):
+            print 'PREBUILT DICTIONARY OF SIZE', size
+            keys = [X() for j in range(size)]
+            d = r_dict(keq, khash)
+            for j in range(size):
+                d[keys[j]] = j
+            dlist.append(d)
+            keyslist.append(keys)
+
+        def fn():
+            for n in range(len(dlist)):
+                d = dlist[n]
+                keys = keyslist[n]
+                assert len(d) == len(keys)
+                i = 0
+                while i < len(keys):
+                    assert d[keys[i]] == i
+                    i += 1
+            return 42
+        return fn
+
+    def test_prebuilt_dicts_of_all_sizes(self):
+        res = self.run('prebuilt_dicts_of_all_sizes')
+        assert res == 42
+
     def define_framework_malloc_raw(cls):
         A = lltype.Struct('A', ('value', lltype.Signed))
 

From pypy.commits at gmail.com  Fri Jan 27 10:45:39 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jan 2017 07:45:39 -0800 (PST)
Subject: [pypy-commit] pypy default: Configure structs in the order in which
 they are typedef'd
Message-ID: <588b6b23.545a1c0a.b4ae8.764a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89798:299d09fc6636
Date: 2017-01-27 15:44 +0000
http://bitbucket.org/pypy/pypy/changeset/299d09fc6636/

Log:	Configure structs in the order in which they are typedef'd

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+from itertools import izip
 from . import cmodel as model
 from .commontypes import COMMON_TYPES, resolve_common_type
 from .error import FFIError, CDefError
@@ -103,7 +104,7 @@
 class Parser(object):
 
     def __init__(self):
-        self._declarations = {}
+        self._declarations = OrderedDict()
         self._included_declarations = set()
         self._anonymous_counter = 0
         self._structnode2type = weakref.WeakKeyDictionary()
@@ -796,7 +797,7 @@
             return
         eci = self.build_eci()
         result = rffi_platform.configure_entries(list(self._config_entries), eci)
-        for entry, TYPE in zip(self._config_entries, result):
+        for entry, TYPE in izip(self._config_entries, result):
             # hack: prevent the source from being pasted into common_header.h
             del TYPE._hints['eci']
             self._config_entries[entry].become(TYPE)
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -142,6 +142,20 @@
     assert isinstance(Object, lltype.Struct)
     hash(Object)
 
+def test_nested_struct():
+    cdef = """
+    typedef struct {
+        int x;
+    } foo;
+    typedef struct {
+        foo y;
+    } bar;
+    """
+    cts = parse_source(cdef)
+    bar = cts.gettype('bar')
+    assert isinstance(bar, lltype.Struct)
+    hash(bar)  # bar is hashable
+
 def test_const():
     cdef = """
     typedef struct {
diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -245,7 +245,8 @@
         resultinfo[entry] = info
 
     result = ConfigResult(eci, resultinfo)
-    return [result.get_entry_result(entry) for entry in entries]
+    for entry in entries:
+        yield result.get_entry_result(entry)
 
 # ____________________________________________________________
 

From pypy.commits at gmail.com  Fri Jan 27 10:56:09 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jan 2017 07:56:09 -0800 (PST)
Subject: [pypy-commit] pypy default: Move PyHeapTypeObject declaration to
 its rightful place in cpyext_object.h
Message-ID: <588b6d99.4a6b1c0a.3b0f2.7582@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89799:f0cf2a50103c
Date: 2017-01-27 15:55 +0000
http://bitbucket.org/pypy/pypy/changeset/f0cf2a50103c/

Log:	Move PyHeapTypeObject declaration to its rightful place in
	cpyext_object.h

diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -305,3 +305,11 @@
 
 } PyTypeObject;
 
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
diff --git a/pypy/module/cpyext/parse/cpyext_typeobject.h b/pypy/module/cpyext/parse/cpyext_typeobject.h
deleted file mode 100644
--- a/pypy/module/cpyext/parse/cpyext_typeobject.h
+++ /dev/null
@@ -1,8 +0,0 @@
-typedef struct {
-    PyTypeObject ht_type;
-    PyNumberMethods as_number;
-    PyMappingMethods as_mapping;
-    PySequenceMethods as_sequence;
-    PyBufferProcs as_buffer;
-    PyObject *ht_name, *ht_slots;
-} PyHeapTypeObject;
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -41,7 +41,6 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
-cts.parse_header(parse_dir / 'cpyext_typeobject.h')
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
 

From pypy.commits at gmail.com  Fri Jan 27 11:07:29 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jan 2017 08:07:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <588b7041.04abdf0a.b4134.eeab@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89800:9b9d04aef631
Date: 2017-01-27 16:06 +0000
http://bitbucket.org/pypy/pypy/changeset/9b9d04aef631/

Log:	hg merge default

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -80,7 +80,8 @@
 
     def errorstr(self, space, use_repr=False):
         "The exception class and value, as a string."
-        self.normalize_exception(space)
+        if not use_repr:    # see write_unraisable()
+            self.normalize_exception(space)
         w_value = self.get_w_value(space)
         if space is None:
             # this part NOT_RPYTHON
@@ -276,6 +277,11 @@
                     first_line = 'Exception ignored in: %s%s\n' % (
                         where, objrepr)
             else:
+                # Note that like CPython, we don't normalize the
+                # exception here.  So from `'foo'.index('bar')` you get
+                # "Exception ValueError: 'substring not found' in x ignored"
+                # but from `raise ValueError('foo')` you get
+                # "Exception ValueError: ValueError('foo',) in x ignored"
                 first_line = ''
             space.appexec([space.wrap(first_line),
                            space.wrap(extra_line),
diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py
--- a/pypy/interpreter/test/test_appinterp.py
+++ b/pypy/interpreter/test/test_appinterp.py
@@ -23,7 +23,9 @@
     (): 
         y y 
     """)
-    assert str(excinfo.value.errorstr(space)).find('y y') != -1 
+    # NOTE: the following test only works because excinfo.value is not
+    # normalized so far
+    assert str(excinfo.value.get_w_value(space)).find('y y') != -1 
 
 def test_simple_applevel(space):
     app = appdef("""app(x,y): 
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+from itertools import izip
 from . import cmodel as model
 from .commontypes import COMMON_TYPES, resolve_common_type
 from .error import FFIError, CDefError
@@ -103,7 +104,7 @@
 class Parser(object):
 
     def __init__(self):
-        self._declarations = {}
+        self._declarations = OrderedDict()
         self._included_declarations = set()
         self._anonymous_counter = 0
         self._structnode2type = weakref.WeakKeyDictionary()
@@ -699,8 +700,7 @@
         self.headers = headers if headers is not None else ['sys/types.h']
         self.parsed_headers = []
         self.sources = []
-        self._Config = type('Config', (object,), {})
-        self._TYPES = {}
+        self._config_entries = OrderedDict()
         self.includes = []
         self.struct_typedefs = {}
         self._handled = set()
@@ -758,10 +758,8 @@
 
     def realize_struct(self, struct):
         type_name = struct.get_type_name()
-        configname = type_name.replace(' ', '__')
-        setattr(self._Config, configname,
-            rffi_platform.Struct(type_name, struct.fields))
-        self._TYPES[configname] = struct.TYPE
+        entry = rffi_platform.Struct(type_name, struct.fields)
+        self._config_entries[entry] = struct.TYPE
         return struct.TYPE
 
     def build_eci(self):
@@ -795,13 +793,15 @@
             elif name.startswith('macro '):
                 name = name[6:]
                 self.add_macro(name, obj)
-        self._Config._compilation_info_ = self.build_eci()
-        for name, TYPE in rffi_platform.configure(self._Config).iteritems():
+        if not self._config_entries:
+            return
+        eci = self.build_eci()
+        result = rffi_platform.configure_entries(list(self._config_entries), eci)
+        for entry, TYPE in izip(self._config_entries, result):
             # hack: prevent the source from being pasted into common_header.h
             del TYPE._hints['eci']
-            if name in self._TYPES:
-                self._TYPES[name].become(TYPE)
-                del self._TYPES[name]
+            self._config_entries[entry].become(TYPE)
+        self._config_entries.clear()
 
     def convert_type(self, obj, quals=0):
         if isinstance(obj, model.DefinedType):
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -280,3 +280,12 @@
 
     destructor tp_finalize;
 } PyTypeObject;
+
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
diff --git a/pypy/module/cpyext/parse/cpyext_typeobject.h b/pypy/module/cpyext/parse/cpyext_typeobject.h
deleted file mode 100644
--- a/pypy/module/cpyext/parse/cpyext_typeobject.h
+++ /dev/null
@@ -1,9 +0,0 @@
-typedef struct {
-    PyTypeObject ht_type;
-    PyNumberMethods as_number;
-    PyMappingMethods as_mapping;
-    PySequenceMethods as_sequence;
-    PyBufferProcs as_buffer;
-    PyObject *ht_name, *ht_slots;
-} PyHeapTypeObject;
-
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -142,6 +142,20 @@
     assert isinstance(Object, lltype.Struct)
     hash(Object)
 
+def test_nested_struct():
+    cdef = """
+    typedef struct {
+        int x;
+    } foo;
+    typedef struct {
+        foo y;
+    } bar;
+    """
+    cts = parse_source(cdef)
+    bar = cts.gettype('bar')
+    assert isinstance(bar, lltype.Struct)
+    hash(bar)  # bar is hashable
+
 def test_const():
     cdef = """
     typedef struct {
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -40,7 +40,6 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
-cts.parse_header(parse_dir / 'cpyext_typeobject.h')
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
 
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -21,6 +21,9 @@
 # that as easily because many details may rely on getting the same hash
 # value before and after translation.  We can, however, pick a random
 # seed once per translation, which should already be quite good.
+#
+# XXX no, it is not: e.g. all Ubuntu installations of the same Ubuntu
+# would get the same seed.  That's not good enough.
 
 @not_rpython
 def select_random_seed():
diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -131,38 +131,32 @@
 # General interface
 
 class ConfigResult:
-    def __init__(self, CConfig, info, entries):
-        self.CConfig = CConfig
+    def __init__(self, eci, info):
+        self.eci = eci
+        self.info = info
         self.result = {}
-        self.info = info
-        self.entries = entries
 
     def get_entry_result(self, entry):
         try:
             return self.result[entry]
         except KeyError:
             pass
-        name = self.entries[entry]
-        info = self.info[name]
+        info = self.info[entry]
         self.result[entry] = entry.build_result(info, self)
         return self.result[entry]
 
-    def get_result(self):
-        return dict([(name, self.result[entry])
-                     for entry, name in self.entries.iteritems()])
 
 class _CWriter(object):
     """ A simple class which aggregates config parts
     """
-    def __init__(self, CConfig):
+    def __init__(self, eci):
         self.path = uniquefilepath()
         self.f = self.path.open("w")
-        self.config = CConfig
+        self.eci = eci
 
     def write_header(self):
         f = self.f
-        CConfig = self.config
-        CConfig._compilation_info_.write_c_header(f)
+        self.eci.write_c_header(f)
         print >> f, C_HEADER
         print >> f
 
@@ -194,8 +188,7 @@
         self.start_main()
         self.f.write(question + "\n")
         self.close()
-        eci = self.config._compilation_info_
-        try_compile_cache([self.path], eci)
+        try_compile_cache([self.path], self.eci)
 
 def configure(CConfig, ignore_errors=False):
     """Examine the local system by running the C compiler.
@@ -208,50 +201,53 @@
         assert not hasattr(CConfig, attr), \
             "Found legacy attribute %s on CConfig" % attr
 
-    entries = []
+    eci = CConfig._compilation_info_
+    entries = {}
     for key in dir(CConfig):
         value = getattr(CConfig, key)
         if isinstance(value, CConfigEntry):
-            entries.append((key, value))
+            entries[key] = value
 
+    res = {}
     if entries:   # can be empty if there are only CConfigSingleEntries
-        writer = _CWriter(CConfig)
-        writer.write_header()
-        for key, entry in entries:
-            writer.write_entry(key, entry)
-
-        writer.start_main()
-        for key, entry in entries:
-            writer.write_entry_main(key)
-        writer.close()
-
-        eci = CConfig._compilation_info_
-        infolist = list(run_example_code(writer.path, eci,
-                                         ignore_errors=ignore_errors))
-        assert len(infolist) == len(entries)
-
-        resultinfo = {}
-        resultentries = {}
-        for info, (key, entry) in zip(infolist, entries):
-            resultinfo[key] = info
-            resultentries[entry] = key
-
-        result = ConfigResult(CConfig, resultinfo, resultentries)
-        for name, entry in entries:
-            result.get_entry_result(entry)
-        res = result.get_result()
-    else:
-        res = {}
+        results = configure_entries(
+            entries.values(), eci, ignore_errors=ignore_errors)
+        for name, result in zip(entries, results):
+            res[name] = result
 
     for key in dir(CConfig):
         value = getattr(CConfig, key)
         if isinstance(value, CConfigSingleEntry):
-            writer = _CWriter(CConfig)
+            writer = _CWriter(eci)
             writer.write_header()
             res[key] = value.question(writer.ask_gcc)
 
     return res
 
+
+def configure_entries(entries, eci, ignore_errors=False):
+    writer = _CWriter(eci)
+    writer.write_header()
+    for i, entry in enumerate(entries):
+        writer.write_entry(str(i), entry)
+
+    writer.start_main()
+    for i, entry in enumerate(entries):
+        writer.write_entry_main(str(i))
+    writer.close()
+
+    infolist = list(run_example_code(
+        writer.path, eci, ignore_errors=ignore_errors))
+    assert len(infolist) == len(entries)
+
+    resultinfo = {}
+    for info, entry in zip(infolist, entries):
+        resultinfo[entry] = info
+
+    result = ConfigResult(eci, resultinfo)
+    for entry in entries:
+        yield result.get_entry_result(entry)
+
 # ____________________________________________________________
 
 
@@ -344,7 +340,7 @@
         allfields = tuple(['c_' + name for name, _ in fields])
         padfields = tuple(padfields)
         name = self.name
-        eci = config_result.CConfig._compilation_info_
+        eci = config_result.eci
         padding_drop = PaddingDrop(name, allfields, padfields, eci)
         hints = {'align': info['align'],
                  'size': info['size'],

From pypy.commits at gmail.com  Fri Jan 27 11:19:02 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jan 2017 08:19:02 -0800 (PST)
Subject: [pypy-commit] pypy default: fix
Message-ID: <588b72f6.977d1c0a.e8896.7fc6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89801:1ae376efcfa5
Date: 2017-01-27 16:18 +0000
http://bitbucket.org/pypy/pypy/changeset/1ae376efcfa5/

Log:	fix

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -113,8 +113,6 @@
 #define PyBUF_SHADOW 0x400
 /* end Py3k buffer interface */
 
-#include <cpyext_typeobject.h>
-
 #define PyObject_Bytes PyObject_Str
 
 /* Flag bits for printing: */

From pypy.commits at gmail.com  Fri Jan 27 11:20:55 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jan 2017 08:20:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <588b7367.92d41c0a.1e98.80e4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89802:e99cc786cb4b
Date: 2017-01-27 16:20 +0000
http://bitbucket.org/pypy/pypy/changeset/e99cc786cb4b/

Log:	hg merge default

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -113,8 +113,6 @@
 #define PyBUF_SHADOW 0x400
 /* end Py3k buffer interface */
 
-#include <cpyext_typeobject.h>
-
 /* Flag bits for printing: */
 #define Py_PRINT_RAW    1       /* No string quotes etc. */
 

From pypy.commits at gmail.com  Fri Jan 27 12:20:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 27 Jan 2017 09:20:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: py3.5 fix: needs a bytes in the middle of
 this wrapped tuple
Message-ID: <588b8175.c80e1c0a.952b2.9666@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89803:54bb0a810cd2
Date: 2017-01-27 18:20 +0100
http://bitbucket.org/pypy/pypy/changeset/54bb0a810cd2/

Log:	py3.5 fix: needs a bytes in the middle of this wrapped tuple

diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -104,7 +104,12 @@
         space.newtuple([space.wrap(6), space.wrap(7)]))
     assert operr.errorstr(space) == "ValueError: (6, 7)"
     operr = OperationError(space.w_UnicodeDecodeError,
-        space.wrap(('unicodeescape', r'\\x', 0, 2, r'truncated \\xXX escape')))
+        space.newtuple([
+            space.wrap('unicodeescape'),
+            space.newbytes(r'\\x'),
+            space.newint(0),
+            space.newint(2),
+            space.wrap(r'truncated \\xXX escape')]))
     assert operr.errorstr(space) == (
         "UnicodeDecodeError: 'unicodeescape' codec can't decode "
         "bytes in position 0-1: truncated \\\\xXX escape")

From pypy.commits at gmail.com  Fri Jan 27 13:32:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 27 Jan 2017 10:32:13 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: avoid another obscure corner case
 by further reducing the diff to default
Message-ID: <588b922d.cfc41c0a.34046.bb02@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89804:6d5589e7f9fa
Date: 2017-01-27 19:31 +0100
http://bitbucket.org/pypy/pypy/changeset/6d5589e7f9fa/

Log:	avoid another obscure corner case by further reducing the diff to
	default

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -1266,10 +1266,14 @@
     DICT = lltype.typeOf(d).TO
     old_entries = d.entries
     d.entries = _ll_empty_array(DICT)
+    # note: we can't remove the index here, because it is possible that
+    # crazy Python code calls d.clear() from the method __eq__() called
+    # from ll_dict_lookup(d).  Instead, stick to the rule that once a
+    # dictionary has got an index, it will always have one.
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
     d.num_live_items = 0
     d.num_ever_used_items = 0
-    ll_no_initial_index(d)
-    d.resize_counter = 0
+    d.resize_counter = DICT_INITSIZE * 2
     # old_entries.delete() XXX
 ll_dict_clear.oopspec = 'odict.clear(d)'
 

From pypy.commits at gmail.com  Fri Jan 27 14:44:39 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jan 2017 11:44:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Rename _PyUnicode_AsString to
 PyUnicode_AsUTF8 (changed in 3.3)
Message-ID: <588ba327.ce9adf0a.bd348.548c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89805:2e61542390d8
Date: 2017-01-27 19:43 +0000
http://bitbucket.org/pypy/pypy/changeset/2e61542390d8/

Log:	Rename _PyUnicode_AsString to PyUnicode_AsUTF8 (changed in 3.3)

diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -10,6 +10,8 @@
 PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char *format, va_list vargs);
 PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char *format, ...);
 
+#define _PyUnicode_AsString PyUnicode_AsUTF8
+
 PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString(PyObject *unicode, Py_ssize_t *size);
 
 Py_LOCAL_INLINE(size_t) Py_UNICODE_strlen(const Py_UNICODE *u)
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -28,7 +28,7 @@
     """Create a new module object, given the definition in module, assuming the
     API version module_api_version.  If that version does not match the version
     of the running interpreter, a RuntimeWarning is emitted.
-    
+
     Most uses of this function should be using PyModule_Create()
     instead; only use this if you are sure you need it."""
 
@@ -123,5 +123,5 @@
     # and returns a "char *" inside this PyStringObject.
     if not isinstance(w_mod, Module):
         raise oefmt(space.w_SystemError, "PyModule_GetName(): not a module")
-    from pypy.module.cpyext.unicodeobject import _PyUnicode_AsString
-    return _PyUnicode_AsString(space, as_pyobj(space, w_mod.w_name))
+    from pypy.module.cpyext.unicodeobject import PyUnicode_AsUTF8
+    return PyUnicode_AsUTF8(space, as_pyobj(space, w_mod.w_name))
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -645,9 +645,9 @@
         w_typename = space.getattr(w_type, space.wrap('__name__'))
         heaptype = cts.cast('PyHeapTypeObject*', pto)
         heaptype.c_ht_name = make_ref(space, w_typename)
-        from pypy.module.cpyext.unicodeobject import _PyUnicode_AsString
+        from pypy.module.cpyext.unicodeobject import PyUnicode_AsUTF8
         pto.c_tp_name = cts.cast('const char *',
-            _PyUnicode_AsString(space, heaptype.c_ht_name))
+            PyUnicode_AsUTF8(space, heaptype.c_ht_name))
     else:
         pto.c_tp_name = cts.cast('const char*', rffi.str2charp(w_type.name))
     # uninitialized fields:
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
-    CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
+    CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api, api_decl,
     bootstrap_function, CONST_STRING,
     CONST_WSTRING, Py_CLEANUP_SUPPORTED, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
@@ -234,8 +234,8 @@
         raise oefmt(space.w_TypeError, "expected unicode object")
     return PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
 
- at cpython_api([PyObject], rffi.CCHARP)
-def _PyUnicode_AsString(space, ref):
+ at api_decl("char * PyUnicode_AsUTF8(PyObject *unicode)", cts)
+def PyUnicode_AsUTF8(space, ref):
     ref_unicode = rffi.cast(PyUnicodeObject, ref)
     if not ref_unicode.c_utf8buffer:
         # Copy unicode buffer

From pypy.commits at gmail.com  Fri Jan 27 16:03:30 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 27 Jan 2017 13:03:30 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: comment
Message-ID: <588bb5a2.9da0df0a.3bf17.69fb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89806:8d1318904037
Date: 2017-01-27 20:03 +0100
http://bitbucket.org/pypy/pypy/changeset/8d1318904037/

Log:	comment

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -1280,7 +1280,7 @@
 def ll_dict_update(dic1, dic2):
     if dic1 == dic2:
         return
-    ll_ensure_indexes(dic2)
+    ll_ensure_indexes(dic2)    # needed for entries.hash() below
     ll_prepare_dict_update(dic1, dic2.num_live_items)
     i = 0
     while i < dic2.num_ever_used_items:

From pypy.commits at gmail.com  Fri Jan 27 16:03:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 27 Jan 2017 13:03:32 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: (untested) remove some code that
 implements keeping the hashes across translation
Message-ID: <588bb5a4.b6a9df0a.3bf85.69f4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89807:52687af2d35a
Date: 2017-01-27 22:00 +0100
http://bitbucket.org/pypy/pypy/changeset/52687af2d35a/

Log:	(untested) remove some code that implements keeping the hashes
	across translation

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -117,9 +117,7 @@
 
 # The following flag is set on nursery objects of which we asked the id
 # or the identityhash.  It means that a space of the size of the object
-# has already been allocated in the nonmovable part.  The same flag is
-# abused to mark prebuilt objects whose hash has been taken during
-# translation and is statically recorded.
+# has already been allocated in the nonmovable part.
 GCFLAG_HAS_SHADOW   = first_gcflag << 3
 
 # The following flag is set temporarily on some objects during a major
@@ -208,10 +206,6 @@
     # by GCFLAG_xxx above.
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
-    # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
-    #     then they are one word longer, the extra word storing the hash.
-
 
     # During a minor collection, the objects in the nursery that are
     # moved outside are changed in-place: their header is replaced with
@@ -2640,40 +2634,22 @@
         return shadow
     _find_shadow._dont_inline_ = True
 
-    @specialize.arg(2)
-    def id_or_identityhash(self, gcobj, is_hash):
+    def id_or_identityhash(self, gcobj):
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
         obj = llmemory.cast_ptr_to_adr(gcobj)
-        #
         if self.is_valid_gc_object(obj):
             if self.is_in_nursery(obj):
                 obj = self._find_shadow(obj)
-            elif is_hash:
-                if self.header(obj).tid & GCFLAG_HAS_SHADOW:
-                    #
-                    # For identityhash(), we need a special case for some
-                    # prebuilt objects: their hash must be the same before
-                    # and after translation.  It is stored as an extra word
-                    # after the object.  But we cannot use it for id()
-                    # because the stored value might clash with a real one.
-                    size = self.get_size(obj)
-                    i = (obj + size).signed[0]
-                    # Important: the returned value is not mangle_hash()ed!
-                    return i
-        #
-        i = llmemory.cast_adr_to_int(obj)
-        if is_hash:
-            i = mangle_hash(i)
-        return i
+        return llmemory.cast_adr_to_int(obj)
     id_or_identityhash._always_inline_ = True
 
     def id(self, gcobj):
-        return self.id_or_identityhash(gcobj, False)
+        return self.id_or_identityhash(gcobj)
 
     def identityhash(self, gcobj):
-        return self.id_or_identityhash(gcobj, True)
+        return mangle_hash(self.id_or_identityhash(gcobj))
 
     # ----------
     # Finalizers
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -104,9 +104,7 @@
 
 # The following flag is set on nursery objects of which we asked the id
 # or the identityhash.  It means that a space of the size of the object
-# has already been allocated in the nonmovable part.  The same flag is
-# abused to mark prebuilt objects whose hash has been taken during
-# translation and is statically recorded.
+# has already been allocated in the nonmovable part.
 GCFLAG_HAS_SHADOW   = first_gcflag << 3
 
 # The following flag is set temporarily on some objects during a major
@@ -149,9 +147,6 @@
     # by GCFLAG_xxx above.
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
-    # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
-    #     then they are one word longer, the extra word storing the hash.
 
     _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True})
 
@@ -1868,40 +1863,22 @@
         return shadow
     _find_shadow._dont_inline_ = True
 
-    @specialize.arg(2)
-    def id_or_identityhash(self, gcobj, is_hash):
+    def id_or_identityhash(self, gcobj):
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
         obj = llmemory.cast_ptr_to_adr(gcobj)
-        #
         if self.is_valid_gc_object(obj):
             if self.is_in_nursery(obj):
                 obj = self._find_shadow(obj)
-            elif is_hash:
-                if self.header(obj).tid & GCFLAG_HAS_SHADOW:
-                    #
-                    # For identityhash(), we need a special case for some
-                    # prebuilt objects: their hash must be the same before
-                    # and after translation.  It is stored as an extra word
-                    # after the object.  But we cannot use it for id()
-                    # because the stored value might clash with a real one.
-                    size = self.get_size(obj)
-                    i = (obj + size).signed[0]
-                    # Important: the returned value is not mangle_hash()ed!
-                    return i
-        #
-        i = llmemory.cast_adr_to_int(obj)
-        if is_hash:
-            i = mangle_hash(i)
-        return i
+        return llmemory.cast_adr_to_int(obj)
     id_or_identityhash._always_inline_ = True
 
     def id(self, gcobj):
-        return self.id_or_identityhash(gcobj, False)
+        return self.id_or_identityhash(gcobj)
 
     def identityhash(self, gcobj):
-        return self.id_or_identityhash(gcobj, True)
+        return mangle_hash(self.id_or_identityhash(gcobj))
 
     # ----------
     # Finalizers
diff --git a/rpython/memory/gc/semispace.py b/rpython/memory/gc/semispace.py
--- a/rpython/memory/gc/semispace.py
+++ b/rpython/memory/gc/semispace.py
@@ -48,9 +48,6 @@
 
     HDR = lltype.Struct('header', ('tid', lltype.Signed))   # XXX or rffi.INT?
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', _GCFLAG_HASH_BASE * 0x2
-    # ^^^ prebuilt objects either have GC_HASH_TAKEN_ADDR or they
-    #     have GC_HASH_HASFIELD (and then they are one word longer).
     FORWARDSTUB = lltype.GcStruct('forwarding_stub',
                                   ('forw', llmemory.Address))
     FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -194,7 +194,6 @@
 
     def gcheader_initdata(self, obj):
         hdr = lltype.malloc(self.HDR, immortal=True)
-        #hdr.hash = lltype.identityhash_nocache(obj._as_ptr())
         return hdr._obj
 
 
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -610,18 +610,8 @@
     def special_funcptr_for_type(self, TYPE):
         return self.layoutbuilder.special_funcptr_for_type(TYPE)
 
-    def gc_header_for(self, obj, needs_hash=False):
+    def gc_header_for(self, obj):
         hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj)
-        withhash, flag = self.gcdata.gc.withhash_flag_is_in_field
-        x = getattr(hdr, withhash)
-        TYPE = lltype.typeOf(x)
-        x = lltype.cast_primitive(lltype.Signed, x)
-        if needs_hash:
-            x |= flag       # set the flag in the header
-        else:
-            x &= ~flag      # clear the flag in the header
-        x = lltype.cast_primitive(TYPE, x)
-        setattr(hdr, withhash, x)
         return hdr
 
     def get_hash_offset(self, T):
@@ -1514,22 +1504,9 @@
 
     def gcheader_initdata(self, obj):
         o = lltype.top_container(obj)
-        needs_hash = self.get_prebuilt_hash(o) is not None
-        hdr = self.gc_header_for(o, needs_hash)
+        hdr = self.gc_header_for(o)
         return hdr._obj
 
-    def DISABLED_get_prebuilt_hash(self, obj):
-        # for prebuilt objects that need to have their hash stored and
-        # restored.  Note that only structures that are StructNodes all
-        # the way have their hash stored (and not e.g. structs with var-
-        # sized arrays at the end).  'obj' must be the top_container.
-        TYPE = lltype.typeOf(obj)
-        if not isinstance(TYPE, lltype.GcStruct):
-            return None
-        if TYPE._is_varsize():
-            return None
-        return getattr(obj, '_hash_cache_', None)
-
     def get_finalizer_queue_index(self, hop):
         fq_tag = hop.spaceop.args[0].value
         assert 'FinalizerQueue TAG' in fq_tag.expr
diff --git a/rpython/memory/gctransform/refcounting.py b/rpython/memory/gctransform/refcounting.py
--- a/rpython/memory/gctransform/refcounting.py
+++ b/rpython/memory/gctransform/refcounting.py
@@ -18,8 +18,7 @@
 class RefcountingGCTransformer(GCTransformer):
     malloc_zero_filled = True
 
-    HDR = lltype.Struct("header", ("refcount", lltype.Signed),
-                                  ("hash", lltype.Signed))
+    HDR = lltype.Struct("header", ("refcount", lltype.Signed))
 
     def __init__(self, translator):
         super(RefcountingGCTransformer, self).__init__(translator, inline=True)
@@ -78,9 +77,7 @@
 
         def ll_identityhash(addr):
             obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
-            h = obj.hash
-            if h == 0:
-                obj.hash = h = llmemory.cast_adr_to_int(addr)
+            h = llmemory.cast_adr_to_int(addr)
             return h
 
         if self.translator:
@@ -178,7 +175,6 @@
             if not self.gcheaderbuilder.get_header(p):
                 hdr = self.gcheaderbuilder.new_header(p)
                 hdr.refcount = sys.maxint // 2
-                hdr.hash = lltype.identityhash_nocache(p)
 
     def static_deallocation_funcptr_for_type(self, TYPE):
         if TYPE in self.static_deallocator_funcptrs:
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -374,9 +374,6 @@
         return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP),
                                         lltype.nullptr(ARRAY_TYPEID_MAP)))
 
-    def get_prebuilt_hash(self, obj):
-        return None
-
 
 class MinimalGCTransformer(BaseGCTransformer):
     def __init__(self, parenttransformer):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -484,17 +484,11 @@
     """RPython equivalent of object.__hash__(x).  This returns the
     so-called 'identity hash', which is the non-overridable default hash
     of Python.  Can be called for any RPython-level object that turns
-    into a GC object, but not NULL.  The value is not guaranteed to be the
-    same before and after translation, except for RPython instances on the
-    lltypesystem.
+    into a GC object, but not NULL.  The value will be different before
+    and after translation.
     """
     assert x is not None
-    result = object.__hash__(x)
-    try:
-        x.__dict__['__precomputed_identity_hash'] = result
-    except (TypeError, AttributeError):
-        pass
-    return result
+    return object.__hash__(x)
 
 def compute_unique_id(x):
     """RPython equivalent of id(x).  The 'x' must be an RPython-level
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1380,20 +1380,11 @@
             return callb(*args)
         raise TypeError("%r instance is not a function" % (self._T,))
 
-    def _identityhash(self, cache=True):
+    def _identityhash(self):
         p = normalizeptr(self)
-        try:
-            return p._obj._hash_cache_
-        except AttributeError:
-            assert self._T._gckind == 'gc'
-            assert self      # not for NULL
-            result = hash(p._obj)
-            if cache:
-                try:
-                    p._obj._hash_cache_ = result
-                except AttributeError:
-                    pass
-            return result
+        assert self._T._gckind == 'gc'
+        assert self      # not for NULL
+        return hash(p._obj)
 
 class _ptr(_abstract_ptr):
     __slots__ = ('_TYPE',
@@ -1759,7 +1750,7 @@
 class _struct(_parentable):
     _kind = "structure"
 
-    __slots__ = ('_hash_cache_', '_compilation_info')
+    __slots__ = ('_compilation_info',)
 
     def __new__(self, TYPE, n=None, initialization=None, parent=None,
                 parentindex=None):
@@ -2442,24 +2433,6 @@
     return SomeInteger()
 
 
-def identityhash_nocache(p):
-    """Version of identityhash() to use from backends that don't care about
-    caching."""
-    assert p
-    return p._identityhash(cache=False)
-
-def init_identity_hash(p, value):
-    """For a prebuilt object p, initialize its hash value to 'value'."""
-    assert isinstance(typeOf(p), Ptr)
-    p = normalizeptr(p)
-    if not p:
-        raise ValueError("cannot change hash(NULL)!")
-    if hasattr(p._obj, '_hash_cache_'):
-        raise ValueError("the hash of %r was already computed" % (p,))
-    if typeOf(p).TO._is_varsize():
-        raise ValueError("init_identity_hash(): not for varsized types")
-    p._obj._hash_cache_ = intmask(value)
-
 def isCompatibleType(TYPE1, TYPE2):
     return TYPE1._is_compatible(TYPE2)
 
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -785,7 +785,6 @@
     def initialize_prebuilt_instance(self, value, classdef, result):
         # must fill in the hash cache before the other ones
         # (see test_circular_hash_initialization)
-        self.initialize_prebuilt_hash(value, result)
         self._initialize_data_flattenrec(self.initialize_prebuilt_data,
                                          value, classdef, result)
 
@@ -943,11 +942,6 @@
             rclass = getclassrepr(self.rtyper, classdef)
             result.typeptr = rclass.getvtable()
 
-    def initialize_prebuilt_hash(self, value, result):
-        llattrvalue = getattr(value, '__precomputed_identity_hash', None)
-        if llattrvalue is not None:
-            lltype.init_identity_hash(result, llattrvalue)
-
     def getfieldrepr(self, attr):
         """Return the repr used for the given attribute."""
         if attr in self.fields:
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -623,52 +623,6 @@
 
 assert not USESLOTS or '__dict__' not in dir(StructNode)
 
-class GcStructNodeWithHash(StructNode):
-    # for the outermost level of nested structures, if it has a _hash_cache_.
-    nodekind = 'struct'
-    if USESLOTS:
-        __slots__ = ()
-
-    def get_hash_typename(self):
-        return 'struct _hashT_%s @' % self.name
-
-    def forward_declaration(self):
-        T = self.getTYPE()
-        assert self.typename == self.implementationtypename  # no array part
-        hash_typename = self.get_hash_typename()
-        hash_offset = self.db.gctransformer.get_hash_offset(T)
-        yield '%s {' % cdecl(hash_typename, '')
-        yield '\tunion {'
-        yield '\t\t%s;' % cdecl(self.implementationtypename, 'head')
-        yield '\t\tchar pad[%s];' % name_signed(hash_offset, self.db)
-        yield '\t} u;'
-        yield '\tlong hash;'
-        yield '};'
-        yield '%s;' % (
-            forward_cdecl(hash_typename, '_hash_' + self.name,
-                          self.db.standalone, self.is_thread_local()),)
-        yield '#define %s _hash_%s.u.head' % (self.name, self.name)
-
-    def implementation(self):
-        hash_typename = self.get_hash_typename()
-        hash = self.db.gctransformer.get_prebuilt_hash(self.obj)
-        assert hash is not None
-        lines = list(self.initializationexpr())
-        lines.insert(0, '%s = { {' % (
-            cdecl(hash_typename, '_hash_' + self.name,
-                  self.is_thread_local()),))
-        lines.append('}, %s /* hash */ };' % name_signed(hash, self.db))
-        return lines
-
-def gcstructnode_factory(db, T, obj):
-    if (db.gctransformer and
-            db.gctransformer.get_prebuilt_hash(obj) is not None):
-        DISABLED
-        cls = GcStructNodeWithHash
-    else:
-        cls = StructNode
-    return cls(db, T, obj)
-
 
 class ArrayNode(ContainerNode):
     nodekind = 'array'
@@ -1058,7 +1012,7 @@
 
 ContainerNodeFactory = {
     Struct:       StructNode,
-    GcStruct:     gcstructnode_factory,
+    GcStruct:     StructNode,
     Array:        ArrayNode,
     GcArray:      ArrayNode,
     FixedSizeArray: FixedSizeArrayNode,

From pypy.commits at gmail.com  Fri Jan 27 16:10:19 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jan 2017 13:10:19 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: Import CPython definitions for the PEP
 393 structs
Message-ID: <588bb73b.d185df0a.ad9ab.6784@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89808:5a7f409a2e8a
Date: 2017-01-26 13:23 +0000
http://bitbucket.org/pypy/pypy/changeset/5a7f409a2e8a/

Log:	Import CPython definitions for the PEP 393 structs

diff --git a/pypy/module/cpyext/parse/cpyext_unicodeobject.h b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
--- a/pypy/module/cpyext/parse/cpyext_unicodeobject.h
+++ b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
@@ -1,13 +1,203 @@
+/* --- Internal Unicode Format -------------------------------------------- */
+
+
+/* Py_UNICODE was the native Unicode storage format (code unit) used by
+   Python and represents a single Unicode element in the Unicode type.
+   With PEP 393, Py_UNICODE is deprecated and replaced with a
+   typedef to wchar_t. */
+
+#define PY_UNICODE_TYPE wchar_t
+typedef wchar_t Py_UNICODE;
+
+/* Py_UCS4 and Py_UCS2 are typedefs for the respective
+   unicode representations. */
 typedef unsigned int Py_UCS4;
-/* On PyPy, Py_UNICODE is always wchar_t */
-#define PY_UNICODE_TYPE wchar_t
-typedef PY_UNICODE_TYPE Py_UNICODE;
+typedef unsigned short Py_UCS2;
+typedef unsigned char Py_UCS1;
 
-#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
+/* --- Unicode Type ------------------------------------------------------- */
 
+
+/* ASCII-only strings created through PyUnicode_New use the PyASCIIObject
+   structure. state.ascii and state.compact are set, and the data
+   immediately follow the structure. utf8_length and wstr_length can be found
+   in the length field; the utf8 pointer is equal to the data pointer. */
 typedef struct {
+    /* There are 4 forms of Unicode strings:
+
+       - compact ascii:
+
+         * structure = PyASCIIObject
+         * test: PyUnicode_IS_COMPACT_ASCII(op)
+         * kind = PyUnicode_1BYTE_KIND
+         * compact = 1
+         * ascii = 1
+         * ready = 1
+         * (length is the length of the utf8 and wstr strings)
+         * (data starts just after the structure)
+         * (since ASCII is decoded from UTF-8, the utf8 string are the data)
+
+       - compact:
+
+         * structure = PyCompactUnicodeObject
+         * test: PyUnicode_IS_COMPACT(op) && !PyUnicode_IS_ASCII(op)
+         * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
+           PyUnicode_4BYTE_KIND
+         * compact = 1
+         * ready = 1
+         * ascii = 0
+         * utf8 is not shared with data
+         * utf8_length = 0 if utf8 is NULL
+         * wstr is shared with data and wstr_length=length
+           if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
+           or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_t)=4
+         * wstr_length = 0 if wstr is NULL
+         * (data starts just after the structure)
+
+       - legacy string, not ready:
+
+         * structure = PyUnicodeObject
+         * test: kind == PyUnicode_WCHAR_KIND
+         * length = 0 (use wstr_length)
+         * hash = -1
+         * kind = PyUnicode_WCHAR_KIND
+         * compact = 0
+         * ascii = 0
+         * ready = 0
+         * interned = SSTATE_NOT_INTERNED
+         * wstr is not NULL
+         * data.any is NULL
+         * utf8 is NULL
+         * utf8_length = 0
+
+       - legacy string, ready:
+
+         * structure = PyUnicodeObject structure
+         * test: !PyUnicode_IS_COMPACT(op) && kind != PyUnicode_WCHAR_KIND
+         * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
+           PyUnicode_4BYTE_KIND
+         * compact = 0
+         * ready = 1
+         * data.any is not NULL
+         * utf8 is shared and utf8_length = length with data.any if ascii = 1
+         * utf8_length = 0 if utf8 is NULL
+         * wstr is shared with data.any and wstr_length = length
+           if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
+           or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_4)=4
+         * wstr_length = 0 if wstr is NULL
+
+       Compact strings use only one memory block (structure + characters),
+       whereas legacy strings use one block for the structure and one block
+       for characters.
+
+       Legacy strings are created by PyUnicode_FromUnicode() and
+       PyUnicode_FromStringAndSize(NULL, size) functions. They become ready
+       when PyUnicode_READY() is called.
+
+       See also _PyUnicode_CheckConsistency().
+    */
     PyObject_HEAD
-    Py_UNICODE *buffer;
-    Py_ssize_t length;
-    char *utf8buffer;
+    Py_ssize_t length;          /* Number of code points in the string */
+    //Py_hash_t hash;             /* Hash value; -1 if not set */
+    struct {
+        /*
+           SSTATE_NOT_INTERNED (0)
+           SSTATE_INTERNED_MORTAL (1)
+           SSTATE_INTERNED_IMMORTAL (2)
+
+           If interned != SSTATE_NOT_INTERNED, the two references from the
+           dictionary to this object are *not* counted in ob_refcnt.
+         */
+        unsigned int interned:2;
+        /* Character size:
+
+           - PyUnicode_WCHAR_KIND (0):
+
+             * character type = wchar_t (16 or 32 bits, depending on the
+               platform)
+
+           - PyUnicode_1BYTE_KIND (1):
+
+             * character type = Py_UCS1 (8 bits, unsigned)
+             * all characters are in the range U+0000-U+00FF (latin1)
+             * if ascii is set, all characters are in the range U+0000-U+007F
+               (ASCII), otherwise at least one character is in the range
+               U+0080-U+00FF
+
+           - PyUnicode_2BYTE_KIND (2):
+
+             * character type = Py_UCS2 (16 bits, unsigned)
+             * all characters are in the range U+0000-U+FFFF (BMP)
+             * at least one character is in the range U+0100-U+FFFF
+
+           - PyUnicode_4BYTE_KIND (4):
+
+             * character type = Py_UCS4 (32 bits, unsigned)
+             * all characters are in the range U+0000-U+10FFFF
+             * at least one character is in the range U+10000-U+10FFFF
+         */
+        unsigned int kind:3;
+        /* Compact is with respect to the allocation scheme. Compact unicode
+           objects only require one memory block while non-compact objects use
+           one block for the PyUnicodeObject struct and another for its data
+           buffer. */
+        unsigned int compact:1;
+        /* The string only contains characters in the range U+0000-U+007F (ASCII)
+           and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
+           set, use the PyASCIIObject structure. */
+        unsigned int ascii:1;
+        /* The ready flag indicates whether the object layout is initialized
+           completely. This means that this is either a compact object, or
+           the data pointer is filled out. The bit is redundant, and helps
+           to minimize the test in PyUnicode_IS_READY(). */
+        unsigned int ready:1;
+        /* Padding to ensure that PyUnicode_DATA() is always aligned to
+           4 bytes (see issue #19537 on m68k). */
+        unsigned int :24;
+    } state;
+    wchar_t *wstr;              /* wchar_t representation (null-terminated) */
+} PyASCIIObject;
+
+/* Non-ASCII strings allocated through PyUnicode_New use the
+   PyCompactUnicodeObject structure. state.compact is set, and the data
+   immediately follow the structure. */
+typedef struct {
+    PyASCIIObject _base;
+    Py_ssize_t utf8_length;     /* Number of bytes in utf8, excluding the
+                                 * terminating \0. */
+    char *utf8;                 /* UTF-8 representation (null-terminated) */
+    Py_ssize_t wstr_length;     /* Number of code points in wstr, possible
+                                 * surrogates count as two code points. */
+} PyCompactUnicodeObject;
+
+/* Strings allocated through PyUnicode_FromUnicode(NULL, len) use the
+   PyUnicodeObject structure. The actual string data is initially in the wstr
+   block, and copied into the data block using _PyUnicode_Ready. */
+typedef struct {
+    PyCompactUnicodeObject _base;
+    union {
+        void *any;
+        Py_UCS1 *latin1;
+        Py_UCS2 *ucs2;
+        Py_UCS4 *ucs4;
+    } data;                     /* Canonical, smallest-form Unicode buffer */
 } PyUnicodeObject;
+
+
+/* --- Flexible String Representation Helper Macros (PEP 393) -------------- */
+
+/* Values for PyASCIIObject.state: */
+
+/* Interning state. */
+#define SSTATE_NOT_INTERNED 0
+#define SSTATE_INTERNED_MORTAL 1
+#define SSTATE_INTERNED_IMMORTAL 2
+
+/* --- Constants ---------------------------------------------------------- */
+
+/* This Unicode character will be used as replacement character during
+   decoding if the errors argument is set to "replace". Note: the
+   Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
+   Unicode 3.0. */
+
+#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UCS4) 0xFFFD)

From pypy.commits at gmail.com  Fri Jan 27 16:10:21 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jan 2017 13:10:21 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: Massage the definitions into something
 we can parse
Message-ID: <588bb73d.8aa3df0a.92e58.6d07@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89809:45e2029d13ef
Date: 2017-01-26 13:43 +0000
http://bitbucket.org/pypy/pypy/changeset/45e2029d13ef/

Log:	Massage the definitions into something we can parse

diff --git a/pypy/module/cpyext/parse/cpyext_unicodeobject.h b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
--- a/pypy/module/cpyext/parse/cpyext_unicodeobject.h
+++ b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
@@ -17,6 +17,62 @@
 
 /* --- Unicode Type ------------------------------------------------------- */
 
+typedef struct {
+        /*
+           SSTATE_NOT_INTERNED (0)
+           SSTATE_INTERNED_MORTAL (1)
+           SSTATE_INTERNED_IMMORTAL (2)
+
+           If interned != SSTATE_NOT_INTERNED, the two references from the
+           dictionary to this object are *not* counted in ob_refcnt.
+         */
+        unsigned int interned;
+        /* Character size:
+
+           - PyUnicode_WCHAR_KIND (0):
+
+             * character type = wchar_t (16 or 32 bits, depending on the
+               platform)
+
+           - PyUnicode_1BYTE_KIND (1):
+
+             * character type = Py_UCS1 (8 bits, unsigned)
+             * all characters are in the range U+0000-U+00FF (latin1)
+             * if ascii is set, all characters are in the range U+0000-U+007F
+               (ASCII), otherwise at least one character is in the range
+               U+0080-U+00FF
+
+           - PyUnicode_2BYTE_KIND (2):
+
+             * character type = Py_UCS2 (16 bits, unsigned)
+             * all characters are in the range U+0000-U+FFFF (BMP)
+             * at least one character is in the range U+0100-U+FFFF
+
+           - PyUnicode_4BYTE_KIND (4):
+
+             * character type = Py_UCS4 (32 bits, unsigned)
+             * all characters are in the range U+0000-U+10FFFF
+             * at least one character is in the range U+10000-U+10FFFF
+         */
+        unsigned int kind;
+        /* Compact is with respect to the allocation scheme. Compact unicode
+           objects only require one memory block while non-compact objects use
+           one block for the PyUnicodeObject struct and another for its data
+           buffer. */
+        unsigned int compact;
+        /* The string only contains characters in the range U+0000-U+007F (ASCII)
+           and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
+           set, use the PyASCIIObject structure. */
+        unsigned int ascii;
+        /* The ready flag indicates whether the object layout is initialized
+           completely. This means that this is either a compact object, or
+           the data pointer is filled out. The bit is redundant, and helps
+           to minimize the test in PyUnicode_IS_READY(). */
+        unsigned int ready;
+        /* Padding to ensure that PyUnicode_DATA() is always aligned to
+           4 bytes (see issue #19537 on m68k). */
+        /* not on PyPy */
+    } _PyASCIIObject_state_t;
 
 /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject
    structure. state.ascii and state.compact are set, and the data
@@ -99,62 +155,7 @@
     PyObject_HEAD
     Py_ssize_t length;          /* Number of code points in the string */
     //Py_hash_t hash;             /* Hash value; -1 if not set */
-    struct {
-        /*
-           SSTATE_NOT_INTERNED (0)
-           SSTATE_INTERNED_MORTAL (1)
-           SSTATE_INTERNED_IMMORTAL (2)
-
-           If interned != SSTATE_NOT_INTERNED, the two references from the
-           dictionary to this object are *not* counted in ob_refcnt.
-         */
-        unsigned int interned:2;
-        /* Character size:
-
-           - PyUnicode_WCHAR_KIND (0):
-
-             * character type = wchar_t (16 or 32 bits, depending on the
-               platform)
-
-           - PyUnicode_1BYTE_KIND (1):
-
-             * character type = Py_UCS1 (8 bits, unsigned)
-             * all characters are in the range U+0000-U+00FF (latin1)
-             * if ascii is set, all characters are in the range U+0000-U+007F
-               (ASCII), otherwise at least one character is in the range
-               U+0080-U+00FF
-
-           - PyUnicode_2BYTE_KIND (2):
-
-             * character type = Py_UCS2 (16 bits, unsigned)
-             * all characters are in the range U+0000-U+FFFF (BMP)
-             * at least one character is in the range U+0100-U+FFFF
-
-           - PyUnicode_4BYTE_KIND (4):
-
-             * character type = Py_UCS4 (32 bits, unsigned)
-             * all characters are in the range U+0000-U+10FFFF
-             * at least one character is in the range U+10000-U+10FFFF
-         */
-        unsigned int kind:3;
-        /* Compact is with respect to the allocation scheme. Compact unicode
-           objects only require one memory block while non-compact objects use
-           one block for the PyUnicodeObject struct and another for its data
-           buffer. */
-        unsigned int compact:1;
-        /* The string only contains characters in the range U+0000-U+007F (ASCII)
-           and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
-           set, use the PyASCIIObject structure. */
-        unsigned int ascii:1;
-        /* The ready flag indicates whether the object layout is initialized
-           completely. This means that this is either a compact object, or
-           the data pointer is filled out. The bit is redundant, and helps
-           to minimize the test in PyUnicode_IS_READY(). */
-        unsigned int ready:1;
-        /* Padding to ensure that PyUnicode_DATA() is always aligned to
-           4 bytes (see issue #19537 on m68k). */
-        unsigned int :24;
-    } state;
+    _PyASCIIObject_state_t state;
     wchar_t *wstr;              /* wchar_t representation (null-terminated) */
 } PyASCIIObject;
 
@@ -175,12 +176,7 @@
    block, and copied into the data block using _PyUnicode_Ready. */
 typedef struct {
     PyCompactUnicodeObject _base;
-    union {
-        void *any;
-        Py_UCS1 *latin1;
-        Py_UCS2 *ucs2;
-        Py_UCS4 *ucs4;
-    } data;                     /* Canonical, smallest-form Unicode buffer */
+    void* data;                     /* Canonical, smallest-form Unicode buffer */
 } PyUnicodeObject;
 
 

From pypy.commits at gmail.com  Fri Jan 27 16:10:23 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jan 2017 13:10:23 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: Fix code to match the new structs
Message-ID: <588bb73f.ec98df0a.9eaae.6d62@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89810:1dc55bdc20a3
Date: 2017-01-27 21:05 +0000
http://bitbucket.org/pypy/pypy/changeset/1dc55bdc20a3/

Log:	Fix code to match the new structs

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -28,7 +28,7 @@
                  }
 #ifdef PYPY_VERSION
                  // Slightly silly test that tp_basicsize is reasonable.
-                 if(s->ob_type->tp_basicsize != sizeof(void*)*6)
+                 if(s->ob_type->tp_basicsize != sizeof(void*)*12)
                      result = s->ob_type->tp_basicsize;
 #endif  // PYPY_VERSION
                  Py_DECREF(s);
@@ -284,22 +284,23 @@
     def test_unicode_resize(self, space):
         py_uni = new_empty_unicode(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
-        py_uni.c_buffer[0] = u'a'
-        py_uni.c_buffer[1] = u'b'
-        py_uni.c_buffer[2] = u'c'
+        buf = get_wbuffer(py_uni)
+        buf[0] = u'a'
+        buf[1] = u'b'
+        buf[2] = u'c'
         ar[0] = rffi.cast(PyObject, py_uni)
         PyUnicode_Resize(space, ar, 3)
         py_uni = rffi.cast(PyUnicodeObject, ar[0])
-        assert py_uni.c_length == 3
-        assert py_uni.c_buffer[1] == u'b'
-        assert py_uni.c_buffer[3] == u'\x00'
+        assert get_wsize(py_uni) == 3
+        assert get_wbuffer(py_uni)[1] == u'b'
+        assert get_wbuffer(py_uni)[3] == u'\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_uni)
         PyUnicode_Resize(space, ar, 10)
         py_uni = rffi.cast(PyUnicodeObject, ar[0])
-        assert py_uni.c_length == 10
-        assert py_uni.c_buffer[1] == 'b'
-        assert py_uni.c_buffer[10] == '\x00'
+        assert get_wsize(py_uni) == 10
+        assert get_wbuffer(py_uni)[1] == 'b'
+        assert get_wbuffer(py_uni)[10] == '\x00'
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -49,17 +49,17 @@
     py_uni = rffi.cast(PyUnicodeObject, py_obj)
 
     buflen = length + 1
-    py_uni.c_length = length
-    py_uni.c_buffer = lltype.malloc(rffi.CWCHARP.TO, buflen,
-                                    flavor='raw', zero=True,
-                                    add_memory_pressure=True)
+    set_wsize(py_uni, length)
+    set_wbuffer(py_uni,
+        lltype.malloc(
+            rffi.CWCHARP.TO, buflen, flavor='raw', zero=True,
+            add_memory_pressure=True))
     return py_uni
 
 def unicode_attach(space, py_obj, w_obj, w_userdata=None):
     "Fills a newly allocated PyUnicodeObject with a unicode string"
-    py_unicode = rffi.cast(PyUnicodeObject, py_obj)
-    py_unicode.c_length = len(space.unicode_w(w_obj))
-    py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO)
+    set_wsize(py_obj, len(space.unicode_w(w_obj)))
+    set_wbuffer(py_obj, lltype.nullptr(rffi.CWCHARP.TO))
 
 def unicode_realize(space, py_obj):
     """
@@ -67,7 +67,7 @@
     be modified after this call.
     """
     py_uni = rffi.cast(PyUnicodeObject, py_obj)
-    s = rffi.wcharpsize2unicode(py_uni.c_buffer, py_uni.c_length)
+    s = rffi.wcharpsize2unicode(get_wbuffer(py_uni), get_wsize(py_uni))
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
     w_obj.__init__(s)
@@ -76,14 +76,37 @@
 
 @slot_function([PyObject], lltype.Void)
 def unicode_dealloc(space, py_obj):
-    py_unicode = rffi.cast(PyUnicodeObject, py_obj)
-    if py_unicode.c_buffer:
-        lltype.free(py_unicode.c_buffer, flavor="raw")
-    if py_unicode.c_utf8buffer:
-        lltype.free(py_unicode.c_utf8buffer, flavor="raw")
+    if get_wbuffer(py_obj):
+        lltype.free(get_wbuffer(py_obj), flavor="raw")
+    if get_utf8(py_obj):
+        lltype.free(get_utf8(py_obj), flavor="raw")
     from pypy.module.cpyext.object import _dealloc
     _dealloc(space, py_obj)
 
+def get_utf8(py_obj):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    return py_obj.c_utf8
+
+def set_utf8(py_obj, buf):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    py_obj.c_utf8 = buf
+
+def get_wsize(py_obj):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    return py_obj.c_wstr_length
+
+def set_wsize(py_obj, value):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    py_obj.c_wstr_length = value
+
+def get_wbuffer(py_obj):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    return py_obj.c_wstr
+
+def set_wbuffer(py_obj, wbuf):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    py_obj.c_wstr = wbuf
+
 @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
 def Py_UNICODE_ISSPACE(space, ch):
     """Return 1 or 0 depending on whether ch is a whitespace character."""
@@ -216,13 +239,12 @@
 def PyUnicode_AS_UNICODE(space, ref):
     """Return a pointer to the internal Py_UNICODE buffer of the object.  ref
     has to be a PyUnicodeObject (not checked)."""
-    ref_unicode = rffi.cast(PyUnicodeObject, ref)
-    if not ref_unicode.c_buffer:
+    if not get_wbuffer(ref):
         # Copy unicode buffer
         w_unicode = from_ref(space, rffi.cast(PyObject, ref))
         u = space.unicode_w(w_unicode)
-        ref_unicode.c_buffer = rffi.unicode2wcharp(u)
-    return ref_unicode.c_buffer
+        set_wbuffer(ref, rffi.unicode2wcharp(u))
+    return get_wbuffer(ref)
 
 @cpython_api([PyObject], rffi.CWCHARP)
 def PyUnicode_AsUnicode(space, ref):
@@ -237,14 +259,14 @@
 @api_decl("char * PyUnicode_AsUTF8(PyObject *unicode)", cts)
 def PyUnicode_AsUTF8(space, ref):
     ref_unicode = rffi.cast(PyUnicodeObject, ref)
-    if not ref_unicode.c_utf8buffer:
+    if not get_utf8(ref_unicode):
         # Copy unicode buffer
         w_unicode = from_ref(space, ref)
         w_encoded = unicodeobject.encode_object(space, w_unicode, "utf-8",
                                                 "strict")
         s = space.bytes_w(w_encoded)
-        ref_unicode.c_utf8buffer = rffi.str2charp(s)
-    return ref_unicode.c_utf8buffer
+        set_utf8(ref_unicode, rffi.str2charp(s))
+    return get_utf8(ref_unicode)
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
 def PyUnicode_GetSize(space, ref):
@@ -254,8 +276,7 @@
     Please migrate to using PyUnicode_GetLength().
     """
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_unicode:
-        ref = rffi.cast(PyUnicodeObject, ref)
-        return ref.c_length
+        return get_wsize(ref)
     else:
         w_obj = from_ref(space, ref)
         return space.len_w(w_obj)
@@ -280,7 +301,7 @@
     required by the application."""
     ref = rffi.cast(PyUnicodeObject, ref)
     c_buffer = PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
-    c_length = ref.c_length
+    c_length = get_wsize(ref)
 
     # If possible, try to copy the 0-termination as well
     if size > c_length:
@@ -566,7 +587,7 @@
 def PyUnicode_Resize(space, ref, newsize):
     # XXX always create a new string so far
     py_uni = rffi.cast(PyUnicodeObject, ref[0])
-    if not py_uni.c_buffer:
+    if not get_wbuffer(py_uni):
         raise oefmt(space.w_SystemError,
                     "PyUnicode_Resize called on already created string")
     try:
@@ -576,11 +597,11 @@
         ref[0] = lltype.nullptr(PyObject.TO)
         raise
     to_cp = newsize
-    oldsize = py_uni.c_length
+    oldsize = get_wsize(py_uni)
     if oldsize < newsize:
         to_cp = oldsize
     for i in range(to_cp):
-        py_newuni.c_buffer[i] = py_uni.c_buffer[i]
+        get_wbuffer(py_newuni)[i] = get_wbuffer(py_uni)[i]
     Py_DecRef(space, ref[0])
     ref[0] = rffi.cast(PyObject, py_newuni)
     return 0

From pypy.commits at gmail.com  Fri Jan 27 17:38:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 27 Jan 2017 14:38:39 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Another place that caches the hash
Message-ID: <588bcbef.8b811c0a.3dca8.0dd4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89811:4a6390ab0e05
Date: 2017-01-27 23:38 +0100
http://bitbucket.org/pypy/pypy/changeset/4a6390ab0e05/

Log:	Another place that caches the hash

diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -576,6 +576,11 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
+    def _cleanup_(self):
+        # in case there are frozenset objects existing during
+        # translation, make sure we don't translate a cached hash
+        self.hash = 0
+
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_FrozensetObject):
             return False

From pypy.commits at gmail.com  Fri Jan 27 22:07:03 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Fri, 27 Jan 2017 19:07:03 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: Match translator's
 usage expectations
Message-ID: <588c0ad7.6787df0a.b4168.b92e@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89812:40a92f4debad
Date: 2017-01-28 14:06 +1100
http://bitbucket.org/pypy/pypy/changeset/40a92f4debad/

Log:	Match translator's usage expectations

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -75,7 +75,7 @@
 
         if default_goal:
             default_goal, = self.backend_select_goals([default_goal])
-            if default_goal in self._maybe_skip():
+            if default_goal in self._disabled:
                 default_goal = None
 
         self.default_goal = default_goal
@@ -208,6 +208,8 @@
         return res
 
     def proceed(self, goals):
+        if not goals:
+            goals = [self.default_goal]
         backend, ts = self.get_backend_and_type_system()
         goals = set(self.backend_select_goals(goals + self.extra_goals))
         if not goals:

From pypy.commits at gmail.com  Sat Jan 28 02:22:27 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Fri, 27 Jan 2017 23:22:27 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: Use
 backend-specific disabled goals
Message-ID: <588c46b3.a1aedf0a.ff632.ed3e@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89813:c64d6e66020b
Date: 2017-01-28 18:21 +1100
http://bitbucket.org/pypy/pypy/changeset/c64d6e66020b/

Log:	Use backend-specific disabled goals

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -75,7 +75,7 @@
 
         if default_goal:
             default_goal, = self.backend_select_goals([default_goal])
-            if default_goal in self._disabled:
+            if default_goal in self.backend_select_goals(self._disabled):
                 default_goal = None
 
         self.default_goal = default_goal
@@ -98,9 +98,9 @@
 
     def backendopt_lltype(self):
         self.rtype_lltype()
-        if 'pyjitpl' in self.extra_goals:
+        if 'pyjitpl_lltype' in self.extra_goals:
             self.pyjitpl_lltype()
-        if 'jittest' in self.extra_goals:
+        if 'jittest_lltype' in self.extra_goals:
             self.jittest_lltype()
         
         return self.run_task(self.task_backendopt_lltype,
@@ -108,7 +108,7 @@
 
     def stackcheckinsertion_lltype(self):
         self.rtype_lltype()
-        if 'backendopt' in self.extra_goals:
+        if 'backendopt_lltype' in self.extra_goals:
             self.backendopt_lltype()
         return self.run_task(self.task_stackcheckinsertion_lltype,
                              'stackcheckinsertion_lltype')
@@ -135,7 +135,7 @@
         return self.run_task(self.task_run_c, 'run_c')
 
     def set_extra_goals(self, goals):
-        self.extra_goals = goals
+        self.extra_goals = self.backend_select_goals(goals)
 
     def set_backend_extra_options(self, extra_options):
         self._backend_extra_options = extra_options

From pypy.commits at gmail.com  Sat Jan 28 03:20:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 00:20:08 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: oups,
 the 'for_testing' logic was never invoked
Message-ID: <588c5438.0a8edf0a.8962.f446@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89814:84a5e4471eab
Date: 2017-01-28 00:08 +0100
http://bitbucket.org/pypy/pypy/changeset/84a5e4471eab/

Log:	oups, the 'for_testing' logic was never invoked

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -23,13 +23,14 @@
 # ____________________________________________________________
 
 class UniqueCache:
+    for_testing = False    # set to True on the class level in test_c.py
+
     def __init__(self, space):
         self.ctvoid = None      # Cache for the 'void' type
         self.ctvoidp = None     # Cache for the 'void *' type
         self.ctchara = None     # Cache for the 'char[]' type
         self.primitives = {}    # Cache for {name: primitive_type}
         self.functions = []     # see _new_function_type()
-        self.for_testing = False
 
 def _clean_cache(space):
     "NOT_RPYTHON"
diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py
--- a/pypy/module/_cffi_backend/test/test_c.py
+++ b/pypy/module/_cffi_backend/test/test_c.py
@@ -36,6 +36,7 @@
     def setup_class(cls):
         testfuncs_w = []
         keepalive_funcs = []
+        UniqueCache.for_testing = True
 
         def find_and_load_library_for_test(space, w_name, w_is_global=None):
             if w_is_global is None:
@@ -86,11 +87,12 @@
             _all_test_c.find_and_load_library = func
             _all_test_c._testfunc = testfunc
         """)
-        UniqueCache.for_testing = True
 
     def teardown_method(self, method):
+        _clean_cache(self.space)
+
+    def teardown_class(cls):
         UniqueCache.for_testing = False
-        _clean_cache(self.space)
 
 
 all_names = ', '.join(Module.interpleveldefs.keys())

From pypy.commits at gmail.com  Sat Jan 28 03:20:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 00:20:10 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Rehash cffi's prebuilt function
 types after translation.
Message-ID: <588c543a.4a6b1c0a.3b0f2.88ee@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89815:1efada6f7e36
Date: 2017-01-28 09:19 +0100
http://bitbucket.org/pypy/pypy/changeset/1efada6f7e36/

Log:	Rehash cffi's prebuilt function types after translation.

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -31,6 +31,26 @@
         self.ctchara = None     # Cache for the 'char[]' type
         self.primitives = {}    # Cache for {name: primitive_type}
         self.functions = []     # see _new_function_type()
+        self.functions_packed = None     # only across translation
+
+    def _cleanup_(self):
+        import gc
+        assert self.functions_packed is None
+        # Note: a full PyPy translation may still have
+        # 'self.functions == []' at this point, possibly depending
+        # on details.  Code tested directly in test_ffi_obj
+        gc.collect()
+        funcs = []
+        for weakdict in self.functions:
+            funcs += weakdict._dict.values()
+        del self.functions[:]
+        self.functions_packed = funcs if len(funcs) > 0 else None
+
+    def unpack_functions(self):
+        for fct in self.functions_packed:
+            _record_function_type(self, fct)
+        self.functions_packed = None
+
 
 def _clean_cache(space):
     "NOT_RPYTHON"
@@ -623,7 +643,7 @@
     for w_arg in fargs:
         y = compute_identity_hash(w_arg)
         x = intmask((1000003 * x) ^ y)
-    x ^= (ellipsis - abi)
+    x ^= ellipsis + 2 * abi
     if unique_cache.for_testing:    # constant-folded to False in translation;
         x &= 3                      # but for test, keep only 2 bits of hash
     return x
@@ -647,6 +667,8 @@
     # one such dict, but in case of hash collision, there might be
     # more.
     unique_cache = space.fromcache(UniqueCache)
+    if unique_cache.functions_packed is not None:
+        unique_cache.unpack_functions()
     func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
     for weakdict in unique_cache.functions:
         ctype = weakdict.get(func_hash)
@@ -675,13 +697,18 @@
     #
     fct = ctypefunc.W_CTypeFunc(space, fargs, fresult, ellipsis, abi)
     unique_cache = space.fromcache(UniqueCache)
-    func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
+    _record_function_type(unique_cache, fct)
+    return fct
+
+def _record_function_type(unique_cache, fct):
+    from pypy.module._cffi_backend import ctypefunc
+    #
+    func_hash = _func_key_hash(unique_cache, fct.fargs, fct.ctitem,
+                               fct.ellipsis, fct.abi)
     for weakdict in unique_cache.functions:
         if weakdict.get(func_hash) is None:
-            weakdict.set(func_hash, fct)
             break
     else:
         weakdict = rweakref.RWeakValueDictionary(int, ctypefunc.W_CTypeFunc)
         unique_cache.functions.append(weakdict)
-        weakdict.set(func_hash, fct)
-    return fct
+    weakdict.set(func_hash, fct)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -1,5 +1,23 @@
+from pypy.module._cffi_backend import newtype
 from pypy.module._cffi_backend.newtype import _clean_cache
 
+
+class TestFFIObj:
+    spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
+
+    def teardown_method(self, meth):
+        _clean_cache(self.space)
+
+    def test_new_function_type_during_translation(self):
+        space = self.space
+        BInt = newtype.new_primitive_type(space, "int")
+        BFunc = newtype.new_function_type(space, space.wrap([BInt]), BInt)
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+        unique_cache = space.fromcache(newtype.UniqueCache)
+        unique_cache._cleanup_()
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+
+
 class AppTestFFIObj:
     spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
 

From pypy.commits at gmail.com  Sat Jan 28 03:59:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 00:59:00 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Boehm: remove completely the
 header, can't use an empty struct because
Message-ID: <588c5d54.810b1c0a.d9417.93a9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89816:63f63ce03c30
Date: 2017-01-28 09:58 +0100
http://bitbucket.org/pypy/pypy/changeset/63f63ce03c30/

Log:	Boehm: remove completely the header, can't use an empty struct
	because it still takes one byte and then there are alignment issues
	for OP_LENGTH_OF_SIMPLE_GCARRAY_FROM_OPAQUE

diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -11,7 +11,7 @@
 class BoehmGCTransformer(GCTransformer):
     malloc_zero_filled = True
     FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
-    HDR = lltype.Struct("header")
+    NO_HEADER = True
 
     def __init__(self, translator, inline=False):
         super(BoehmGCTransformer, self).__init__(translator, inline=inline)
@@ -29,10 +29,7 @@
         ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length
         ll_malloc_varsize = mh.ll_malloc_varsize
 
-        HDRPTR = lltype.Ptr(self.HDR)
-
         def ll_identityhash(addr):
-            obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
             h = ~llmemory.cast_adr_to_int(addr)
             return h
 
@@ -192,10 +189,6 @@
                           resulttype = lltype.Signed)
         hop.genop('int_invert', [v_int], resultvar=hop.spaceop.result)
 
-    def gcheader_initdata(self, obj):
-        hdr = lltype.malloc(self.HDR, immortal=True)
-        return hdr._obj
-
 
 ########## weakrefs ##########
 # Boehm: weakref objects are small structures containing only a Boehm
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -610,10 +610,6 @@
     def special_funcptr_for_type(self, TYPE):
         return self.layoutbuilder.special_funcptr_for_type(TYPE)
 
-    def gc_header_for(self, obj):
-        hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj)
-        return hdr
-
     def get_hash_offset(self, T):
         type_id = self.get_type_id(T)
         assert not self.gcdata.q_is_varsize(type_id)
@@ -1504,7 +1500,7 @@
 
     def gcheader_initdata(self, obj):
         o = lltype.top_container(obj)
-        hdr = self.gc_header_for(o)
+        hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(o)
         return hdr._obj
 
     def get_finalizer_queue_index(self, hop):
diff --git a/rpython/memory/gctransform/refcounting.py b/rpython/memory/gctransform/refcounting.py
--- a/rpython/memory/gctransform/refcounting.py
+++ b/rpython/memory/gctransform/refcounting.py
@@ -76,7 +76,6 @@
         ll_malloc_varsize = mh.ll_malloc_varsize
 
         def ll_identityhash(addr):
-            obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
             h = llmemory.cast_adr_to_int(addr)
             return h
 
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -485,7 +485,7 @@
     so-called 'identity hash', which is the non-overridable default hash
     of Python.  Can be called for any RPython-level object that turns
     into a GC object, but not NULL.  The value will be different before
-    and after translation.
+    and after translation (WARNING: this is a change with older RPythons!)
     """
     assert x is not None
     return object.__hash__(x)
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -645,7 +645,7 @@
     OP_CAST_OPAQUE_PTR = OP_CAST_POINTER
 
     def OP_LENGTH_OF_SIMPLE_GCARRAY_FROM_OPAQUE(self, op):
-        return ('%s = *(long *)(((char *)%s) + sizeof(struct pypy_header0));'
+        return ('%s = *(long *)(((char *)%s) + RPY_SIZE_OF_GCHEADER);'
                 '  /* length_of_simple_gcarray_from_opaque */'
             % (self.expr(op.result), self.expr(op.args[0])))
 
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -12,7 +12,9 @@
 from rpython.rlib.rfloat import isfinite, isinf
 
 
-def needs_gcheader(T):
+def needs_gcheader(gctransformer, T):
+    if getattr(gctransformer, 'NO_HEADER', False):   # for boehm
+        return False
     if not isinstance(T, ContainerType):
         return False
     if T._gckind != 'gc':
@@ -87,7 +89,7 @@
         STRUCT = self.STRUCT
         if self.varlength is not None:
             self.normalizedtypename = db.gettype(STRUCT, who_asks=self)
-        if needs_gcheader(self.STRUCT):
+        if needs_gcheader(db.gctransformer, self.STRUCT):
             HDR = db.gcpolicy.struct_gcheader_definition(self)
             if HDR is not None:
                 gc_field = ("_gcheader", db.gettype(HDR, who_asks=self))
@@ -213,7 +215,7 @@
         self.computegcinfo(db.gcpolicy)
         if self.varlength is not None:
             self.normalizedtypename = db.gettype(ARRAY, who_asks=self)
-        if needs_gcheader(ARRAY):
+        if needs_gcheader(db.gctransformer, ARRAY):
             HDR = db.gcpolicy.array_gcheader_definition(self)
             if HDR is not None:
                 gc_field = ("_gcheader", db.gettype(HDR, who_asks=self))
@@ -546,8 +548,8 @@
 
     def __init__(self, db, T, obj):
         ContainerNode.__init__(self, db, T, obj)
-        if needs_gcheader(T):
-            gct = self.db.gctransformer
+        gct = self.db.gctransformer
+        if needs_gcheader(gct, T):
             if gct is not None:
                 self.gc_init = gct.gcheader_initdata(self.obj)
             else:
@@ -577,7 +579,7 @@
 
         data = []
 
-        if needs_gcheader(T):
+        if needs_gcheader(self.db.gctransformer, T):
             data.append(('gcheader', self.gc_init))
 
         for name in defnode.fieldnames:
@@ -631,8 +633,8 @@
 
     def __init__(self, db, T, obj):
         ContainerNode.__init__(self, db, T, obj)
-        if needs_gcheader(T):
-            gct = self.db.gctransformer
+        gct = self.db.gctransformer
+        if needs_gcheader(gct, T):
             if gct is not None:
                 self.gc_init = gct.gcheader_initdata(self.obj)
             else:
@@ -655,7 +657,7 @@
     def initializationexpr(self, decoration=''):
         T = self.getTYPE()
         yield '{'
-        if needs_gcheader(T):
+        if needs_gcheader(self.db.gctransformer, T):
             lines = generic_initializationexpr(self.db, self.gc_init, 'gcheader',
                                                '%sgcheader' % (decoration,))
             for line in lines:
diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h
--- a/rpython/translator/c/src/mem.h
+++ b/rpython/translator/c/src/mem.h
@@ -135,6 +135,12 @@
 #define OP_GC_FQ_NEXT_DEAD(tag, r)       (r = NULL)
 #endif
 
+#if defined(PYPY_USING_BOEHM_GC) || defined(PYPY_USING_NO_GC_AT_ALL)
+#  define RPY_SIZE_OF_GCHEADER  0
+#else
+#  define RPY_SIZE_OF_GCHEADER  sizeof(struct pypy_header0)
+#endif
+
 /************************************************************/
 /* weakref support */
 

From pypy.commits at gmail.com  Sat Jan 28 05:10:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 02:10:37 -0800 (PST)
Subject: [pypy-commit] pypy default: small optimization
Message-ID: <588c6e1d.cc881c0a.7c41.ad7b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89817:76e31bc8fefd
Date: 2017-01-28 11:09 +0100
http://bitbucket.org/pypy/pypy/changeset/76e31bc8fefd/

Log:	small optimization

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1243,6 +1243,9 @@
     struct = SERVERNAME_CALLBACKS.get(rffi.cast(lltype.Signed, arg))
     w_ctx = struct.w_ctx
     space = struct.space
+    # annotation: ensures that we ignore the case 'space is None'
+    # (it would propagate to a few methods like errorstr())
+    assert w_ctx is not None and space is not None
     w_callback = struct.w_set_hostname
     if not w_ctx.servername_callback:
         # Possible race condition.

From pypy.commits at gmail.com  Sat Jan 28 06:25:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 03:25:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Issue a SyntaxWarning in a case where
 CPython (and PyPy) give a rather
Message-ID: <588c7fb9.ce9adf0a.bd348.3ee2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89818:13beb9aa557f
Date: 2017-01-28 12:25 +0100
http://bitbucket.org/pypy/pypy/changeset/13beb9aa557f/

Log:	Issue a SyntaxWarning in a case where CPython (and PyPy) give a
	rather nonsensical result

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -547,6 +547,13 @@
         for item in list(consider):
             item.walkabout(self)
         self.pop_scope()
+        # http://bugs.python.org/issue10544: was never fixed in CPython,
+        # but we can at least issue a SyntaxWarning in the meantime
+        if new_scope.is_generator:
+            msg = ("'yield' inside a list or generator comprehension behaves "
+                   "unexpectedly (http://bugs.python.org/issue10544)")
+            misc.syntax_warning(self.space, msg, self.compile_info.filename,
+                                node.lineno, node.col_offset)
 
     def visit_ListComp(self, listcomp):
         self._visit_comprehension(listcomp, listcomp.generators, listcomp.elt)
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -343,6 +343,7 @@
             assert ex.match(self.space, self.space.w_SyntaxError)
 
     def test_globals_warnings(self):
+        # also tests some other constructions that give a warning
         space = self.space
         w_mod = space.appexec((), '():\n import warnings\n return warnings\n') #sys.getmodule('warnings')
         w_filterwarnings = space.getattr(w_mod, space.wrap('filterwarnings'))
@@ -364,6 +365,18 @@
     print x
     x = 2
     global x
+''', '''
+def wrong_listcomp():
+    return [(yield 42) for i in j]
+''', '''
+def wrong_gencomp():
+    return ((yield 42) for i in j)
+''', '''
+def wrong_dictcomp():
+    return {(yield 42):2 for i in j}
+''', '''
+def wrong_setcomp():
+    return {(yield 42) for i in j}
 '''):
 
             space.call_args(w_filterwarnings, filter_arg)

From pypy.commits at gmail.com  Sat Jan 28 07:09:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 04:09:56 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: test fixes or removals
Message-ID: <588c8a14.2da9df0a.7f654.4983@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89819:9f2a9fe34a09
Date: 2017-01-28 13:09 +0100
http://bitbucket.org/pypy/pypy/changeset/9f2a9fe34a09/

Log:	test fixes or removals

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3704,25 +3704,6 @@
         s = a.build_types(f, [int])
         assert s.const == 0
 
-    def test_hash_sideeffect(self):
-        class X:
-            pass
-        x1 = X()
-        x2 = X()
-        x3 = X()
-        d = {(2, x1): 5, (3, x2): 7}
-        def f(n, m):
-            if   m == 1: x = x1
-            elif m == 2: x = x2
-            else:        x = x3
-            return d[n, x]
-        a = self.RPythonAnnotator()
-        s = a.build_types(f, [int, int])
-        assert s.knowntype == int
-        assert hasattr(x1, '__precomputed_identity_hash')
-        assert hasattr(x2, '__precomputed_identity_hash')
-        assert not hasattr(x3, '__precomputed_identity_hash')
-
     def test_contains_of_empty_dict(self):
         class A(object):
             def meth(self):
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -166,7 +166,6 @@
     foo = Foo()
     h = compute_hash(foo)
     assert h == object.__hash__(foo)
-    assert h == getattr(foo, '__precomputed_identity_hash')
     assert compute_hash(None) == 0
 
 def test_compute_hash_float():
@@ -182,7 +181,6 @@
     foo = Foo()
     h = compute_identity_hash(foo)
     assert h == object.__hash__(foo)
-    assert h == getattr(foo, '__precomputed_identity_hash')
 
 def test_compute_unique_id():
     from rpython.rlib.rarithmetic import intmask
@@ -410,36 +408,6 @@
         res = self.interpret(f, [])
         assert res == 1
 
-    def test_compute_hash_across_translation(self):
-        class Foo(object):
-            pass
-        q = Foo()
-
-        def f(i):
-            assert compute_hash(None) == 0
-            assert compute_hash(i) == h_42
-            assert compute_hash(i + 1.0) == h_43_dot_0
-            assert compute_hash((i + 3) / 6.0) == h_7_dot_5
-            assert compute_hash("Hello" + str(i)) == h_Hello42
-            if i == 42:
-                p = None
-            else:
-                p = Foo()
-            assert compute_hash(p) == h_None
-            assert compute_hash(("world", None, i, 7.5)) == h_tuple
-            assert compute_hash(q) == h_q
-            return i * 2
-        h_42 = compute_hash(42)
-        h_43_dot_0 = compute_hash(43.0)
-        h_7_dot_5 = compute_hash(7.5)
-        h_Hello42 = compute_hash("Hello42")
-        h_None = compute_hash(None)
-        h_tuple = compute_hash(("world", None, 42, 7.5))
-        h_q = compute_hash(q)
-
-        res = self.interpret(f, [42])
-        assert res == 84
-
     def test_fetch_translated_config(self):
         assert fetch_translated_config() is None
         def f():
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -749,22 +749,10 @@
     assert hash3 == identityhash(s3)
     assert hash3 == identityhash(s3.super)
     assert hash3 == identityhash(s3.super.super)
-    py.test.raises(ValueError, init_identity_hash, s3, hash3^1)
-    py.test.raises(ValueError, init_identity_hash, s3.super, hash3^4)
-    py.test.raises(ValueError, init_identity_hash, s3.super.super, hash3^9)
-
-    s3 = malloc(S3)
-    init_identity_hash(s3.super, -123)
-    assert -123 == identityhash(s3)
-    assert -123 == identityhash(s3.super)
-    assert -123 == identityhash(s3.super.super)
-    py.test.raises(ValueError, init_identity_hash, s3, 4313)
-    py.test.raises(ValueError, init_identity_hash, s3.super, 0)
-    py.test.raises(ValueError, init_identity_hash, s3.super.super, -124)
 
     from rpython.rtyper.lltypesystem import llmemory
     p3 = cast_opaque_ptr(llmemory.GCREF, s3)
-    assert -123 == identityhash(p3)
+    assert hash3 == identityhash(p3)
 
     A = GcArray(Signed)
     a = malloc(A, 3)
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -459,8 +459,7 @@
         res = self.interpret(f, [3])
         assert ''.join(res.chars) == 'CCls'
 
-    def test_hash_preservation(self):
-        from rpython.rlib.objectmodel import current_object_addr_as_int
+    def test_compute_identity_hash(self):
         from rpython.rlib.objectmodel import compute_identity_hash
         class C:
             pass
@@ -468,27 +467,15 @@
             pass
         c = C()
         d = D()
-        h_c = compute_identity_hash(c)
-        h_d = compute_identity_hash(d)
         #
         def f():
             d2 = D()
             return (compute_identity_hash(d2),
-                    current_object_addr_as_int(d2),
                     compute_identity_hash(c),
                     compute_identity_hash(d))
 
-        res = self.interpret(f, [])
-        # xxx the following test is too precise, checking the exact
-        # implementation.  On Python 2.7 it doesn't work anyway, because
-        # object.__hash__(x) is different from id(x).  The test is disabled
-        # for now, and nobody should rely on compute_identity_hash() returning
-        # a value that is (or was) the current_object_addr_as_int().
-        # --- disabled: assert res.item0 == res.item1
-        # the following property is essential on top of the lltypesystem
-        # otherwise prebuilt dictionaries are broken.
-        assert res.item2 == h_c
-        assert res.item3 == h_d
+        self.interpret(f, [])
+        # check does not crash
 
     def test_circular_hash_initialization(self):
         class B:
diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py
--- a/rpython/translator/c/test/test_lltyped.py
+++ b/rpython/translator/c/test/test_lltyped.py
@@ -332,6 +332,7 @@
         assert res == 3050
 
     def test_gcarray_nolength(self):
+        py.test.skip("GcArrays should never be 'nolength'")
         A = GcArray(Signed, hints={'nolength': True})
         a1 = malloc(A, 3, immortal=True)
         a1[0] = 30
diff --git a/rpython/translator/tool/test/test_staticsizereport.py b/rpython/translator/tool/test/test_staticsizereport.py
--- a/rpython/translator/tool/test/test_staticsizereport.py
+++ b/rpython/translator/tool/test/test_staticsizereport.py
@@ -59,13 +59,14 @@
         assert guess_size(func.builder.db, dictvalnode, set()) > 100
         assert guess_size(func.builder.db, dictvalnode2, set()) == (
             (4 * S + 2 * P) +     # struct dicttable
-            (S + 16) +            # indexes, length 16
+            # (S + 16) +          # indexes, length 16, but is absent here
             (S + S + S))          # entries, length 1
         r_set = set()
         dictnode_size = guess_size(db, test_dictnode, r_set)
         assert dictnode_size == (
             (4 * S + 2 * P) +      # struct dicttable
-            (S + 2 * 8192) +       # indexes, length 8192, rffi.USHORT
+            # (S + 2 * 8192) +     # indexes, length 8192, rffi.USHORT,
+                                   # but is absent here during translation
             (S + (S + S) * 3840) + # entries, length 3840
             (S + S + 6) * 3840)    # 3840 strings with 5 chars each (+1 final)
         assert guess_size(func.builder.db, fixarrayvalnode, set()) == 100 * rffi.sizeof(lltype.Signed) + 1 * rffi.sizeof(lltype.Signed)

From pypy.commits at gmail.com  Sat Jan 28 10:56:30 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 07:56:30 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Make rsiphash initialize itself at
 runtime, either with a random seed or
Message-ID: <588cbf2e.09bb1c0a.66c7a.12d8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89821:a36ffe6d56ba
Date: 2017-01-28 16:55 +0100
http://bitbucket.org/pypy/pypy/changeset/a36ffe6d56ba/

Log:	Make rsiphash initialize itself at runtime, either with a random
	seed or with the value of PYTHONHASHSEED, like CPython (the name
	PYTHONHASHSEED can be changed by the RPython interpreter if needed).

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -1,9 +1,10 @@
-import sys, os, struct
+import sys, os
 from contextlib import contextmanager
-from rpython.rlib import rarithmetic
+from rpython.rlib import rarithmetic, rurandom
 from rpython.rlib.objectmodel import not_rpython, always_inline
+from rpython.rlib.objectmodel import we_are_translated, dont_inline
 from rpython.rlib.rgc import no_collect
-from rpython.rlib.rarithmetic import r_uint64
+from rpython.rlib.rarithmetic import r_uint64, r_uint32, r_uint
 from rpython.rlib.rawstorage import misaligned_is_fine
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -16,37 +17,82 @@
     _le64toh = rarithmetic.byteswap
 
 
-# Initialize the values of the secret seed: two 64-bit constants.
-# CPython picks a new seed every time 'python' starts.  PyPy cannot do
-# that as easily because many details may rely on getting the same hash
-# value before and after translation.  We can, however, pick a random
-# seed once per translation, which should already be quite good.
-#
-# XXX no, it is not: e.g. all Ubuntu installations of the same Ubuntu
-# would get the same seed.  That's not good enough.
+class Seed:
+    k0l = k1l = r_uint64(0)
+    initialized = False
+seed = Seed()
 
- at not_rpython
-def select_random_seed():
-    global k0, k1    # note: the globals k0, k1 are already byte-swapped
-    v0, v1 = struct.unpack("QQ", os.urandom(16))
-    k0 = r_uint64(v0)
-    k1 = r_uint64(v1)
 
-select_random_seed()
+def select_random_seed(s):
+    """'s' is a string of length 16"""
+    seed.k0l = (
+      ord(s[0]) | ord(s[1]) << 8 | ord(s[2]) << 16 | ord(s[3]) << 24 |
+      ord(s[4]) << 32 | ord(s[5]) << 40 | ord(s[6]) << 48 | ord(s[7]) << 56)
+    seed.k1l = (
+      ord(s[8]) | ord(s[9]) << 8 | ord(s[10]) << 16 | ord(s[11]) << 24 |
+      ord(s[12]) << 32 | ord(s[13]) << 40 | ord(s[14]) << 48 | ord(s[15]) << 56)
+
+
+random_ctx = rurandom.init_urandom()
+
+def lcg_urandom(value):
+    # Quite unsure what the point of this function is, given that a hash
+    # seed of the form '%s\x00\x00\x00..' should be just as hard to
+    # guess as this one.  We copy it anyway from CPython for the case
+    # where 'value' is a 32-bit unsigned number, but if it is not, we
+    # fall back to the '%s\x00\x00\x00..' form.
+    if value == '0':
+        value = ''
+    try:
+        x = r_uint(r_uint32(value))
+    except (ValueError, OverflowError):
+        x = r_uint(0)
+    if str(x) == value:
+        s = ''
+        for index in range(16):
+            x *= 214013
+            x += 2531011
+            x = r_uint(r_uint32(x))
+            s += chr((x >> 16) & 0xff)
+    else:
+        if len(value) < 16:
+            s = value + '\x00' * (16 - len(value))
+        else:
+            s = value[:16]
+    return s
+
+env_var_name = "PYTHONHASHSEED"
+
+ at dont_inline
+def initialize_from_env():
+    # This uses the same algorithms as CPython 3.5.  The environment
+    # variable we read also defaults to "PYTHONHASHSEED".  If needed,
+    # a different RPython interpreter can patch the value of the
+    # global variable 'env_var_name', or completely patch this function
+    # with a different one.
+    value = os.environ.get(env_var_name)
+    if len(value) > 0 and value != "random":
+        s = lcg_urandom(value)
+    else:
+        s = rurandom.urandom(random_ctx, 16)
+    select_random_seed(s)
+    seed.initialized = True
+
 
 @contextmanager
 def choosen_seed(new_k0, new_k1, test_misaligned_path=False):
-    global k0, k1, misaligned_is_fine
-    old = k0, k1, misaligned_is_fine
-    k0 = _le64toh(r_uint64(new_k0))
-    k1 = _le64toh(r_uint64(new_k1))
+    """For tests."""
+    global misaligned_is_fine
+    old = seed.k0l, seed.k1l, misaligned_is_fine
+    seed.k0l = _le64toh(r_uint64(new_k0))
+    seed.k1l = _le64toh(r_uint64(new_k1))
     if test_misaligned_path:
         misaligned_is_fine = False
     yield
-    k0, k1, misaligned_is_fine = old
+    seed.k0l, seed.k1l, misaligned_is_fine = old
 
 def get_current_seed():
-    return _le64toh(k0), _le64toh(k1)
+    return _le64toh(seed.k0l), _le64toh(seed.k1l)
 
 
 magic0 = r_uint64(0x736f6d6570736575)
@@ -82,15 +128,18 @@
     """Takes an address pointer and a size.  Returns the hash as a r_uint64,
     which can then be casted to the expected type."""
 
-    direct = (misaligned_is_fine or
-                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
-
+    if we_are_translated() and not seed.initialized:
+        initialize_from_env()
+    k0 = seed.k0l
+    k1 = seed.k1l
     b = r_uint64(size) << 56
     v0 = k0 ^ magic0
     v1 = k1 ^ magic1
     v2 = k0 ^ magic2
     v3 = k1 ^ magic3
 
+    direct = (misaligned_is_fine or
+                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
     index = 0
     if direct:
         while size >= 8:
@@ -113,7 +162,6 @@
                 r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48 |
                 r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 7)) << 56
             )
-            mi = _le64toh(mi)
             size -= 8
             index += 8
             v3 ^= mi
diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -57,6 +57,8 @@
                              immortal=True, zero=True)
 
     def urandom(context, n, signal_checker=None):
+        # NOTE: no dictionaries here: rsiphash24 calls this to
+        # initialize the random seed of string hashes
         provider = context[0]
         if not provider:
             # This handle is never explicitly released. The operating
@@ -139,6 +141,8 @@
 
     def urandom(context, n, signal_checker=None):
         "Read n bytes from /dev/urandom."
+        # NOTE: no dictionaries here: rsiphash24 calls this to
+        # initialize the random seed of string hashes
         result = []
         if SYS_getrandom is not None:
             n = _getrandom(n, result, signal_checker)
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
--- a/rpython/rlib/test/test_rsiphash.py
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -1,4 +1,6 @@
+import os
 from rpython.rlib.rsiphash import siphash24, choosen_seed
+from rpython.rlib.rsiphash import initialize_from_env, seed
 from rpython.rtyper.lltypesystem import llmemory, rffi
 
 
@@ -42,3 +44,24 @@
 def test_siphash24():
     for expected, string in CASES:
         assert check(string) == expected
+
+def test_fix_seed():
+    p = rffi.str2charp("foo")
+    adr = llmemory.cast_ptr_to_adr(p)
+
+    os.environ['PYTHONHASHSEED'] = '0'
+    initialize_from_env()
+    assert siphash24(adr, 3) == 15988776847138518036 # checked with CPython 3.5
+
+    os.environ['PYTHONHASHSEED'] = '123'
+    initialize_from_env()
+    assert siphash24(adr, 3) == 12577370453467666022 # checked with CPython 3.5
+
+    os.environ['PYTHONHASHSEED'] = 'random'
+    initialize_from_env()
+    hash1 = siphash24(adr, 3)
+    initialize_from_env()
+    hash2 = siphash24(adr, 3)
+    assert hash1 != hash2
+
+    rffi.free_charp(p)

From pypy.commits at gmail.com  Sat Jan 28 10:56:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 07:56:29 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Remove the precomputed hash
 systematically when writing STR/UNICODE
Message-ID: <588cbf2d.ccaddf0a.91179.8d09@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89820:82f0db9460ee
Date: 2017-01-28 16:51 +0100
http://bitbucket.org/pypy/pypy/changeset/82f0db9460ee/

Log:	Remove the precomputed hash systematically when writing STR/UNICODE
	objects to C, if using a non-default hash function for strings

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -513,7 +513,7 @@
 
 # ----------
 
-HASH_ALGORITHM = "rpython"  # XXX Is there a better name?
+HASH_ALGORITHM = "rpython"    # the default, no source of randomness possible
 HASH_ALGORITHM_FIXED = False
 
 @not_rpython
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -1,7 +1,7 @@
 from weakref import WeakValueDictionary
 
 from rpython.annotator import model as annmodel
-from rpython.rlib import jit, types
+from rpython.rlib import jit, types, objectmodel
 from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated,
     ll_hash_string, keepalive_until_here, specialize, enforceargs)
 from rpython.rlib.signature import signature
@@ -169,7 +169,10 @@
             for i in range(len(value)):
                 p.chars[i] = cast_primitive(self.base, value[i])
             p.hash = 0
-            self.ll.ll_strhash(p)   # precompute the hash
+            if objectmodel.HASH_ALGORITHM == "rpython":
+                self.ll.ll_strhash(p)   # precompute the hash
+                # but it is pointless if this hash wouldn't end up in the
+                # C code anyway: see "remove_hash" in translator/c/node.py
             self.CACHE[value] = p
             return p
 
@@ -400,6 +403,7 @@
 
     @staticmethod
     def ll_strfasthash(s):
+        ll_assert(s.hash != 0, "ll_strfasthash: hash==0")
         return s.hash     # assumes that the hash is already computed
 
     @staticmethod
@@ -1258,7 +1262,8 @@
                               'gethash': LLHelpers.ll_strhash,
                               'length': LLHelpers.ll_length,
                               'find': LLHelpers.ll_find,
-                              'rfind': LLHelpers.ll_rfind}))
+                              'rfind': LLHelpers.ll_rfind},
+                    hints={'remove_hash': True}))
 UNICODE.become(GcStruct('rpy_unicode', ('hash', Signed),
                         ('chars', Array(UniChar, hints={'immutable': True})),
                         adtmeths={'malloc' : staticAdtMethod(mallocunicode),
@@ -1266,8 +1271,8 @@
                                   'copy_contents' : staticAdtMethod(copy_unicode_contents),
                                   'copy_contents_from_str' : staticAdtMethod(copy_unicode_contents),
                                   'gethash': LLHelpers.ll_strhash,
-                                  'length': LLHelpers.ll_length}
-                        ))
+                                  'length': LLHelpers.ll_length},
+                    hints={'remove_hash': True}))
 
 
 # TODO: make the public interface of the rstr module cleaner
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -8,7 +8,7 @@
 from rpython.translator.c.support import cdecl, forward_cdecl, somelettersfrom
 from rpython.translator.c.support import c_char_array_constant, barebonearray
 from rpython.translator.c.primitive import PrimitiveType, name_signed
-from rpython.rlib import exports
+from rpython.rlib import exports, objectmodel
 from rpython.rlib.rfloat import isfinite, isinf
 
 
@@ -585,12 +585,20 @@
         for name in defnode.fieldnames:
             data.append((name, getattr(self.obj, name)))
 
+        if T._hints.get('remove_hash'):
+            # hack for rstr.STR and UNICODE
+            if objectmodel.HASH_ALGORITHM != "rpython":
+                i = 0
+                while data[i][0] != 'hash':
+                    i += 1
+                data[i] = ('hash', 0)
+
         # Reasonably, you should only initialise one of the fields of a union
         # in C.  This is possible with the syntax '.fieldname value' or
         # '.fieldname = value'.  But here we don't know which of the
         # fields need initialization, so XXX we pick the first one
         # arbitrarily.
-        if hasattr(T, "_hints") and T._hints.get('union'):
+        if T._hints.get('union'):
             data = data[0:1]
 
         if 'get_padding_drop' in T._hints:

From pypy.commits at gmail.com  Sat Jan 28 11:15:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 08:15:27 -0800 (PST)
Subject: [pypy-commit] cffi default: Expand the warning box
Message-ID: <588cc39f.8b811c0a.3dca8.2076@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2876:b7deca8bf7f4
Date: 2017-01-28 17:15 +0100
http://bitbucket.org/cffi/cffi/changeset/b7deca8bf7f4/

Log:	Expand the warning box

diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -776,13 +776,21 @@
     Callbacks are provided for the ABI mode or for backward
     compatibility.  If you are using the out-of-line API mode, it is
     recommended to use the `extern "Python"`_ mechanism instead of
-    callbacks: it gives faster and cleaner code.  It also avoids a
-    SELinux issue whereby the setting of ``deny_execmem`` must be left
-    to ``off`` in order to use callbacks.  (A fix in cffi was
-    attempted---see the ``ffi_closure_alloc`` branch---but was not
-    merged because it creates potential memory corruption with
-    ``fork()``.  For more information, `see here.`__)
+    callbacks: it gives faster and cleaner code.  It also avoids several
+    issues with old-style callbacks:
 
+    - On less common architecture, libffi is more likely to crash on
+      callbacks (`e.g. on NetBSD`__);
+
+    - On hardened systems like PAX and SELinux, the extra memory
+      protections can interfere (for example, on SELinux you need to
+      run with ``deny_execmem`` set to ``off``).
+
+    Note also that a cffi fix for the latter issue was attempted---see
+    the ``ffi_closure_alloc`` branch---but was not merged because it
+    creates potential `memory corruption`__ with ``fork()``.
+
+.. __: https://github.com/pyca/pyopenssl/issues/596
 .. __: https://bugzilla.redhat.com/show_bug.cgi?id=1249685
 
 Warning: like ffi.new(), ffi.callback() returns a cdata that has

From pypy.commits at gmail.com  Sat Jan 28 11:31:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 08:31:55 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: prebuilt RWeakValueDictionary can
 be non-empty. Force a rehashing the first time we use them
Message-ID: <588cc77b.eb86df0a.3d2da.a7bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89822:7dccd4ef8d5d
Date: 2017-01-28 17:31 +0100
http://bitbucket.org/pypy/pypy/changeset/7dccd4ef8d5d/

Log:	prebuilt RWeakValueDictionary can be non-empty. Force a rehashing
	the first time we use them

diff --git a/rpython/rlib/_rweakvaldict.py b/rpython/rlib/_rweakvaldict.py
--- a/rpython/rlib/_rweakvaldict.py
+++ b/rpython/rlib/_rweakvaldict.py
@@ -76,12 +76,16 @@
             bk = self.rtyper.annotator.bookkeeper
             classdef = bk.getuniqueclassdef(weakdict._valueclass)
             r_value = getinstancerepr(self.rtyper, classdef)
+            any_value = False
             for dictkey, dictvalue in weakdict._dict.items():
                 llkey = self.r_key.convert_const(dictkey)
                 llvalue = r_value.convert_const(dictvalue)
                 if llvalue:
                     llvalue = lltype.cast_pointer(rclass.OBJECTPTR, llvalue)
                     self.ll_set_nonnull(l_dict, llkey, llvalue)
+                    any_value = True
+            if any_value:
+                l_dict.resize_counter = -1
             return l_dict
 
     def rtype_method_get(self, hop):
@@ -114,6 +118,8 @@
 
     @jit.dont_look_inside
     def ll_get(self, d, llkey):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         #llop.debug_print(lltype.Void, i, 'get')
@@ -132,6 +138,8 @@
 
     @jit.dont_look_inside
     def ll_set_nonnull(self, d, llkey, llvalue):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         valueref = weakref_create(llvalue)    # GC effects here, before the rest
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
@@ -147,6 +155,8 @@
 
     @jit.dont_look_inside
     def ll_set_null(self, d, llkey):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         if d.entries.everused(i):
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -3,6 +3,7 @@
 from rpython.rlib import rgc
 from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rtyper.test.test_llinterp import interpret
+from rpython.translator.c.test.test_genc import compile
 
 class X(object):
     pass
@@ -213,3 +214,18 @@
         assert d.get(keys[3]) is None
     f()
     interpret(f, [])
+
+def test_translation_prebuilt():
+    class K:
+        pass
+    d = RWeakValueDictionary(K, X)
+    k1 = K(); k2 = K()
+    x1 = X(); x2 = X()
+    d.set(k1, x1)
+    d.set(k2, x2)
+    def f():
+        assert d.get(k1) is x1
+        assert d.get(k2) is x2
+    f()
+    fc = compile(f, [], gcpolicy="boehm", rweakref=True)
+    fc()

From pypy.commits at gmail.com  Sat Jan 28 12:04:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 09:04:15 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Cleanups and comments
Message-ID: <588ccf0f.2da9df0a.7f654.a6fc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89823:6879b32267d1
Date: 2017-01-28 17:45 +0100
http://bitbucket.org/pypy/pypy/changeset/6879b32267d1/

Log:	Cleanups and comments

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -287,7 +287,7 @@
                     for ek, ev in items:
                         result.dictdef.generalize_key(self.immutablevalue(ek))
                         result.dictdef.generalize_value(self.immutablevalue(ev))
-                        result.dictdef.seen_prebuilt_key(ek)
+                        #dictdef.seen_prebuilt_key(ek)---not needed any more
                     seen_elements = len(items)
                     # if the dictionary grew during the iteration,
                     # start over again
diff --git a/rpython/annotator/dictdef.py b/rpython/annotator/dictdef.py
--- a/rpython/annotator/dictdef.py
+++ b/rpython/annotator/dictdef.py
@@ -115,13 +115,5 @@
     def generalize_value(self, s_value):
         self.dictvalue.generalize(s_value)
 
-    def seen_prebuilt_key(self, x):
-        # In case we are an r_dict, we don't ask for the hash ourselves.
-        # Note that if the custom hashing function ends up asking for
-        # the hash of x, then it must use compute_hash() itself, so it
-        # works out.
-        if not self.dictkey.custom_eq_hash:
-            compute_hash(x)
-
     def __repr__(self):
         return '<{%r: %r}>' % (self.dictkey.s_value, self.dictvalue.s_value)
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -610,11 +610,6 @@
     def special_funcptr_for_type(self, TYPE):
         return self.layoutbuilder.special_funcptr_for_type(TYPE)
 
-    def get_hash_offset(self, T):
-        type_id = self.get_type_id(T)
-        assert not self.gcdata.q_is_varsize(type_id)
-        return self.gcdata.q_fixed_size(type_id)
-
     def finish_tables(self):
         group = self.layoutbuilder.close_table()
         log.info("assigned %s typeids" % (len(group.members), ))
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -465,8 +465,14 @@
 
     Note that this can return 0 or -1 too.
 
-    It returns the same number, both before and after translation.
-    Dictionaries don't need to be rehashed after translation.
+    NOTE: It returns a different number before and after translation!
+    Dictionaries will be rehashed when the translated program starts.
+    Be careful about other places that store or depend on a hash value:
+    if such a place can exist before translation, you should add for
+    example a _cleanup_() method to clear this cache during translation.
+
+    (Nowadays we could completely remove compute_hash() and decide that
+    hash(x) is valid RPython instead, at least for the types listed here.)
     """
     if isinstance(x, (str, unicode)):
         return _hash_string(x)

From pypy.commits at gmail.com  Sat Jan 28 12:04:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 09:04:18 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: simplify code
Message-ID: <588ccf12.c3e31c0a.86816.3832@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89824:d6966f4c0ec7
Date: 2017-01-28 18:03 +0100
http://bitbucket.org/pypy/pypy/changeset/d6966f4c0ec7/

Log:	simplify code

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -170,7 +170,6 @@
                             ('subclassrange_max', Signed),
                             ('rtti', Ptr(RuntimeTypeInfo)),
                             ('name', Ptr(rstr.STR)),
-                            ('hash', Signed),
                             ('instantiate', Ptr(FuncType([], OBJECTPTR))),
                             hints={'immutable': True}))
 # non-gc case
@@ -338,7 +337,6 @@
 
     def fill_vtable_root(self, vtable):
         """Initialize the head of the vtable."""
-        vtable.hash = hash(self)
         # initialize the 'subclassrange_*' and 'name' fields
         if self.classdef is not None:
             #vtable.parenttypeptr = self.rbase.getvtable()
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -16,7 +16,7 @@
 from rpython.rtyper.lltypesystem import llmemory
 from rpython.rtyper.lltypesystem.lltype import (
     typeOf, Void, ForwardReference, Struct, Bool, Char, Ptr, malloc, nullptr,
-    Array, Signed, cast_pointer, getfunctionptr)
+    Array, Signed, cast_pointer, getfunctionptr, cast_ptr_to_int)
 from rpython.rtyper.rmodel import (Repr, inputconst, CanBeNull, mangle,
     warning, impossible_repr)
 from rpython.tool.pairtype import pair, pairtype
@@ -1072,10 +1072,7 @@
 
 
 def ll_cls_hash(cls):
-    if not cls:
-        return 0
-    else:
-        return cls.hash
+    return cast_ptr_to_int(cls)
 
 class __extend__(pairtype(ClassesPBCRepr, rclass.ClassRepr)):
     def convert_from_to((r_clspbc, r_cls), v, llops):

From pypy.commits at gmail.com  Sat Jan 28 12:10:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 28 Jan 2017 09:10:19 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Add get_ll_fasthash_function on
 NoneRepr (though that's only for dicts
Message-ID: <588cd07b.d185df0a.ad9ab.a4a6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89825:96df52d20b1b
Date: 2017-01-28 18:09 +0100
http://bitbucket.org/pypy/pypy/changeset/96df52d20b1b/

Log:	Add get_ll_fasthash_function on NoneRepr (though that's only for
	dicts whose only key is None). Add comments elsewhere.

diff --git a/rpython/rtyper/rfloat.py b/rpython/rtyper/rfloat.py
--- a/rpython/rtyper/rfloat.py
+++ b/rpython/rtyper/rfloat.py
@@ -26,6 +26,9 @@
     def get_ll_hash_function(self):
         return _hash_float
 
+    # no get_ll_fasthash_function: the hash is a bit slow, better cache
+    # it inside dict entries
+
     def rtype_bool(_, hop):
         vlist = hop.inputargs(Float)
         return hop.genop('float_is_true', vlist, resulttype=Bool)
diff --git a/rpython/rtyper/rnone.py b/rpython/rtyper/rnone.py
--- a/rpython/rtyper/rnone.py
+++ b/rpython/rtyper/rnone.py
@@ -25,6 +25,8 @@
     def get_ll_hash_function(self):
         return ll_none_hash
 
+    get_ll_fasthash_function = get_ll_hash_function
+
     rtype_simple_call = none_call
     rtype_call_args = none_call
 
diff --git a/rpython/rtyper/rtuple.py b/rpython/rtyper/rtuple.py
--- a/rpython/rtyper/rtuple.py
+++ b/rpython/rtyper/rtuple.py
@@ -208,6 +208,9 @@
     def get_ll_hash_function(self):
         return gen_hash_function(self.items_r)
 
+    # no get_ll_fasthash_function: the hash is a bit slow, better cache
+    # it inside dict entries
+
     ll_str = property(gen_str_function)
 
     def make_iterator_repr(self, variant=None):

From pypy.commits at gmail.com  Sun Jan 29 12:51:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 29 Jan 2017 09:51:01 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Another test (fails for unrelated
 reasons so far)
Message-ID: <588e2b85.10941c0a.3041f.bd6e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89826:0138d2b39b12
Date: 2017-01-28 18:52 +0100
http://bitbucket.org/pypy/pypy/changeset/0138d2b39b12/

Log:	Another test (fails for unrelated reasons so far)

diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -1,6 +1,13 @@
+import sys, os
+
+if __name__ == '__main__':
+    # hack for test_translation_prebuilt_2()
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__),
+                                    '..', '..', '..'))
+
 import py
 from rpython.annotator.model import UnionError
-from rpython.rlib import rgc
+from rpython.rlib import rgc, nonconst
 from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rtyper.test.test_llinterp import interpret
 from rpython.translator.c.test.test_genc import compile
@@ -215,7 +222,7 @@
     f()
     interpret(f, [])
 
-def test_translation_prebuilt():
+def test_translation_prebuilt_1():
     class K:
         pass
     d = RWeakValueDictionary(K, X)
@@ -229,3 +236,25 @@
     f()
     fc = compile(f, [], gcpolicy="boehm", rweakref=True)
     fc()
+
+def _test_translation_prebuilt_2():
+    from rpython.rlib import objectmodel
+    objectmodel.set_hash_algorithm("siphash24")
+    d = RWeakValueDictionary(str, X)
+    k1 = "key1"; k2 = "key2"
+    x1 = X(); x2 = X()
+    d.set(k1, x1)
+    d.set(k2, x2)
+    def f():
+        i = nonconst.NonConstant(1)
+        assert d.get("key%d" % (i,)) is x1
+        assert d.get("key%d" % (i+1,)) is x2
+    fc = compile(f, [], gcpolicy="boehm", rweakref=True)
+    fc()
+
+def test_translation_prebuilt_2():
+    import subprocess
+    subprocess.check_call([sys.executable, __file__])
+
+if __name__ == "__main__":
+    _test_translation_prebuilt_2()

From pypy.commits at gmail.com  Sun Jan 29 12:51:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 29 Jan 2017 09:51:03 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: A mechanism to add a function to
 call at program start-up. This version
Message-ID: <588e2b87.9d711c0a.71d4d.bb8c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89827:7b2e8a4c0b4b
Date: 2017-01-29 17:45 +0100
http://bitbucket.org/pypy/pypy/changeset/7b2e8a4c0b4b/

Log:	A mechanism to add a function to call at program start-up. This
	version uses different trade-offs than call_initial_function() from
	translator/unsimplify.py. Notably, it ensures it is called even if
	the main entry point is not (embedding).

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -513,6 +513,9 @@
     # __________________________________________________________
     # misc LL operation implementations
 
+    def op_call_at_startup(self, *args):
+        pass
+
     def op_debug_view(self, *ll_objects):
         from rpython.translator.tool.lltracker import track
         track(*ll_objects)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -539,6 +539,7 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
+    'call_at_startup':      LLOp(),
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py
--- a/rpython/translator/c/database.py
+++ b/rpython/translator/c/database.py
@@ -60,6 +60,7 @@
         self.completed = False
 
         self.instrument_ncounter = 0
+        self.call_at_startup = set()
 
     def gettypedefnode(self, T, varlength=None):
         if varlength is None:
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -940,3 +940,9 @@
                 cdecl(typename, ''),
                 self.expr(op.args[0]),
                 self.expr(op.result))
+
+    def OP_CALL_AT_STARTUP(self, op):
+        assert isinstance(op.args[0], Constant)
+        func = self.expr(op.args[0])
+        self.db.call_at_startup.add(func)
+        return '/* call_at_startup %s */' % (func,)
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -822,6 +822,9 @@
             for line in lines:
                 print >> f, '\t'+line
 
+    for extra in database.call_at_startup:
+        print >> f, '\t%s();\t/* call_at_startup */' % (extra,)
+
     print >> f, '}'
 
 def commondefs(defines):
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1062,6 +1062,28 @@
         out = cbuilder.cmdexec('')
         assert out.strip() == expected
 
+    def test_call_at_startup(self):
+        from rpython.rtyper.lltypesystem import lltype
+        from rpython.rtyper.lltypesystem.lloperation import llop
+        from rpython.rtyper.annlowlevel import llhelper
+        class State:
+            seen = 0
+        state = State()
+        def startup():
+            state.seen += 1
+        F = lltype.Ptr(lltype.FuncType([], lltype.Void))
+        def entry_point(argv):
+            state.seen += 100
+            assert state.seen == 101
+            print 'ok'
+            ll = llhelper(F, startup)
+            llop.call_at_startup(lltype.Void, ll)
+            return 0
+
+        t, cbuilder = self.compile(entry_point)
+        out = cbuilder.cmdexec('')
+        assert out.strip() == 'ok'
+
 
 class TestMaemo(TestStandalone):
     def setup_class(cls):

From pypy.commits at gmail.com  Sun Jan 29 12:51:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 29 Jan 2017 09:51:07 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Write the exact same logic as
 CPython. Write a test, which doesn't pass
Message-ID: <588e2b8b.c3a4df0a.ee62f.33c7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89829:6df13d671d16
Date: 2017-01-29 18:50 +0100
http://bitbucket.org/pypy/pypy/changeset/6df13d671d16/

Log:	Write the exact same logic as CPython. Write a test, which doesn't
	pass so far.

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -5,17 +5,19 @@
 of siphash-2-4 on all RPython strings and unicodes in your program
 after translation.
 """
-import sys, os
+import sys, os, errno
 from contextlib import contextmanager
 from rpython.rlib import rarithmetic, rurandom
 from rpython.rlib.objectmodel import not_rpython, always_inline
 from rpython.rlib.objectmodel import we_are_translated, dont_inline
-from rpython.rlib import rgc, jit
+from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib import rgc, jit, rposix
 from rpython.rlib.rarithmetic import r_uint64, r_uint32, r_uint
 from rpython.rlib.rawstorage import misaligned_is_fine
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.rtyper.annlowlevel import llhelper
 
 
 if sys.byteorder == 'little':
@@ -47,32 +49,8 @@
 
 
 random_ctx = rurandom.init_urandom()
-
-def lcg_urandom(value):
-    # Quite unsure what the point of this function is, given that a hash
-    # seed of the form '%s\x00\x00\x00..' should be just as hard to
-    # guess as this one.  We copy it anyway from CPython for the case
-    # where 'value' is a 32-bit unsigned number, but if it is not, we
-    # fall back to the '%s\x00\x00\x00..' form.
-    if value == '0':
-        value = ''
-    try:
-        x = r_uint(r_uint32(value))
-    except (ValueError, OverflowError):
-        x = r_uint(0)
-    if str(x) == value:
-        s = ''
-        for index in range(16):
-            x *= 214013
-            x += 2531011
-            x = r_uint(r_uint32(x))
-            s += chr((x >> 16) & 0xff)
-    else:
-        if len(value) < 16:
-            s = value + '\x00' * (16 - len(value))
-        else:
-            s = value[:16]
-    return s
+strtoul = rffi.llexternal("strtoul", [rffi.CCHARP, rffi.CCHARPP, rffi.INT],
+                          rffi.ULONG, save_err=rffi.RFFI_SAVE_ERRNO)
 
 env_var_name = "PYTHONHASHSEED"
 
@@ -83,12 +61,46 @@
     # global variable 'env_var_name', or just pass a different init
     # function to enable_siphash24().
     value = os.environ.get(env_var_name)
-    if len(value) > 0 and value != "random":
-        s = lcg_urandom(value)
+    if value and value != "random":
+        with rffi.scoped_view_charp(value) as ptr:
+            with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as endptr:
+                endptr[0] = ptr
+                seed = strtoul(ptr, endptr, 10)
+                full = endptr[0][0] == '\x00'
+        seed = lltype.cast_primitive(lltype.Unsigned, seed)
+        if not full or seed > r_uint(4294967295) or (
+            rposix.get_saved_errno() == errno.ERANGE and
+            seed == lltype.cast_primitive(lltype.Unsigned,
+                                          rffi.cast(rffi.ULONG, -1))):
+            os.write(2,
+                "PYTHONHASHSEED must be \"random\" or an integer "
+                "in range [0; 4294967295]")
+            os._exit(1)
+        if not seed:
+            # disable the randomized hash
+            s = '\x00' * 16
+        else:
+            s = lcg_urandom(seed)
     else:
-        s = rurandom.urandom(random_ctx, 16)
+        try:
+            s = rurandom.urandom(random_ctx, 16)
+        except Exception as e:
+            os.write(2,
+                "%s: failed to get random numbers to initialize Python\n" %
+                (e.__class__.__name__,))
+            os._exit(1)
+            raise   # makes the annotator happy
     select_random_seed(s)
 
+def lcg_urandom(x):
+    s = ''
+    for index in range(16):
+        x *= 214013
+        x += 2531011
+        s += chr((x >> 16) & 0xff)
+    return s
+
+
 _FUNC = lltype.Ptr(lltype.FuncType([], lltype.Void))
 
 def enable_siphash24(*init):
@@ -104,7 +116,7 @@
         (init_func,) = init
     else:
         init_func = initialize_from_env
-    llop.call_at_startup(lltype.Void, llexternal(_FUNC, init_func))
+    llop.call_at_startup(lltype.Void, llhelper(_FUNC, init_func))
 
 def _internal_enable_siphash24():
     pass
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
--- a/rpython/rlib/test/test_rsiphash.py
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -1,7 +1,10 @@
 import os
 from rpython.rlib.rsiphash import siphash24, _siphash24, choosen_seed
-from rpython.rlib.rsiphash import initialize_from_env, seed
+from rpython.rlib.rsiphash import initialize_from_env, enable_siphash24
+from rpython.rlib.objectmodel import compute_hash
+from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.lltypesystem import llmemory, rffi
+from rpython.translator.c.test.test_genc import compile
 
 
 CASES = [
@@ -51,9 +54,9 @@
         assert siphash24("foo") == 15988776847138518036
         # value checked with CPython 3.5
 
-        os.environ['PYTHONHASHSEED'] = '123'
+        os.environ['PYTHONHASHSEED'] = '4000000000'
         initialize_from_env()
-        assert siphash24("foo") == 12577370453467666022
+        assert siphash24("foo") == 13829150778707464258
         # value checked with CPython 3.5
 
         for env in ['', 'random']:
@@ -68,3 +71,44 @@
             del os.environ['PYTHONHASHSEED']
         else:
             os.environ['PYTHONHASHSEED'] = old_val
+
+def test_translated():
+    d1 = {"foo": 123}
+    d2 = {u"foo": 456, u"\u1234": 789}
+
+    def entrypoint():
+        enable_siphash24()
+        return '%d %d %d %d %d %d' % (
+            d1.get("foo", -1),     compute_hash("bar"),
+            d2.get(u"foo", -1),    compute_hash(u"foo"),
+            d2.get(u"\u1234", -1), compute_hash(u"\u1234"))
+
+    fn = compile(entrypoint, [])
+
+    old_val = os.environ.get('PYTHONHASHSEED', None)
+    try:
+        os.environ['PYTHONHASHSEED'] = '0'
+        s1 = fn()
+        assert map(int, s1.split()) == [
+            123, intmask(15988776847138518036),
+            456, intmask(15988776847138518036),
+            789, intmask(16003099094427356855)]
+
+        os.environ['PYTHONHASHSEED'] = '3987654321'
+        s1 = fn()
+        assert map(int, s1.split()) == [
+            123, intmask(5890804383681474441),
+            456, intmask(5890804383681474441),
+            789, intmask(10331001347733193222)]
+
+        for env in ['', 'random']:
+            os.environ['PYTHONHASHSEED'] = env
+            s1 = fn()
+            s2 = fn()
+            assert s1 != s2
+
+    finally:
+        if old_val is None:
+            del os.environ['PYTHONHASHSEED']
+        else:
+            os.environ['PYTHONHASHSEED'] = old_val

From pypy.commits at gmail.com  Sun Jan 29 12:51:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 29 Jan 2017 09:51:05 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: in-progress: generate the random
 seed at runtime
Message-ID: <588e2b89.48301c0a.71c9c.baa0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89828:746716a29381
Date: 2017-01-29 18:10 +0100
http://bitbucket.org/pypy/pypy/changeset/746716a29381/

Log:	in-progress: generate the random seed at runtime

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -519,21 +519,11 @@
 
 # ----------
 
-HASH_ALGORITHM = "rpython"    # the default, no source of randomness possible
-HASH_ALGORITHM_FIXED = False
-
- at not_rpython
-def set_hash_algorithm(algo):
-    """Must be called very early, before any string is hashed with
-    compute_hash()!"""
-    global HASH_ALGORITHM
-    if HASH_ALGORITHM != algo:
-        assert not HASH_ALGORITHM_FIXED, "compute_hash() already called!"
-        assert algo in ("rpython", "siphash24")
-        HASH_ALGORITHM = algo
-
-
-def _hash_string_rpython(s):
+def _hash_string(s):
+    """The default algorithm behind compute_hash() for a string or a unicode.
+    There is a mechanism to use another one in programs after translation.
+    See rsiphash.py, which implements the algorithm of CPython >= 3.4.
+    """
     from rpython.rlib.rarithmetic import intmask
 
     length = len(s)
@@ -547,100 +537,8 @@
     x ^= length
     return intmask(x)
 
-
- at not_rpython
-def _hash_string_siphash24(s):
-    """This version is called when untranslated only."""
-    import array
-    from rpython.rlib.rsiphash import siphash24
-    from rpython.rtyper.lltypesystem import lltype, rffi
-    from rpython.rlib.rarithmetic import intmask
-
-    if not isinstance(s, str):
-        if isinstance(s, unicode):
-            lst = map(ord, s)
-        else:
-            lst = map(ord, s.chars)    # for rstr.STR or UNICODE
-        # NOTE: a latin-1 unicode string must have the same hash as the
-        # corresponding byte string.
-        if all(n <= 0xFF for n in lst):
-            kind = "B"
-        elif rffi.sizeof(lltype.UniChar) == 4:
-            kind = "I"
-        else:
-            kind = "H"
-        s = array.array(kind, lst).tostring()
-    ptr = rffi.str2charp(s)
-    x = siphash24(ptr, len(s))
-    rffi.free_charp(ptr)
-    return intmask(x)
-
-def ll_hash_string_siphash24(ll_s):
-    """Called from lltypesystem/rstr.py.  'll_s' is a rstr.STR or UNICODE."""
-    from rpython.rlib.rsiphash import siphash24
-    from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
-    from rpython.rlib.rarithmetic import intmask
-
-    length = len(ll_s.chars)
-    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:
-        # no GC operation from here!
-        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
-    else:
-        # NOTE: a latin-1 unicode string must have the same hash as the
-        # corresponding byte string.  If the unicode is all within
-        # 0-255, then we need to allocate a byte buffer and copy the
-        # latin-1 encoding in it manually.
-        for i in range(length):
-            if ord(ll_s.chars[i]) > 0xFF:
-                # no GC operation from here!
-                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
-                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
-                break
-        else:
-            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
-            i = 0
-            while i < length:
-                p[i] = chr(ord(ll_s.chars[i]))
-                i += 1
-            x = siphash24(llmemory.cast_ptr_to_adr(p), length)
-            lltype.free(p, flavor='raw')
-            return intmask(x)
-    x = siphash24(addr, length)
-    keepalive_until_here(ll_s)
-    return intmask(x)
-ll_hash_string_siphash24._jit_look_inside_ = False
-
-
- at not_rpython
-def _hash_string(s):
-    """The algorithm behind compute_hash() for a string or a unicode.
-    This version is only for untranslated usage, and 's' is a str or unicode.
-    """
-    global HASH_ALGORITHM_FIXED
-    HASH_ALGORITHM_FIXED = True
-    if HASH_ALGORITHM == "rpython":
-        return _hash_string_rpython(s)
-    if HASH_ALGORITHM == "siphash24":
-        return _hash_string_siphash24(s)
-    raise NotImplementedError
-
 def ll_hash_string(ll_s):
-    """The algorithm behind compute_hash() for a string or a unicode.
-    This version is called from lltypesystem/rstr.py, and 'll_s' is a
-    rstr.STR or rstr.UNICODE.
-    """
-    if not we_are_translated():
-        global HASH_ALGORITHM_FIXED
-        HASH_ALGORITHM_FIXED = True
-    if HASH_ALGORITHM == "rpython":
-        return _hash_string_rpython(ll_s.chars)
-    if HASH_ALGORITHM == "siphash24":
-        if we_are_translated():
-            return ll_hash_string_siphash24(ll_s)
-        else:
-            return _hash_string_siphash24(ll_s)
-    raise NotImplementedError
-
+    return _hash_string(ll_s.chars)
 
 def _hash_float(f):
     """The algorithm behind compute_hash() for a float.
@@ -698,6 +596,21 @@
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
+    _about_ = ll_hash_string
+    # this is only used when annotating the code in rstr.py, and so
+    # it always occurs after the RPython program signalled its intent
+    # to use a different hash.  The code below overwrites the use of
+    # ll_hash_string() to make the annotator think a possibly different
+    # function was called.
+
+    def compute_annotation(self):
+        from rpython.annotator import model as annmodel
+        bk = self.bookkeeper
+        translator = bk.annotator.translator
+        fn = getattr(translator, 'll_hash_string', ll_hash_string)
+        return annmodel.SomePBC([bk.getdesc(fn)])
+
+class Entry(ExtRegistryEntry):
     _about_ = compute_identity_hash
 
     def compute_result_annotation(self, s_x):
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -1,13 +1,21 @@
+"""
+This module implements siphash-2-4, the hashing algorithm for strings
+and unicodes.  You can use it explicitly by calling siphash24() with
+a byte string, or you can use enable_siphash24() to enable the use
+of siphash-2-4 on all RPython strings and unicodes in your program
+after translation.
+"""
 import sys, os
 from contextlib import contextmanager
 from rpython.rlib import rarithmetic, rurandom
 from rpython.rlib.objectmodel import not_rpython, always_inline
 from rpython.rlib.objectmodel import we_are_translated, dont_inline
-from rpython.rlib.rgc import no_collect
+from rpython.rlib import rgc, jit
 from rpython.rlib.rarithmetic import r_uint64, r_uint32, r_uint
 from rpython.rlib.rawstorage import misaligned_is_fine
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.extregistry import ExtRegistryEntry
 
 
 if sys.byteorder == 'little':
@@ -19,18 +27,23 @@
 
 class Seed:
     k0l = k1l = r_uint64(0)
-    initialized = False
 seed = Seed()
 
 
+def _decode64(s):
+    return (r_uint64(ord(s[0])) |
+            r_uint64(ord(s[1])) << 8 |
+            r_uint64(ord(s[2])) << 16 |
+            r_uint64(ord(s[3])) << 24 |
+            r_uint64(ord(s[4])) << 32 |
+            r_uint64(ord(s[5])) << 40 |
+            r_uint64(ord(s[6])) << 48 |
+            r_uint64(ord(s[7])) << 56)
+
 def select_random_seed(s):
     """'s' is a string of length 16"""
-    seed.k0l = (
-      ord(s[0]) | ord(s[1]) << 8 | ord(s[2]) << 16 | ord(s[3]) << 24 |
-      ord(s[4]) << 32 | ord(s[5]) << 40 | ord(s[6]) << 48 | ord(s[7]) << 56)
-    seed.k1l = (
-      ord(s[8]) | ord(s[9]) << 8 | ord(s[10]) << 16 | ord(s[11]) << 24 |
-      ord(s[12]) << 32 | ord(s[13]) << 40 | ord(s[14]) << 48 | ord(s[15]) << 56)
+    seed.k0l = _decode64(s)
+    seed.k1l = _decode64(s[8:16])
 
 
 random_ctx = rurandom.init_urandom()
@@ -63,20 +76,85 @@
 
 env_var_name = "PYTHONHASHSEED"
 
- at dont_inline
 def initialize_from_env():
     # This uses the same algorithms as CPython 3.5.  The environment
     # variable we read also defaults to "PYTHONHASHSEED".  If needed,
     # a different RPython interpreter can patch the value of the
-    # global variable 'env_var_name', or completely patch this function
-    # with a different one.
+    # global variable 'env_var_name', or just pass a different init
+    # function to enable_siphash24().
     value = os.environ.get(env_var_name)
     if len(value) > 0 and value != "random":
         s = lcg_urandom(value)
     else:
         s = rurandom.urandom(random_ctx, 16)
     select_random_seed(s)
-    seed.initialized = True
+
+_FUNC = lltype.Ptr(lltype.FuncType([], lltype.Void))
+
+def enable_siphash24(*init):
+    """
+    Enable the use of siphash-2-4 for all RPython strings and unicodes
+    in the translated program.  You must call this function anywhere
+    from your interpreter (from a place that is annotated).  Optionally,
+    you can pass a function to call to initialize the state; the default
+    is 'initialize_from_env' above.  Don't call this more than once.
+    """
+    _internal_enable_siphash24()
+    if init:
+        (init_func,) = init
+    else:
+        init_func = initialize_from_env
+    llop.call_at_startup(lltype.Void, llexternal(_FUNC, init_func))
+
+def _internal_enable_siphash24():
+    pass
+
+class Entry(ExtRegistryEntry):
+    _about_ = _internal_enable_siphash24
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        if hasattr(translator, 'll_hash_string'):
+            assert translator.ll_hash_string == ll_hash_string_siphash24
+        else:
+            translator.ll_hash_string = ll_hash_string_siphash24
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+
+ at rgc.no_collect
+def ll_hash_string_siphash24(ll_s):
+    """Called indirectly from lltypesystem/rstr.py, by redirection from
+    objectmodel.ll_string_hash().
+    """
+    from rpython.rlib.rarithmetic import intmask
+
+    # This function is entirely @rgc.no_collect.
+    length = len(ll_s.chars)
+    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:   # regular STR
+        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
+    else:
+        # NOTE: a latin-1 unicode string must have the same hash as the
+        # corresponding byte string.  If the unicode is all within
+        # 0-255, then we need to allocate a byte buffer and copy the
+        # latin-1 encoding in it manually.
+        for i in range(length):
+            if ord(ll_s.chars[i]) > 0xFF:
+                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
+                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
+                break
+        else:
+            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
+            i = 0
+            while i < length:
+                p[i] = chr(ord(ll_s.chars[i]))
+                i += 1
+            x = _siphash24(llmemory.cast_ptr_to_adr(p), length)
+            lltype.free(p, flavor='raw')
+            return intmask(x)
+    x = _siphash24(addr, length)
+    keepalive_until_here(ll_s)
+    return intmask(x)
 
 
 @contextmanager
@@ -123,13 +201,11 @@
     return v0, v1, v2, v3
 
 
- at no_collect
-def siphash24(addr_in, size):
+ at rgc.no_collect
+def _siphash24(addr_in, size):
     """Takes an address pointer and a size.  Returns the hash as a r_uint64,
     which can then be casted to the expected type."""
 
-    if we_are_translated() and not seed.initialized:
-        initialize_from_env()
     k0 = seed.k0l
     k1 = seed.k1l
     b = r_uint64(size) << 56
@@ -206,3 +282,13 @@
     v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
 
     return (v0 ^ v1) ^ (v2 ^ v3)
+
+
+ at jit.dont_look_inside
+def siphash24(s):
+    """'s' is a normal string.  Returns its siphash-2-4 as a r_uint64.
+    Don't forget to cast the result to a regular integer if needed,
+    e.g. with rarithmetic.intmask().
+    """
+    with rffi.scoped_nonmovingbuffer(s) as p:
+        return _siphash24(llmemory.cast_ptr_to_adr(p), len(s))
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
--- a/rpython/rlib/test/test_rsiphash.py
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -1,5 +1,5 @@
 import os
-from rpython.rlib.rsiphash import siphash24, choosen_seed
+from rpython.rlib.rsiphash import siphash24, _siphash24, choosen_seed
 from rpython.rlib.rsiphash import initialize_from_env, seed
 from rpython.rtyper.lltypesystem import llmemory, rffi
 
@@ -30,13 +30,11 @@
 ]
 
 def check(s):
-    p = rffi.str2charp(s)
     q = rffi.str2charp('?' + s)
     with choosen_seed(0x8a9f065a358479f4, 0x11cb1e9ee7f40e1f,
                       test_misaligned_path=True):
-        x = siphash24(llmemory.cast_ptr_to_adr(p), len(s))
-        y = siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
-    rffi.free_charp(p)
+        x = siphash24(s)
+        y = _siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
     rffi.free_charp(q)
     assert x == y
     return x
@@ -46,22 +44,27 @@
         assert check(string) == expected
 
 def test_fix_seed():
-    p = rffi.str2charp("foo")
-    adr = llmemory.cast_ptr_to_adr(p)
+    old_val = os.environ.get('PYTHONHASHSEED', None)
+    try:
+        os.environ['PYTHONHASHSEED'] = '0'
+        initialize_from_env()
+        assert siphash24("foo") == 15988776847138518036
+        # value checked with CPython 3.5
 
-    os.environ['PYTHONHASHSEED'] = '0'
-    initialize_from_env()
-    assert siphash24(adr, 3) == 15988776847138518036 # checked with CPython 3.5
+        os.environ['PYTHONHASHSEED'] = '123'
+        initialize_from_env()
+        assert siphash24("foo") == 12577370453467666022
+        # value checked with CPython 3.5
 
-    os.environ['PYTHONHASHSEED'] = '123'
-    initialize_from_env()
-    assert siphash24(adr, 3) == 12577370453467666022 # checked with CPython 3.5
-
-    os.environ['PYTHONHASHSEED'] = 'random'
-    initialize_from_env()
-    hash1 = siphash24(adr, 3)
-    initialize_from_env()
-    hash2 = siphash24(adr, 3)
-    assert hash1 != hash2
-
-    rffi.free_charp(p)
+        for env in ['', 'random']:
+            os.environ['PYTHONHASHSEED'] = env
+            initialize_from_env()
+            hash1 = siphash24("foo")
+            initialize_from_env()
+            hash2 = siphash24("foo")
+            assert hash1 != hash2     # extremely unlikely
+    finally:
+        if old_val is None:
+            del os.environ['PYTHONHASHSEED']
+        else:
+            os.environ['PYTHONHASHSEED'] = old_val
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -829,7 +829,7 @@
         return assert_str0(charpsize2str(cp, size))
     charp2str._annenforceargs_ = [lltype.SomePtr(TYPEP)]
 
-    # str -> char*, bool, bool
+    # str -> char*, flag
     # Can't inline this because of the raw address manipulation.
     @jit.dont_look_inside
     def get_nonmovingbuffer(data):
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -169,10 +169,7 @@
             for i in range(len(value)):
                 p.chars[i] = cast_primitive(self.base, value[i])
             p.hash = 0
-            if objectmodel.HASH_ALGORITHM == "rpython":
-                self.ll.ll_strhash(p)   # precompute the hash
-                # but it is pointless if this hash wouldn't end up in the
-                # C code anyway: see "remove_hash" in translator/c/node.py
+            self.ll.ll_strhash(p)   # precompute the hash
             self.CACHE[value] = p
             return p
 
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -586,8 +586,10 @@
             data.append((name, getattr(self.obj, name)))
 
         if T._hints.get('remove_hash'):
-            # hack for rstr.STR and UNICODE
-            if objectmodel.HASH_ALGORITHM != "rpython":
+            # hack for rstr.STR and UNICODE: remove their .hash value
+            # and write 0 in the C sources, if we're using a non-default
+            # hash function.
+            if hasattr(self.db.translator, 'll_hash_string'):
                 i = 0
                 while data[i][0] != 'hash':
                     i += 1

From pypy.commits at gmail.com  Sun Jan 29 16:52:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 29 Jan 2017 13:52:04 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: fix a bug found by the test (and
 also fix and improve the test itself)
Message-ID: <588e6404.545a1c0a.b4ae8.15ad@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89830:bf4314421560
Date: 2017-01-29 22:51 +0100
http://bitbucket.org/pypy/pypy/changeset/bf4314421560/

Log:	fix a bug found by the test (and also fix and improve the test
	itself)

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -74,7 +74,7 @@
                                           rffi.cast(rffi.ULONG, -1))):
             os.write(2,
                 "PYTHONHASHSEED must be \"random\" or an integer "
-                "in range [0; 4294967295]")
+                "in range [0; 4294967295]\n")
             os._exit(1)
         if not seed:
             # disable the randomized hash
@@ -149,7 +149,9 @@
         # NOTE: a latin-1 unicode string must have the same hash as the
         # corresponding byte string.  If the unicode is all within
         # 0-255, then we need to allocate a byte buffer and copy the
-        # latin-1 encoding in it manually.
+        # latin-1 encoding in it manually.  Note also that we give a
+        # different hash result than CPython on ucs4 platforms, for
+        # unicode strings where CPython uses 2 bytes per character.
         for i in range(length):
             if ord(ll_s.chars[i]) > 0xFF:
                 addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
--- a/rpython/rlib/test/test_rsiphash.py
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -74,38 +74,72 @@
 
 def test_translated():
     d1 = {"foo": 123}
-    d2 = {u"foo": 456, u"\u1234": 789}
+    d2 = {u"foo": 456, u"\u1234\u5678": 789}
+    class G:
+        pass
+    g = G()
+    g.v1 = d1.copy()
+    g.v2 = d2.copy()
 
-    def entrypoint():
+    def fetch(n):
+        if n == 0: return d1.get("foo", -1)
+        if n == 1: return g.v1.get("foo", -1)
+        if n == 2: return compute_hash("foo")
+        if n == 3: return d2.get(u"foo", -1)
+        if n == 4: return g.v2.get(u"foo", -1)
+        if n == 5: return compute_hash(u"foo")
+        if n == 6: return d2.get(u"\u1234\u5678", -1)
+        if n == 7: return g.v2.get(u"\u1234\u5678", -1)
+        if n == 8: return compute_hash(u"\u1234\u5678")
+        assert 0
+
+    def entrypoint(n):
         enable_siphash24()
-        return '%d %d %d %d %d %d' % (
-            d1.get("foo", -1),     compute_hash("bar"),
-            d2.get(u"foo", -1),    compute_hash(u"foo"),
-            d2.get(u"\u1234", -1), compute_hash(u"\u1234"))
+        g.v1["bar"] = -2
+        g.v2[u"bar"] = -2
+        if n >= 0:    # get items one by one, because otherwise it may
+                      # be the case that one line influences the next
+            return str(fetch(n))
+        else:
+            # ...except in random mode, because we want all results
+            # to be computed with the same seed
+            return ' '.join([str(fetch(n)) for n in range(9)])
 
-    fn = compile(entrypoint, [])
+    fn = compile(entrypoint, [int])
+
+    def getall():
+        return [int(fn(i)) for i in range(9)]
 
     old_val = os.environ.get('PYTHONHASHSEED', None)
     try:
         os.environ['PYTHONHASHSEED'] = '0'
-        s1 = fn()
-        assert map(int, s1.split()) == [
-            123, intmask(15988776847138518036),
-            456, intmask(15988776847138518036),
-            789, intmask(16003099094427356855)]
+        s1 = getall()
+        assert s1[:8] == [
+            123, 123, intmask(15988776847138518036),
+            456, 456, intmask(15988776847138518036),
+            789, 789]
+        assert s1[8] in [intmask(17593683438421985039),    # ucs2 mode
+                         intmask(94801584261658677)]       # ucs4 mode
 
         os.environ['PYTHONHASHSEED'] = '3987654321'
-        s1 = fn()
-        assert map(int, s1.split()) == [
-            123, intmask(5890804383681474441),
-            456, intmask(5890804383681474441),
-            789, intmask(10331001347733193222)]
+        s1 = getall()
+        assert s1[:8] == [
+            123, 123, intmask(5890804383681474441),
+            456, 456, intmask(5890804383681474441),
+            789, 789]
+        assert s1[8] in [intmask(4192582507672183374),     # ucs2 mode
+                         intmask(7179255293164649778)]     # ucs4 mode
 
         for env in ['', 'random']:
             os.environ['PYTHONHASHSEED'] = env
-            s1 = fn()
-            s2 = fn()
-            assert s1 != s2
+            s1 = map(int, fn(-1).split())
+            s2 = map(int, fn(-1).split())
+            assert s1[0:2]+s1[3:5]+s1[6:8] == [123, 123, 456, 456, 789, 789]
+            assert s1[2] == s1[5]
+            assert s2[0:2]+s2[3:5]+s2[6:8] == [123, 123, 456, 456, 789, 789]
+            assert s2[2] == s2[5]
+            #
+            assert len(set([s1[2], s2[2], s1[8], s2[8]])) == 4
 
     finally:
         if old_val is None:
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -888,13 +888,18 @@
     assert d.num_live_items == d.num_ever_used_items
     assert not d.indexes
     #
-    # recompute all hashes, if they are stored in d.entries
+    # recompute all hashes.  Needed if they are stored in d.entries,
+    # but do it anyway: otherwise, e.g. a string-keyed dictionary
+    # won't have a fasthash on its strings if their hash is still
+    # uncomputed.
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    if hasattr(ENTRY, 'f_hash'):
-        for i in range(d.num_ever_used_items):
-            assert d.entries.valid(i)
-            d_entry = d.entries[i]
-            d_entry.f_hash = d.keyhash(d_entry.key)
+    for i in range(d.num_ever_used_items):
+        assert d.entries.valid(i)
+        d_entry = d.entries[i]
+        h = d.keyhash(d_entry.key)
+        if hasattr(ENTRY, 'f_hash'):
+            d_entry.f_hash = h
+        #else: purely for the side-effect it can have on d_entry.key
     #
     # Use the smallest acceptable size for ll_dict_reindex
     new_size = DICT_INITSIZE
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -3,7 +3,7 @@
 from rpython.annotator import model as annmodel
 from rpython.rlib import jit, types, objectmodel
 from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated,
-    ll_hash_string, keepalive_until_here, specialize, enforceargs)
+    ll_hash_string, keepalive_until_here, specialize, enforceargs, dont_inline)
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.error import TyperError
@@ -383,6 +383,7 @@
             return 0
 
     @staticmethod
+    @dont_inline
     def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the

From pypy.commits at gmail.com  Mon Jan 30 05:30:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 30 Jan 2017 02:30:42 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: make --hash a PyPy-specific option, fixes
Message-ID: <588f15d2.e4361c0a.ac9ab.e55b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89831:dcfe8dc6240f
Date: 2017-01-30 11:25 +0100
http://bitbucket.org/pypy/pypy/changeset/dcfe8dc6240f/

Log:	make --hash a PyPy-specific option, fixes

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -196,6 +196,13 @@
                default=False,
                requires=[("objspace.usemodules.cpyext", False)]),
 
+    ChoiceOption("hash",
+                 "The hash function to use for strings: fnv from CPython 2.7"
+                 " or siphash24 from CPython >= 3.4",
+                 ["fnv", "siphash24"],
+                 default="fnv",
+                 cmdline="--hash"),
+
     OptionDescription("std", "Standard Object Space Options", [
         BoolOption("withtproxy", "support transparent proxies",
                    default=True),
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -35,6 +35,7 @@
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
         w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+        hashfunc = space.config.objspace.hash
     else:
         w_initstdio = space.appexec([], """():
             return lambda unbuffered: None
@@ -45,6 +46,10 @@
             from rpython.jit.backend.hlinfo import highleveljitinfo
             highleveljitinfo.sys_executable = argv[0]
 
+        if hashfunc == "siphash24":
+            from rpython.rlib import rsiphash
+            rsiphash.enable_siphash24()
+
         #debug("entry point starting")
         #for arg in argv:
         #    debug(" argv -> " + arg)
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -201,10 +201,6 @@
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
-    ChoiceOption("hash",
-                 "The hash to use for strings",
-                 ["rpython", "siphash24"],
-                 default="rpython", cmdline="--hash"),
 
     OptionDescription("backendopt", "Backend Optimization Options", [
         # control inlining
@@ -394,12 +390,6 @@
         if sys.platform == "darwin" or sys.platform =="win32":
             raise ConfigError("'asmgcc' not supported on this platform")
 
-def apply_extra_settings(config):
-    # make the setting of config.hash definitive
-    from rpython.rlib.objectmodel import set_hash_algorithm
-    config.translation.hash = config.translation.hash
-    set_hash_algorithm(config.translation.hash)
-
 # ----------------------------------------------------------------
 
 def set_platform(config):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -521,6 +521,12 @@
 
 def _hash_string(s):
     """The default algorithm behind compute_hash() for a string or a unicode.
+    This is a modified Fowler-Noll-Vo (FNV) hash.  According to Wikipedia,
+    FNV needs carefully-computed constants called FNV primes and FNV offset
+    basis, which are absent from the present algorithm.  Nevertheless,
+    this matches CPython 2.7 without -R, which has proven a good hash in
+    practice (even if not crypographical nor randomizable).
+
     There is a mechanism to use another one in programs after translation.
     See rsiphash.py, which implements the algorithm of CPython >= 3.4.
     """
diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -11,8 +11,7 @@
 from rpython.config.config import (to_optparse, OptionDescription, BoolOption,
     ArbitraryOption, StrOption, IntOption, Config, ChoiceOption, OptHelpFormatter)
 from rpython.config.translationoption import (get_combined_translation_config,
-    set_opt_level, OPT_LEVELS, DEFAULT_OPT_LEVEL, set_platform, CACHE_DIR,
-    apply_extra_settings)
+    set_opt_level, OPT_LEVELS, DEFAULT_OPT_LEVEL, set_platform, CACHE_DIR)
 
 # clean up early rpython/_cache
 try:
@@ -178,9 +177,6 @@
     if 'handle_config' in targetspec_dic:
         targetspec_dic['handle_config'](config, translateconfig)
 
-    # apply extra settings
-    apply_extra_settings(config)
-
     return targetspec_dic, translateconfig, config, args
 
 def show_help(translateconfig, opt_parser, targetspec_dic, config):

From pypy.commits at gmail.com  Mon Jan 30 07:56:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 30 Jan 2017 04:56:41 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: obscure translation issues
Message-ID: <588f3809.85b2df0a.1da10.a2f8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89832:daf38d35f189
Date: 2017-01-30 13:56 +0100
http://bitbucket.org/pypy/pypy/changeset/daf38d35f189/

Log:	obscure translation issues

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1324,6 +1324,12 @@
         raise wrap_oserror(space, e)
     return space.wrap(res)
 
+class SigCheck:
+    pass
+_sigcheck = SigCheck()
+def _signal_checker():
+    _sigcheck.space.getexecutioncontext().checksignals()
+
 @unwrap_spec(n=int)
 def urandom(space, n):
     """urandom(n) -> str
@@ -1331,9 +1337,12 @@
     Return a string of n random bytes suitable for cryptographic use.
     """
     context = get(space).random_context
-    signal_checker = space.getexecutioncontext().checksignals
     try:
-        return space.wrap(rurandom.urandom(context, n, signal_checker))
+        # urandom() takes a final argument that should be a regular function,
+        # not a bound method like 'getexecutioncontext().checksignals'.
+        # Otherwise, we can't use it from several independent places.
+        _sigcheck.space = space
+        return space.wrap(rurandom.urandom(context, n, _signal_checker))
     except OSError as e:
         raise wrap_oserror(space, e)
 
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -441,7 +441,7 @@
             except OSError as e:
                 os.write(2, "Could not start GDB: %s" % (
                     os.strerror(e.errno)))
-                raise SystemExit
+                os._exit(1)
         else:
             time.sleep(1)  # give the GDB time to attach
 
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -14,6 +14,7 @@
 from rpython.rlib import rgc, jit, rposix
 from rpython.rlib.rarithmetic import r_uint64, r_uint32, r_uint
 from rpython.rlib.rawstorage import misaligned_is_fine
+from rpython.rlib.nonconst import NonConstant
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -116,6 +117,8 @@
         (init_func,) = init
     else:
         init_func = initialize_from_env
+    if NonConstant(0):
+        init_func()   # pre-annotate it
     llop.call_at_startup(lltype.Void, llhelper(_FUNC, init_func))
 
 def _internal_enable_siphash24():

From pypy.commits at gmail.com  Mon Jan 30 09:39:07 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Mon, 30 Jan 2017 06:39:07 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: Patch
 hack_for_cffi_modules to call dependent tasks
Message-ID: <588f500b.496f1c0a.52e7f.5162@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89833:c3390fd8ebf4
Date: 2017-01-31 01:38 +1100
http://bitbucket.org/pypy/pypy/changeset/c3390fd8ebf4/

Log:	Patch hack_for_cffi_modules to call dependent tasks

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -330,12 +330,9 @@
         # ugly hack to modify target goal from compile_* to build_cffi_imports
         # this should probably get cleaned up and merged with driver.create_exe
         from rpython.tool.runsubprocess import run_subprocess
-        from rpython.translator.driver import taskdef
-        import types
 
         compile_goal, = driver.backend_select_goals(['compile'])
-        @taskdef([compile_goal], "Create cffi bindings for modules")
-        def task_build_cffi_imports(self):
+        def task_build_cffi_imports():
             ''' Use cffi to compile cffi interfaces to modules'''
             filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py')
             status, out, err = run_subprocess(str(driver.compute_exe_name()),
@@ -343,8 +340,13 @@
             sys.stdout.write(out)
             sys.stderr.write(err)
             # otherwise, ignore errors
-        driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
-        driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, [compile_goal]
+
+        def build_cffi_imports():
+            getattr(driver, compile_goal)()
+            driver.run_task(task_build_cffi_imports, 'build_cffi_imports')
+
+        driver.task_build_cffi_imports = task_build_cffi_imports
+        driver.build_cffi_imports = build_cffi_imports
         driver.default_goal = 'build_cffi_imports'
         # HACKHACKHACK end
 

From pypy.commits at gmail.com  Mon Jan 30 10:07:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 30 Jan 2017 07:07:24 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: fix comments
Message-ID: <588f56ac.5190df0a.5d65e.d47a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89834:6bc7be33e2c1
Date: 2017-01-30 16:06 +0100
http://bitbucket.org/pypy/pypy/changeset/6bc7be33e2c1/

Log:	fix comments

diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py
--- a/rpython/translator/c/test/test_boehm.py
+++ b/rpython/translator/c/test/test_boehm.py
@@ -392,7 +392,7 @@
         assert res[2] != compute_hash(c)     # likely
         assert res[3] != compute_hash(d)     # likely *not* preserved
         assert res[4] == compute_hash(("Hi", None, (7.5, 2)))
-        # ^^ true as long as we're using the 'rpython' hash for strings
+        # ^^ true as long as we're using the default 'fnv' hash for strings
         #    and not e.g. siphash24
 
     def test_finalizer_queue(self):
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -600,7 +600,7 @@
         assert res[2] != compute_hash(c)     # likely
         assert res[3] != compute_hash(d)     # likely *not* preserved
         assert res[4] == compute_hash(("Hi", None, (7.5, 2)))
-        # ^^ true as long as we're using the 'rpython' hash for strings
+        # ^^ true as long as we're using the default 'fnv' hash for strings
         #    and not e.g. siphash24
 
     def _test_hash_string(self, algo):

From pypy.commits at gmail.com  Mon Jan 30 10:07:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 30 Jan 2017 07:07:26 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: hg merge default
Message-ID: <588f56ae.eb88df0a.0977.d86b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89835:59ed8e907ea8
Date: 2017-01-30 16:06 +0100
http://bitbucket.org/pypy/pypy/changeset/59ed8e907ea8/

Log:	hg merge default

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1243,6 +1243,9 @@
     struct = SERVERNAME_CALLBACKS.get(rffi.cast(lltype.Signed, arg))
     w_ctx = struct.w_ctx
     space = struct.space
+    # annotation: ensures that we ignore the case 'space is None'
+    # (it would propagate to a few methods like errorstr())
+    assert w_ctx is not None and space is not None
     w_callback = struct.w_set_hostname
     if not w_ctx.servername_callback:
         # Possible race condition.
diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+from itertools import izip
 from . import cmodel as model
 from .commontypes import COMMON_TYPES, resolve_common_type
 from .error import FFIError, CDefError
@@ -103,7 +104,7 @@
 class Parser(object):
 
     def __init__(self):
-        self._declarations = {}
+        self._declarations = OrderedDict()
         self._included_declarations = set()
         self._anonymous_counter = 0
         self._structnode2type = weakref.WeakKeyDictionary()
@@ -699,8 +700,7 @@
         self.headers = headers if headers is not None else ['sys/types.h']
         self.parsed_headers = []
         self.sources = []
-        self._Config = type('Config', (object,), {})
-        self._TYPES = {}
+        self._config_entries = OrderedDict()
         self.includes = []
         self.struct_typedefs = {}
         self._handled = set()
@@ -758,10 +758,8 @@
 
     def realize_struct(self, struct):
         type_name = struct.get_type_name()
-        configname = type_name.replace(' ', '__')
-        setattr(self._Config, configname,
-            rffi_platform.Struct(type_name, struct.fields))
-        self._TYPES[configname] = struct.TYPE
+        entry = rffi_platform.Struct(type_name, struct.fields)
+        self._config_entries[entry] = struct.TYPE
         return struct.TYPE
 
     def build_eci(self):
@@ -795,13 +793,15 @@
             elif name.startswith('macro '):
                 name = name[6:]
                 self.add_macro(name, obj)
-        self._Config._compilation_info_ = self.build_eci()
-        for name, TYPE in rffi_platform.configure(self._Config).iteritems():
+        if not self._config_entries:
+            return
+        eci = self.build_eci()
+        result = rffi_platform.configure_entries(list(self._config_entries), eci)
+        for entry, TYPE in izip(self._config_entries, result):
             # hack: prevent the source from being pasted into common_header.h
             del TYPE._hints['eci']
-            if name in self._TYPES:
-                self._TYPES[name].become(TYPE)
-                del self._TYPES[name]
+            self._config_entries[entry].become(TYPE)
+        self._config_entries.clear()
 
     def convert_type(self, obj, quals=0):
         if isinstance(obj, model.DefinedType):
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -113,8 +113,6 @@
 #define PyBUF_SHADOW 0x400
 /* end Py3k buffer interface */
 
-#include <cpyext_typeobject.h>
-
 #define PyObject_Bytes PyObject_Str
 
 /* Flag bits for printing: */
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -305,3 +305,11 @@
 
 } PyTypeObject;
 
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
diff --git a/pypy/module/cpyext/parse/cpyext_typeobject.h b/pypy/module/cpyext/parse/cpyext_typeobject.h
deleted file mode 100644
--- a/pypy/module/cpyext/parse/cpyext_typeobject.h
+++ /dev/null
@@ -1,8 +0,0 @@
-typedef struct {
-    PyTypeObject ht_type;
-    PyNumberMethods as_number;
-    PyMappingMethods as_mapping;
-    PySequenceMethods as_sequence;
-    PyBufferProcs as_buffer;
-    PyObject *ht_name, *ht_slots;
-} PyHeapTypeObject;
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -142,6 +142,20 @@
     assert isinstance(Object, lltype.Struct)
     hash(Object)
 
+def test_nested_struct():
+    cdef = """
+    typedef struct {
+        int x;
+    } foo;
+    typedef struct {
+        foo y;
+    } bar;
+    """
+    cts = parse_source(cdef)
+    bar = cts.gettype('bar')
+    assert isinstance(bar, lltype.Struct)
+    hash(bar)  # bar is hashable
+
 def test_const():
     cdef = """
     typedef struct {
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -41,7 +41,6 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
-cts.parse_header(parse_dir / 'cpyext_typeobject.h')
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
 
diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -131,38 +131,32 @@
 # General interface
 
 class ConfigResult:
-    def __init__(self, CConfig, info, entries):
-        self.CConfig = CConfig
+    def __init__(self, eci, info):
+        self.eci = eci
+        self.info = info
         self.result = {}
-        self.info = info
-        self.entries = entries
 
     def get_entry_result(self, entry):
         try:
             return self.result[entry]
         except KeyError:
             pass
-        name = self.entries[entry]
-        info = self.info[name]
+        info = self.info[entry]
         self.result[entry] = entry.build_result(info, self)
         return self.result[entry]
 
-    def get_result(self):
-        return dict([(name, self.result[entry])
-                     for entry, name in self.entries.iteritems()])
 
 class _CWriter(object):
     """ A simple class which aggregates config parts
     """
-    def __init__(self, CConfig):
+    def __init__(self, eci):
         self.path = uniquefilepath()
         self.f = self.path.open("w")
-        self.config = CConfig
+        self.eci = eci
 
     def write_header(self):
         f = self.f
-        CConfig = self.config
-        CConfig._compilation_info_.write_c_header(f)
+        self.eci.write_c_header(f)
         print >> f, C_HEADER
         print >> f
 
@@ -194,8 +188,7 @@
         self.start_main()
         self.f.write(question + "\n")
         self.close()
-        eci = self.config._compilation_info_
-        try_compile_cache([self.path], eci)
+        try_compile_cache([self.path], self.eci)
 
 def configure(CConfig, ignore_errors=False):
     """Examine the local system by running the C compiler.
@@ -208,50 +201,53 @@
         assert not hasattr(CConfig, attr), \
             "Found legacy attribute %s on CConfig" % attr
 
-    entries = []
+    eci = CConfig._compilation_info_
+    entries = {}
     for key in dir(CConfig):
         value = getattr(CConfig, key)
         if isinstance(value, CConfigEntry):
-            entries.append((key, value))
+            entries[key] = value
 
+    res = {}
     if entries:   # can be empty if there are only CConfigSingleEntries
-        writer = _CWriter(CConfig)
-        writer.write_header()
-        for key, entry in entries:
-            writer.write_entry(key, entry)
-
-        writer.start_main()
-        for key, entry in entries:
-            writer.write_entry_main(key)
-        writer.close()
-
-        eci = CConfig._compilation_info_
-        infolist = list(run_example_code(writer.path, eci,
-                                         ignore_errors=ignore_errors))
-        assert len(infolist) == len(entries)
-
-        resultinfo = {}
-        resultentries = {}
-        for info, (key, entry) in zip(infolist, entries):
-            resultinfo[key] = info
-            resultentries[entry] = key
-
-        result = ConfigResult(CConfig, resultinfo, resultentries)
-        for name, entry in entries:
-            result.get_entry_result(entry)
-        res = result.get_result()
-    else:
-        res = {}
+        results = configure_entries(
+            entries.values(), eci, ignore_errors=ignore_errors)
+        for name, result in zip(entries, results):
+            res[name] = result
 
     for key in dir(CConfig):
         value = getattr(CConfig, key)
         if isinstance(value, CConfigSingleEntry):
-            writer = _CWriter(CConfig)
+            writer = _CWriter(eci)
             writer.write_header()
             res[key] = value.question(writer.ask_gcc)
 
     return res
 
+
+def configure_entries(entries, eci, ignore_errors=False):
+    writer = _CWriter(eci)
+    writer.write_header()
+    for i, entry in enumerate(entries):
+        writer.write_entry(str(i), entry)
+
+    writer.start_main()
+    for i, entry in enumerate(entries):
+        writer.write_entry_main(str(i))
+    writer.close()
+
+    infolist = list(run_example_code(
+        writer.path, eci, ignore_errors=ignore_errors))
+    assert len(infolist) == len(entries)
+
+    resultinfo = {}
+    for info, entry in zip(infolist, entries):
+        resultinfo[entry] = info
+
+    result = ConfigResult(eci, resultinfo)
+    for entry in entries:
+        yield result.get_entry_result(entry)
+
 # ____________________________________________________________
 
 
@@ -344,7 +340,7 @@
         allfields = tuple(['c_' + name for name, _ in fields])
         padfields = tuple(padfields)
         name = self.name
-        eci = config_result.CConfig._compilation_info_
+        eci = config_result.eci
         padding_drop = PaddingDrop(name, allfields, padfields, eci)
         hints = {'align': info['align'],
                  'size': info['size'],

From pypy.commits at gmail.com  Mon Jan 30 10:57:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 30 Jan 2017 07:57:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <588f6284.ce9adf0a.bd348.f377@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89836:530080182f9b
Date: 2017-01-30 16:56 +0100
http://bitbucket.org/pypy/pypy/changeset/530080182f9b/

Log:	hg merge default


From pypy.commits at gmail.com  Mon Jan 30 11:03:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 30 Jan 2017 08:03:25 -0800 (PST)
Subject: [pypy-commit] pypy default: Replace @api_decl(...,
 cts) with @cts.decl(...)
Message-ID: <588f63cd.c80e1c0a.952b2.7c52@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89837:6873f23470b6
Date: 2017-01-30 16:02 +0000
http://bitbucket.org/pypy/pypy/changeset/6873f23470b6/

Log:	Replace @api_decl(..., cts) with @cts.decl(...)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -483,19 +483,23 @@
         return unwrapper
     return decorate
 
+def api_func_from_cdef(func, cdef, cts,
+        error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+    func._always_inline_ = 'try'
+    cdecl = cts.parse_func(cdef)
+    RESULT = cdecl.get_llresult(cts)
+    api_function = ApiFunction(
+        cdecl.get_llargs(cts), RESULT, func,
+        error=_compute_error(error, RESULT), cdecl=cdecl)
+    FUNCTIONS_BY_HEADER[header][cdecl.name] = api_function
+    unwrapper = api_function.get_unwrapper()
+    unwrapper.func = func
+    unwrapper.api_func = api_function
+    return unwrapper
+
 def api_decl(cdef, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
     def decorate(func):
-        func._always_inline_ = 'try'
-        cdecl = cts.parse_func(cdef)
-        RESULT = cdecl.get_llresult(cts)
-        api_function = ApiFunction(
-            cdecl.get_llargs(cts), RESULT, func,
-            error=_compute_error(error, RESULT), cdecl=cdecl)
-        FUNCTIONS_BY_HEADER[header][cdecl.name] = api_function
-        unwrapper = api_function.get_unwrapper()
-        unwrapper.func = func
-        unwrapper.api_func = api_function
-        return unwrapper
+        return api_func_from_cdef(func, cdef, cts, error=error, header=header)
     return decorate
 
 def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
@@ -676,7 +680,17 @@
                              % (cpyname, ))
 build_exported_objects()
 
-cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h', 'stddef.h'])
+
+class CpyextTypeSpace(CTypeSpace):
+    def decl(self, cdef):
+        def decorate(func):
+            return api_func_from_cdef(
+                    func, cdef, self, error=error, header=header)
+        return decorate
+
+
+CPYEXT_BASE_HEADERS = ['sys/types.h', 'stdarg.h', 'stdio.h', 'stddef.h']
+cts = CpyextTypeSpace(headers=CPYEXT_BASE_HEADERS)
 cts.parse_header(parse_dir / 'cpyext_object.h')
 
 Py_ssize_t = cts.gettype('Py_ssize_t')
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,7 +11,7 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr, slot_function, cts, api_decl)
+    PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -282,7 +282,7 @@
 def PyCFunction_NewEx(space, ml, w_self, w_name):
     return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name))
 
- at api_decl("PyCFunction PyCFunction_GetFunction(PyObject *)", cts)
+ at cts.decl("PyCFunction PyCFunction_GetFunction(PyObject *)")
 def PyCFunction_GetFunction(space, w_obj):
     try:
         cfunction = space.interp_w(W_PyCFunctionObject, w_obj)

From pypy.commits at gmail.com  Mon Jan 30 11:09:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 30 Jan 2017 08:09:51 -0800 (PST)
Subject: [pypy-commit] pypy default: fix
Message-ID: <588f654f.c80e1c0a.952b2.7f4c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89838:c173189d53b2
Date: 2017-01-30 16:09 +0000
http://bitbucket.org/pypy/pypy/changeset/c173189d53b2/

Log:	fix

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -682,7 +682,7 @@
 
 
 class CpyextTypeSpace(CTypeSpace):
-    def decl(self, cdef):
+    def decl(self, cdef, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
         def decorate(func):
             return api_func_from_cdef(
                     func, cdef, self, error=error, header=header)

From pypy.commits at gmail.com  Mon Jan 30 11:14:00 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 30 Jan 2017 08:14:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <588f6648.4a6b1c0a.3b0f2.7bb4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89839:f5231270d7c6
Date: 2017-01-30 16:13 +0000
http://bitbucket.org/pypy/pypy/changeset/f5231270d7c6/

Log:	hg merge default

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -486,19 +486,23 @@
         return unwrapper
     return decorate
 
+def api_func_from_cdef(func, cdef, cts,
+        error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+    func._always_inline_ = 'try'
+    cdecl = cts.parse_func(cdef)
+    RESULT = cdecl.get_llresult(cts)
+    api_function = ApiFunction(
+        cdecl.get_llargs(cts), RESULT, func,
+        error=_compute_error(error, RESULT), cdecl=cdecl)
+    FUNCTIONS_BY_HEADER[header][cdecl.name] = api_function
+    unwrapper = api_function.get_unwrapper()
+    unwrapper.func = func
+    unwrapper.api_func = api_function
+    return unwrapper
+
 def api_decl(cdef, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
     def decorate(func):
-        func._always_inline_ = 'try'
-        cdecl = cts.parse_func(cdef)
-        RESULT = cdecl.get_llresult(cts)
-        api_function = ApiFunction(
-            cdecl.get_llargs(cts), RESULT, func,
-            error=_compute_error(error, RESULT), cdecl=cdecl)
-        FUNCTIONS_BY_HEADER[header][cdecl.name] = api_function
-        unwrapper = api_function.get_unwrapper()
-        unwrapper.func = func
-        unwrapper.api_func = api_function
-        return unwrapper
+        return api_func_from_cdef(func, cdef, cts, error=error, header=header)
     return decorate
 
 def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
@@ -681,7 +685,17 @@
                              % (cpyname, ))
 build_exported_objects()
 
-cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h', 'stddef.h'])
+
+class CpyextTypeSpace(CTypeSpace):
+    def decl(self, cdef, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+        def decorate(func):
+            return api_func_from_cdef(
+                    func, cdef, self, error=error, header=header)
+        return decorate
+
+
+CPYEXT_BASE_HEADERS = ['sys/types.h', 'stdarg.h', 'stdio.h', 'stddef.h']
+cts = CpyextTypeSpace(headers=CPYEXT_BASE_HEADERS)
 cts.parse_header(parse_dir / 'cpyext_object.h')
 
 Py_ssize_t = cts.gettype('Py_ssize_t')
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,7 +11,7 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr, slot_function, cts, api_decl)
+    PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -272,7 +272,7 @@
 def PyCFunction_NewEx(space, ml, w_self, w_name):
     return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name))
 
- at api_decl("PyCFunction PyCFunction_GetFunction(PyObject *)", cts)
+ at cts.decl("PyCFunction PyCFunction_GetFunction(PyObject *)")
 def PyCFunction_GetFunction(space, w_obj):
     try:
         cfunction = space.interp_w(W_PyCFunctionObject, w_obj)

From pypy.commits at gmail.com  Mon Jan 30 11:15:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 30 Jan 2017 08:15:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-siphash24: Fix the parameters in
 sys.hash_info
Message-ID: <588f6687.c80e1c0a.952b2.8178@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-siphash24
Changeset: r89842:26fb67149ae5
Date: 2017-01-30 17:12 +0100
http://bitbucket.org/pypy/pypy/changeset/26fb67149ae5/

Log:	Fix the parameters in sys.hash_info

diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -5,7 +5,6 @@
 from pypy.objspace.std.complexobject import HASH_IMAG
 from pypy.objspace.std.floatobject import HASH_INF, HASH_NAN
 from pypy.objspace.std.intobject import HASH_MODULUS
-from pypy.objspace.std.bytesobject import HASH_ALGORITHM
 from pypy.interpreter import gateway
 from rpython.rlib import rbigint, rfloat
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -79,11 +78,22 @@
     return space.call_function(w_int_info, space.newtuple(info_w))
 
 def get_hash_info(space):
-    HASH_HASH_BITS = 8 * rffi.sizeof(lltype.Signed)
-    HASH_SEED_BITS = 0    # XXX don't know what this is supposed to be
+    HASH_ALGORITHM = space.config.objspace.hash
+    if space.config.objspace.hash == "fnv":
+        HASH_HASH_BITS = 8 * rffi.sizeof(lltype.Signed)
+        HASH_SEED_BITS = 0
+        #   CPython has  ^ > 0  here, but the seed of "fnv" is of limited
+        #   use, so we don't implement it
+    elif space.config.objspace.hash == "siphash24":
+        HASH_HASH_BITS = 64
+        HASH_SEED_BITS = 128
+    else:
+        assert 0, "please add the parameters for this different hash function"
+
+    HASH_WIDTH = 8 * rffi.sizeof(lltype.Signed)
     HASH_CUTOFF = 0
     info_w = [
-        space.wrap(8 * rffi.sizeof(lltype.Signed)),
+        space.wrap(HASH_WIDTH),
         space.wrap(HASH_MODULUS),
         space.wrap(HASH_INF),
         space.wrap(HASH_NAN),
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -3,7 +3,7 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin, newlist_hint,
-    resizelist_hint, HASH_ALGORITHM)
+    resizelist_hint)
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder
 

From pypy.commits at gmail.com  Mon Jan 30 11:15:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 30 Jan 2017 08:15:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-siphash24: py3.5 defaults to "siphash24"
 instead of "fnv"
Message-ID: <588f6685.cda0df0a.dace6.e80a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-siphash24
Changeset: r89841:a87943ab010b
Date: 2017-01-30 17:00 +0100
http://bitbucket.org/pypy/pypy/changeset/a87943ab010b/

Log:	py3.5 defaults to "siphash24" instead of "fnv"

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -205,7 +205,7 @@
                  "The hash function to use for strings: fnv from CPython 2.7"
                  " or siphash24 from CPython >= 3.4",
                  ["fnv", "siphash24"],
-                 default="fnv",
+                 default="siphash24",
                  cmdline="--hash"),
 
     OptionDescription("std", "Standard Object Space Options", [

From pypy.commits at gmail.com  Mon Jan 30 11:19:24 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 30 Jan 2017 08:19:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Replace @api_decl(...,
 cts) with @cts.decl(...)
Message-ID: <588f678c.9d711c0a.71d4d.7ffd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89843:3bda7ec5d342
Date: 2017-01-30 16:18 +0000
http://bitbucket.org/pypy/pypy/changeset/3bda7ec5d342/

Log:	Replace @api_decl(..., cts) with @cts.decl(...)

diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -1,6 +1,5 @@
 from pypy.module.cpyext.api import (
-    cpython_api, PyObject, CONST_STRING, CANNOT_FAIL,
-    cts, api_decl)
+    cpython_api, PyObject, CONST_STRING, CANNOT_FAIL, cts)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.module import Module
@@ -51,10 +50,10 @@
     return PyImport_Import(space, space.wrap(rffi.charp2str(name)))
 
 
- at api_decl(
+ at cts.decl(
     '''PyObject* PyImport_ImportModuleLevelObject(
         PyObject *name, PyObject *given_globals, PyObject *locals,
-        PyObject *given_fromlist, int level)''', cts)
+        PyObject *given_fromlist, int level)''')
 def PyImport_ImportModuleLevelObject(space, w_name, w_glob, w_loc, w_fromlist, level):
     level = rffi.cast(lltype.Signed, level)
     if w_glob is None:
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t, slot_function, api_decl, cts,
+    PyVarObject, Py_buffer, size_t, slot_function, cts,
     PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
@@ -248,7 +248,7 @@
         return space.wrap("<NULL>")
     return space.str(w_obj)
 
- at api_decl("PyObject * PyObject_Bytes(PyObject *v)", cts)
+ at cts.decl("PyObject * PyObject_Bytes(PyObject *v)")
 def PyObject_Bytes(space, w_obj):
     if w_obj is None:
         return space.newbytes("<NULL>")
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
-    CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api, api_decl,
+    CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
     bootstrap_function, CONST_STRING,
     CONST_WSTRING, Py_CLEANUP_SUPPORTED, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
@@ -234,7 +234,7 @@
         raise oefmt(space.w_TypeError, "expected unicode object")
     return PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
 
- at api_decl("char * PyUnicode_AsUTF8(PyObject *unicode)", cts)
+ at cts.decl("char * PyUnicode_AsUTF8(PyObject *unicode)")
 def PyUnicode_AsUTF8(space, ref):
     ref_unicode = rffi.cast(PyUnicodeObject, ref)
     if not ref_unicode.c_utf8buffer:

From pypy.commits at gmail.com  Mon Jan 30 11:22:08 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 30 Jan 2017 08:22:08 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: hg merge py3.5
Message-ID: <588f6830.c3a4df0a.ee62f.f932@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89844:7f5f8c45c219
Date: 2017-01-30 16:21 +0000
http://bitbucket.org/pypy/pypy/changeset/7f5f8c45c219/

Log:	hg merge py3.5

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -547,6 +547,13 @@
         for item in list(consider):
             item.walkabout(self)
         self.pop_scope()
+        # http://bugs.python.org/issue10544: was never fixed in CPython,
+        # but we can at least issue a SyntaxWarning in the meantime
+        if new_scope.is_generator:
+            msg = ("'yield' inside a list or generator comprehension behaves "
+                   "unexpectedly (http://bugs.python.org/issue10544)")
+            misc.syntax_warning(self.space, msg, self.compile_info.filename,
+                                node.lineno, node.col_offset)
 
     def visit_ListComp(self, listcomp):
         self._visit_comprehension(listcomp, listcomp.generators, listcomp.elt)
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -343,6 +343,7 @@
             assert ex.match(self.space, self.space.w_SyntaxError)
 
     def test_globals_warnings(self):
+        # also tests some other constructions that give a warning
         space = self.space
         w_mod = space.appexec((), '():\n import warnings\n return warnings\n') #sys.getmodule('warnings')
         w_filterwarnings = space.getattr(w_mod, space.wrap('filterwarnings'))
@@ -364,6 +365,18 @@
     print x
     x = 2
     global x
+''', '''
+def wrong_listcomp():
+    return [(yield 42) for i in j]
+''', '''
+def wrong_gencomp():
+    return ((yield 42) for i in j)
+''', '''
+def wrong_dictcomp():
+    return {(yield 42):2 for i in j}
+''', '''
+def wrong_setcomp():
+    return {(yield 42) for i in j}
 '''):
 
             space.call_args(w_filterwarnings, filter_arg)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -486,19 +486,23 @@
         return unwrapper
     return decorate
 
+def api_func_from_cdef(func, cdef, cts,
+        error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+    func._always_inline_ = 'try'
+    cdecl = cts.parse_func(cdef)
+    RESULT = cdecl.get_llresult(cts)
+    api_function = ApiFunction(
+        cdecl.get_llargs(cts), RESULT, func,
+        error=_compute_error(error, RESULT), cdecl=cdecl)
+    FUNCTIONS_BY_HEADER[header][cdecl.name] = api_function
+    unwrapper = api_function.get_unwrapper()
+    unwrapper.func = func
+    unwrapper.api_func = api_function
+    return unwrapper
+
 def api_decl(cdef, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
     def decorate(func):
-        func._always_inline_ = 'try'
-        cdecl = cts.parse_func(cdef)
-        RESULT = cdecl.get_llresult(cts)
-        api_function = ApiFunction(
-            cdecl.get_llargs(cts), RESULT, func,
-            error=_compute_error(error, RESULT), cdecl=cdecl)
-        FUNCTIONS_BY_HEADER[header][cdecl.name] = api_function
-        unwrapper = api_function.get_unwrapper()
-        unwrapper.func = func
-        unwrapper.api_func = api_function
-        return unwrapper
+        return api_func_from_cdef(func, cdef, cts, error=error, header=header)
     return decorate
 
 def slot_function(argtypes, restype, error=_NOT_SPECIFIED):
@@ -681,7 +685,17 @@
                              % (cpyname, ))
 build_exported_objects()
 
-cts = CTypeSpace(headers=['sys/types.h', 'stdarg.h', 'stdio.h', 'stddef.h'])
+
+class CpyextTypeSpace(CTypeSpace):
+    def decl(self, cdef, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+        def decorate(func):
+            return api_func_from_cdef(
+                    func, cdef, self, error=error, header=header)
+        return decorate
+
+
+CPYEXT_BASE_HEADERS = ['sys/types.h', 'stdarg.h', 'stdio.h', 'stddef.h']
+cts = CpyextTypeSpace(headers=CPYEXT_BASE_HEADERS)
 cts.parse_header(parse_dir / 'cpyext_object.h')
 
 Py_ssize_t = cts.gettype('Py_ssize_t')
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -1,6 +1,5 @@
 from pypy.module.cpyext.api import (
-    cpython_api, PyObject, CONST_STRING, CANNOT_FAIL,
-    cts, api_decl)
+    cpython_api, PyObject, CONST_STRING, CANNOT_FAIL, cts)
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.module import Module
@@ -51,10 +50,10 @@
     return PyImport_Import(space, space.wrap(rffi.charp2str(name)))
 
 
- at api_decl(
+ at cts.decl(
     '''PyObject* PyImport_ImportModuleLevelObject(
         PyObject *name, PyObject *given_globals, PyObject *locals,
-        PyObject *given_fromlist, int level)''', cts)
+        PyObject *given_fromlist, int level)''')
 def PyImport_ImportModuleLevelObject(space, w_name, w_glob, w_loc, w_fromlist, level):
     level = rffi.cast(lltype.Signed, level)
     if w_glob is None:
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -11,7 +11,7 @@
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
     build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
-    PyTypeObjectPtr, slot_function, cts, api_decl)
+    PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -272,7 +272,7 @@
 def PyCFunction_NewEx(space, ml, w_self, w_name):
     return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name))
 
- at api_decl("PyCFunction PyCFunction_GetFunction(PyObject *)", cts)
+ at cts.decl("PyCFunction PyCFunction_GetFunction(PyObject *)")
 def PyCFunction_GetFunction(space, w_obj):
     try:
         cfunction = space.interp_w(W_PyCFunctionObject, w_obj)
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t, slot_function, api_decl, cts,
+    PyVarObject, Py_buffer, size_t, slot_function, cts,
     PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
@@ -248,7 +248,7 @@
         return space.wrap("<NULL>")
     return space.str(w_obj)
 
- at api_decl("PyObject * PyObject_Bytes(PyObject *v)", cts)
+ at cts.decl("PyObject * PyObject_Bytes(PyObject *v)")
 def PyObject_Bytes(space, w_obj):
     if w_obj is None:
         return space.newbytes("<NULL>")
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
-    CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api, api_decl,
+    CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
     bootstrap_function, CONST_STRING,
     CONST_WSTRING, Py_CLEANUP_SUPPORTED, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
@@ -256,7 +256,7 @@
         raise oefmt(space.w_TypeError, "expected unicode object")
     return PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
 
- at api_decl("char * PyUnicode_AsUTF8(PyObject *unicode)", cts)
+ at cts.decl("char * PyUnicode_AsUTF8(PyObject *unicode)")
 def PyUnicode_AsUTF8(space, ref):
     ref_unicode = rffi.cast(PyUnicodeObject, ref)
     if not get_utf8(ref_unicode):

From pypy.commits at gmail.com  Mon Jan 30 11:25:50 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Mon, 30 Jan 2017 08:25:50 -0800 (PST)
Subject: [pypy-commit] pypy real-mode-translator-driver: backendopt should
 run before stackcheckinsertion if it has not been disabled.
Message-ID: <588f690e.8c7e1c0a.8fb6e.7d7a@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r89845:0c50e082bdd7
Date: 2017-01-31 03:24 +1100
http://bitbucket.org/pypy/pypy/changeset/0c50e082bdd7/

Log:	backendopt should run before stackcheckinsertion if it has not been
	disabled.

	Also, fix logging when there is nothing left to do.

diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -108,7 +108,8 @@
 
     def stackcheckinsertion_lltype(self):
         self.rtype_lltype()
-        if 'backendopt_lltype' in self.extra_goals:
+        disabled = self.backend_select_goals(self._disabled)
+        if 'backendopt_lltype' not in disabled:
             self.backendopt_lltype()
         return self.run_task(self.task_stackcheckinsertion_lltype,
                              'stackcheckinsertion_lltype')
@@ -213,7 +214,7 @@
         backend, ts = self.get_backend_and_type_system()
         goals = set(self.backend_select_goals(goals + self.extra_goals))
         if not goals:
-            self.log('Nothing to do.')
+            self.log.info('Nothing to do.')
         self.extra_goals += goals
 
         # run C goals first to catch missing boehm.

From pypy.commits at gmail.com  Mon Jan 30 11:58:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 30 Jan 2017 08:58:53 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: fix tests
Message-ID: <588f70cd.c53f1c0a.c9a44.2643@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89846:9e79f3fff9b6
Date: 2017-01-30 17:58 +0100
http://bitbucket.org/pypy/pypy/changeset/9e79f3fff9b6/

Log:	fix tests

diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -1,10 +1,3 @@
-import sys, os
-
-if __name__ == '__main__':
-    # hack for test_translation_prebuilt_2()
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__),
-                                    '..', '..', '..'))
-
 import py
 from rpython.annotator.model import UnionError
 from rpython.rlib import rgc, nonconst
@@ -238,23 +231,16 @@
     fc()
 
 def _test_translation_prebuilt_2():
-    from rpython.rlib import objectmodel
-    objectmodel.set_hash_algorithm("siphash24")
+    from rpython.rlib import rsiphash
     d = RWeakValueDictionary(str, X)
     k1 = "key1"; k2 = "key2"
     x1 = X(); x2 = X()
     d.set(k1, x1)
     d.set(k2, x2)
     def f():
+        rsiphash.enable_siphash24()
         i = nonconst.NonConstant(1)
         assert d.get("key%d" % (i,)) is x1
         assert d.get("key%d" % (i+1,)) is x2
     fc = compile(f, [], gcpolicy="boehm", rweakref=True)
     fc()
-
-def test_translation_prebuilt_2():
-    import subprocess
-    subprocess.check_call([sys.executable, __file__])
-
-if __name__ == "__main__":
-    _test_translation_prebuilt_2()
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -513,9 +513,6 @@
     # __________________________________________________________
     # misc LL operation implementations
 
-    def op_call_at_startup(self, *args):
-        pass
-
     def op_debug_view(self, *ll_objects):
         from rpython.translator.tool.lltracker import track
         track(*ll_objects)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -539,7 +539,7 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
-    'call_at_startup':      LLOp(),
+    'call_at_startup':      LLOp(canrun=True),
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -742,6 +742,9 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
+def op_call_at_startup(init_func):
+    pass    # do nothing
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -193,6 +193,7 @@
 
     def gen_block(self, block):
         if 1:      # (preserve indentation)
+            self._current_block = block
             myblocknum = self.blocknum[block]
             if block in self.inlinable_blocks:
                 # debug comment
@@ -942,7 +943,16 @@
                 self.expr(op.result))
 
     def OP_CALL_AT_STARTUP(self, op):
-        assert isinstance(op.args[0], Constant)
-        func = self.expr(op.args[0])
+        c = op.args[0]
+        if not isinstance(c, Constant):
+            # Bah, maybe it comes from a same_as(const) just before...
+            # Can occur if running without backendopts
+            for op1 in self._current_block.operations:
+                if op1.result is op.args[0]:
+                    assert op1.opname == "same_as"
+                    c = op1.args[0]
+                    break
+            assert isinstance(c, Constant)
+        func = self.expr(c)
         self.db.call_at_startup.add(func)
         return '/* call_at_startup %s */' % (func,)
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -2,11 +2,6 @@
 
 import math
 import sys, os
-
-if __name__ == '__main__':
-    # hack for test_hash_string_siphash24()
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__),
-                                    '..', '..', '..', '..'))
 import py
 
 from rpython.rlib.rstackovf import StackOverflow
@@ -604,8 +599,6 @@
         #    and not e.g. siphash24
 
     def _test_hash_string(self, algo):
-        from rpython.rlib import objectmodel
-        objectmodel.set_hash_algorithm(algo)
         s = "hello"
         u = u"world"
         v = u"\u1234\u2318+\u2bcd\u2102"
@@ -616,6 +609,9 @@
         assert hash_u == compute_hash("world")    #    a latin-1 unicode
         #
         def fn(length):
+            if algo == "siphash24":
+                from rpython.rlib import rsiphash
+                rsiphash.enable_siphash24()
             assert length >= 1
             return str((compute_hash(s),
                         compute_hash(u),
@@ -630,21 +626,23 @@
         f = self.getcompiled(fn, [int])
         res = f(5)
         res = [int(a) for a in res[1:-1].split(",")]
-        assert res[0] == hash_s
-        assert res[1] == hash_u
-        assert res[2] == hash_v
-        assert res[3] == hash_s
-        assert res[4] == hash_u
-        assert res[5] == hash_v
+        if algo == "fnv":
+            assert res[0] == hash_s
+            assert res[1] == hash_u
+            assert res[2] == hash_v
+        else:
+            assert res[0] != hash_s
+            assert res[1] != hash_u
+            assert res[2] != hash_v
+        assert res[3] == res[0]
+        assert res[4] == res[1]
+        assert res[5] == res[2]
 
-    def test_hash_string_rpython(self):
-        self._test_hash_string("rpython")
+    def test_hash_string_fnv(self):
+        self._test_hash_string("fnv")
 
     def test_hash_string_siphash24(self):
-        import subprocess
-        subprocess.check_call([sys.executable, __file__, "siphash24",
-                               self.__class__.__module__,
-                               self.__class__.__name__])
+        self._test_hash_string("siphash24")
 
     def test_list_basic_ops(self):
         def list_basic_ops(i, j):
@@ -945,11 +943,3 @@
         f = self.getcompiled(func, [int])
         res = f(2)
         assert res == 1     # and not 2
-
-
-if __name__ == '__main__':
-    # for test_hash_string_siphash24()
-    algo, clsmodule, clsname = sys.argv[1:]
-    mod = __import__(clsmodule, None, None, [clsname])
-    cls = getattr(mod, clsname)
-    cls()._test_hash_string(algo)

From pypy.commits at gmail.com  Mon Jan 30 11:14:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 30 Jan 2017 08:14:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-siphash24: merge of py3.5 and rpython-hash
Message-ID: <588f6683.50a4df0a.2bbd1.0013@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-siphash24
Changeset: r89840:b16aafab3cca
Date: 2017-01-30 17:00 +0100
http://bitbucket.org/pypy/pypy/changeset/b16aafab3cca/

Log:	merge of py3.5 and rpython-hash

diff too long, truncating to 2000 out of 2764 lines

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -201,6 +201,13 @@
                "issue, you can disable them here",
                default=True),
 
+    ChoiceOption("hash",
+                 "The hash function to use for strings: fnv from CPython 2.7"
+                 " or siphash24 from CPython >= 3.4",
+                 ["fnv", "siphash24"],
+                 default="fnv",
+                 cmdline="--hash"),
+
     OptionDescription("std", "Standard Object Space Options", [
         BoolOption("withtproxy", "support transparent proxies",
                    default=True),
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -36,6 +36,7 @@
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
         w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+        hashfunc = space.config.objspace.hash
     else:
         w_initstdio = space.appexec([], """():
             return lambda unbuffered: None
@@ -46,6 +47,10 @@
             from rpython.jit.backend.hlinfo import highleveljitinfo
             highleveljitinfo.sys_executable = argv[0]
 
+        if hashfunc == "siphash24":
+            from rpython.rlib import rsiphash
+            rsiphash.enable_siphash24()
+
         #debug("entry point starting")
         #for arg in argv:
         #    debug(" argv -> " + arg)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -23,13 +23,34 @@
 # ____________________________________________________________
 
 class UniqueCache:
+    for_testing = False    # set to True on the class level in test_c.py
+
     def __init__(self, space):
         self.ctvoid = None      # Cache for the 'void' type
         self.ctvoidp = None     # Cache for the 'void *' type
         self.ctchara = None     # Cache for the 'char[]' type
         self.primitives = {}    # Cache for {name: primitive_type}
         self.functions = []     # see _new_function_type()
-        self.for_testing = False
+        self.functions_packed = None     # only across translation
+
+    def _cleanup_(self):
+        import gc
+        assert self.functions_packed is None
+        # Note: a full PyPy translation may still have
+        # 'self.functions == []' at this point, possibly depending
+        # on details.  Code tested directly in test_ffi_obj
+        gc.collect()
+        funcs = []
+        for weakdict in self.functions:
+            funcs += weakdict._dict.values()
+        del self.functions[:]
+        self.functions_packed = funcs if len(funcs) > 0 else None
+
+    def unpack_functions(self):
+        for fct in self.functions_packed:
+            _record_function_type(self, fct)
+        self.functions_packed = None
+
 
 def _clean_cache(space):
     "NOT_RPYTHON"
@@ -622,7 +643,7 @@
     for w_arg in fargs:
         y = compute_identity_hash(w_arg)
         x = intmask((1000003 * x) ^ y)
-    x ^= (ellipsis - abi)
+    x ^= ellipsis + 2 * abi
     if unique_cache.for_testing:    # constant-folded to False in translation;
         x &= 3                      # but for test, keep only 2 bits of hash
     return x
@@ -646,6 +667,8 @@
     # one such dict, but in case of hash collision, there might be
     # more.
     unique_cache = space.fromcache(UniqueCache)
+    if unique_cache.functions_packed is not None:
+        unique_cache.unpack_functions()
     func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
     for weakdict in unique_cache.functions:
         ctype = weakdict.get(func_hash)
@@ -674,13 +697,18 @@
     #
     fct = ctypefunc.W_CTypeFunc(space, fargs, fresult, ellipsis, abi)
     unique_cache = space.fromcache(UniqueCache)
-    func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
+    _record_function_type(unique_cache, fct)
+    return fct
+
+def _record_function_type(unique_cache, fct):
+    from pypy.module._cffi_backend import ctypefunc
+    #
+    func_hash = _func_key_hash(unique_cache, fct.fargs, fct.ctitem,
+                               fct.ellipsis, fct.abi)
     for weakdict in unique_cache.functions:
         if weakdict.get(func_hash) is None:
-            weakdict.set(func_hash, fct)
             break
     else:
         weakdict = rweakref.RWeakValueDictionary(int, ctypefunc.W_CTypeFunc)
         unique_cache.functions.append(weakdict)
-        weakdict.set(func_hash, fct)
-    return fct
+    weakdict.set(func_hash, fct)
diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py
--- a/pypy/module/_cffi_backend/test/test_c.py
+++ b/pypy/module/_cffi_backend/test/test_c.py
@@ -37,6 +37,7 @@
     def setup_class(cls):
         testfuncs_w = []
         keepalive_funcs = []
+        UniqueCache.for_testing = True
 
         test_lib_c = tmpdir.join('_test_lib.c')
         src_test_lib_c = py.path.local(__file__).dirpath().join('_test_lib.c')
@@ -100,11 +101,12 @@
             _all_test_c.find_and_load_library = func
             _all_test_c._testfunc = testfunc
         """)
-        UniqueCache.for_testing = True
 
     def teardown_method(self, method):
+        _clean_cache(self.space)
+
+    def teardown_class(cls):
         UniqueCache.for_testing = False
-        _clean_cache(self.space)
 
 
 all_names = ', '.join(Module.interpleveldefs.keys())
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -1,5 +1,23 @@
+from pypy.module._cffi_backend import newtype
 from pypy.module._cffi_backend.newtype import _clean_cache
 
+
+class TestFFIObj:
+    spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
+
+    def teardown_method(self, meth):
+        _clean_cache(self.space)
+
+    def test_new_function_type_during_translation(self):
+        space = self.space
+        BInt = newtype.new_primitive_type(space, "int")
+        BFunc = newtype.new_function_type(space, space.wrap([BInt]), BInt)
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+        unique_cache = space.fromcache(newtype.UniqueCache)
+        unique_cache._cleanup_()
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+
+
 class AppTestFFIObj:
     spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
 
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -194,6 +194,15 @@
         W_WeakrefBase.__init__(self, space, w_obj, w_callable)
         self.w_hash = None
 
+    def _cleanup_(self):
+        # When a prebuilt weakref is frozen inside a translation, if
+        # this weakref has got an already-cached w_hash, then throw it
+        # away.  That's because the hash value will change after
+        # translation.  It will be recomputed the first time we ask for
+        # it.  Note that such a frozen weakref, if not dead, will point
+        # to a frozen object, so it will never die.
+        self.w_hash = None
+
     def descr__init__weakref(self, space, w_obj, w_callable=None,
                              __args__=None):
         if __args__.arguments_w:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2127,6 +2127,12 @@
         raise wrap_oserror(space, e, eintr_retry=False)
     return space.wrap(res)
 
+class SigCheck:
+    pass
+_sigcheck = SigCheck()
+def _signal_checker():
+    _sigcheck.space.getexecutioncontext().checksignals()
+
 @unwrap_spec(size=int)
 def urandom(space, size):
     """urandom(size) -> str
@@ -2134,9 +2140,12 @@
     Return a string of 'size' random bytes suitable for cryptographic use.
     """
     context = get(space).random_context
-    signal_checker = space.getexecutioncontext().checksignals
     try:
-        return space.newbytes(rurandom.urandom(context, n, signal_checker))
+        # urandom() takes a final argument that should be a regular function,
+        # not a bound method like 'getexecutioncontext().checksignals'.
+        # Otherwise, we can't use it from several independent places.
+        _sigcheck.space = space
+        return space.newbytes(rurandom.urandom(context, n, _signal_checker))
     except OSError as e:
         # 'rurandom' should catch and retry internally if it gets EINTR
         # (at least in os.read(), which is probably enough in practice)
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -563,6 +563,11 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
+    def _cleanup_(self):
+        # in case there are frozenset objects existing during
+        # translation, make sure we don't translate a cached hash
+        self.hash = 0
+
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_FrozensetObject):
             return False
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -287,7 +287,7 @@
                     for ek, ev in items:
                         result.dictdef.generalize_key(self.immutablevalue(ek))
                         result.dictdef.generalize_value(self.immutablevalue(ev))
-                        result.dictdef.seen_prebuilt_key(ek)
+                        #dictdef.seen_prebuilt_key(ek)---not needed any more
                     seen_elements = len(items)
                     # if the dictionary grew during the iteration,
                     # start over again
diff --git a/rpython/annotator/dictdef.py b/rpython/annotator/dictdef.py
--- a/rpython/annotator/dictdef.py
+++ b/rpython/annotator/dictdef.py
@@ -115,13 +115,5 @@
     def generalize_value(self, s_value):
         self.dictvalue.generalize(s_value)
 
-    def seen_prebuilt_key(self, x):
-        # In case we are an r_dict, we don't ask for the hash ourselves.
-        # Note that if the custom hashing function ends up asking for
-        # the hash of x, then it must use compute_hash() itself, so it
-        # works out.
-        if not self.dictkey.custom_eq_hash:
-            compute_hash(x)
-
     def __repr__(self):
         return '<{%r: %r}>' % (self.dictkey.s_value, self.dictvalue.s_value)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3704,25 +3704,6 @@
         s = a.build_types(f, [int])
         assert s.const == 0
 
-    def test_hash_sideeffect(self):
-        class X:
-            pass
-        x1 = X()
-        x2 = X()
-        x3 = X()
-        d = {(2, x1): 5, (3, x2): 7}
-        def f(n, m):
-            if   m == 1: x = x1
-            elif m == 2: x = x2
-            else:        x = x3
-            return d[n, x]
-        a = self.RPythonAnnotator()
-        s = a.build_types(f, [int, int])
-        assert s.knowntype == int
-        assert hasattr(x1, '__precomputed_identity_hash')
-        assert hasattr(x2, '__precomputed_identity_hash')
-        assert not hasattr(x3, '__precomputed_identity_hash')
-
     def test_contains_of_empty_dict(self):
         class A(object):
             def meth(self):
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -201,10 +201,6 @@
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
-    ChoiceOption("hash",
-                 "The hash to use for strings",
-                 ["rpython", "siphash24"],
-                 default="rpython", cmdline="--hash"),
 
     OptionDescription("backendopt", "Backend Optimization Options", [
         # control inlining
@@ -394,12 +390,6 @@
         if sys.platform == "darwin" or sys.platform =="win32":
             raise ConfigError("'asmgcc' not supported on this platform")
 
-def apply_extra_settings(config):
-    # make the setting of config.hash definitive
-    from rpython.rlib.objectmodel import set_hash_algorithm
-    config.translation.hash = config.translation.hash
-    set_hash_algorithm(config.translation.hash)
-
 # ----------------------------------------------------------------
 
 def set_platform(config):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -117,9 +117,7 @@
 
 # The following flag is set on nursery objects of which we asked the id
 # or the identityhash.  It means that a space of the size of the object
-# has already been allocated in the nonmovable part.  The same flag is
-# abused to mark prebuilt objects whose hash has been taken during
-# translation and is statically recorded.
+# has already been allocated in the nonmovable part.
 GCFLAG_HAS_SHADOW   = first_gcflag << 3
 
 # The following flag is set temporarily on some objects during a major
@@ -208,10 +206,6 @@
     # by GCFLAG_xxx above.
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
-    # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
-    #     then they are one word longer, the extra word storing the hash.
-
 
     # During a minor collection, the objects in the nursery that are
     # moved outside are changed in-place: their header is replaced with
@@ -2640,40 +2634,22 @@
         return shadow
     _find_shadow._dont_inline_ = True
 
-    @specialize.arg(2)
-    def id_or_identityhash(self, gcobj, is_hash):
+    def id_or_identityhash(self, gcobj):
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
         obj = llmemory.cast_ptr_to_adr(gcobj)
-        #
         if self.is_valid_gc_object(obj):
             if self.is_in_nursery(obj):
                 obj = self._find_shadow(obj)
-            elif is_hash:
-                if self.header(obj).tid & GCFLAG_HAS_SHADOW:
-                    #
-                    # For identityhash(), we need a special case for some
-                    # prebuilt objects: their hash must be the same before
-                    # and after translation.  It is stored as an extra word
-                    # after the object.  But we cannot use it for id()
-                    # because the stored value might clash with a real one.
-                    size = self.get_size(obj)
-                    i = (obj + size).signed[0]
-                    # Important: the returned value is not mangle_hash()ed!
-                    return i
-        #
-        i = llmemory.cast_adr_to_int(obj)
-        if is_hash:
-            i = mangle_hash(i)
-        return i
+        return llmemory.cast_adr_to_int(obj)
     id_or_identityhash._always_inline_ = True
 
     def id(self, gcobj):
-        return self.id_or_identityhash(gcobj, False)
+        return self.id_or_identityhash(gcobj)
 
     def identityhash(self, gcobj):
-        return self.id_or_identityhash(gcobj, True)
+        return mangle_hash(self.id_or_identityhash(gcobj))
 
     # ----------
     # Finalizers
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -104,9 +104,7 @@
 
 # The following flag is set on nursery objects of which we asked the id
 # or the identityhash.  It means that a space of the size of the object
-# has already been allocated in the nonmovable part.  The same flag is
-# abused to mark prebuilt objects whose hash has been taken during
-# translation and is statically recorded.
+# has already been allocated in the nonmovable part.
 GCFLAG_HAS_SHADOW   = first_gcflag << 3
 
 # The following flag is set temporarily on some objects during a major
@@ -149,9 +147,6 @@
     # by GCFLAG_xxx above.
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
-    # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
-    #     then they are one word longer, the extra word storing the hash.
 
     _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True})
 
@@ -1868,40 +1863,22 @@
         return shadow
     _find_shadow._dont_inline_ = True
 
-    @specialize.arg(2)
-    def id_or_identityhash(self, gcobj, is_hash):
+    def id_or_identityhash(self, gcobj):
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
         obj = llmemory.cast_ptr_to_adr(gcobj)
-        #
         if self.is_valid_gc_object(obj):
             if self.is_in_nursery(obj):
                 obj = self._find_shadow(obj)
-            elif is_hash:
-                if self.header(obj).tid & GCFLAG_HAS_SHADOW:
-                    #
-                    # For identityhash(), we need a special case for some
-                    # prebuilt objects: their hash must be the same before
-                    # and after translation.  It is stored as an extra word
-                    # after the object.  But we cannot use it for id()
-                    # because the stored value might clash with a real one.
-                    size = self.get_size(obj)
-                    i = (obj + size).signed[0]
-                    # Important: the returned value is not mangle_hash()ed!
-                    return i
-        #
-        i = llmemory.cast_adr_to_int(obj)
-        if is_hash:
-            i = mangle_hash(i)
-        return i
+        return llmemory.cast_adr_to_int(obj)
     id_or_identityhash._always_inline_ = True
 
     def id(self, gcobj):
-        return self.id_or_identityhash(gcobj, False)
+        return self.id_or_identityhash(gcobj)
 
     def identityhash(self, gcobj):
-        return self.id_or_identityhash(gcobj, True)
+        return mangle_hash(self.id_or_identityhash(gcobj))
 
     # ----------
     # Finalizers
diff --git a/rpython/memory/gc/semispace.py b/rpython/memory/gc/semispace.py
--- a/rpython/memory/gc/semispace.py
+++ b/rpython/memory/gc/semispace.py
@@ -48,9 +48,6 @@
 
     HDR = lltype.Struct('header', ('tid', lltype.Signed))   # XXX or rffi.INT?
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', _GCFLAG_HASH_BASE * 0x2
-    # ^^^ prebuilt objects either have GC_HASH_TAKEN_ADDR or they
-    #     have GC_HASH_HASFIELD (and then they are one word longer).
     FORWARDSTUB = lltype.GcStruct('forwarding_stub',
                                   ('forw', llmemory.Address))
     FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -11,7 +11,7 @@
 class BoehmGCTransformer(GCTransformer):
     malloc_zero_filled = True
     FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
-    HDR = lltype.Struct("header", ("hash", lltype.Signed))
+    NO_HEADER = True
 
     def __init__(self, translator, inline=False):
         super(BoehmGCTransformer, self).__init__(translator, inline=inline)
@@ -29,13 +29,8 @@
         ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length
         ll_malloc_varsize = mh.ll_malloc_varsize
 
-        HDRPTR = lltype.Ptr(self.HDR)
-
         def ll_identityhash(addr):
-            obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
-            h = obj.hash
-            if h == 0:
-                obj.hash = h = ~llmemory.cast_adr_to_int(addr)
+            h = ~llmemory.cast_adr_to_int(addr)
             return h
 
         if self.translator:
@@ -194,11 +189,6 @@
                           resulttype = lltype.Signed)
         hop.genop('int_invert', [v_int], resultvar=hop.spaceop.result)
 
-    def gcheader_initdata(self, obj):
-        hdr = lltype.malloc(self.HDR, immortal=True)
-        hdr.hash = lltype.identityhash_nocache(obj._as_ptr())
-        return hdr._obj
-
 
 ########## weakrefs ##########
 # Boehm: weakref objects are small structures containing only a Boehm
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -610,25 +610,6 @@
     def special_funcptr_for_type(self, TYPE):
         return self.layoutbuilder.special_funcptr_for_type(TYPE)
 
-    def gc_header_for(self, obj, needs_hash=False):
-        hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj)
-        withhash, flag = self.gcdata.gc.withhash_flag_is_in_field
-        x = getattr(hdr, withhash)
-        TYPE = lltype.typeOf(x)
-        x = lltype.cast_primitive(lltype.Signed, x)
-        if needs_hash:
-            x |= flag       # set the flag in the header
-        else:
-            x &= ~flag      # clear the flag in the header
-        x = lltype.cast_primitive(TYPE, x)
-        setattr(hdr, withhash, x)
-        return hdr
-
-    def get_hash_offset(self, T):
-        type_id = self.get_type_id(T)
-        assert not self.gcdata.q_is_varsize(type_id)
-        return self.gcdata.q_fixed_size(type_id)
-
     def finish_tables(self):
         group = self.layoutbuilder.close_table()
         log.info("assigned %s typeids" % (len(group.members), ))
@@ -1514,22 +1495,9 @@
 
     def gcheader_initdata(self, obj):
         o = lltype.top_container(obj)
-        needs_hash = self.get_prebuilt_hash(o) is not None
-        hdr = self.gc_header_for(o, needs_hash)
+        hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(o)
         return hdr._obj
 
-    def get_prebuilt_hash(self, obj):
-        # for prebuilt objects that need to have their hash stored and
-        # restored.  Note that only structures that are StructNodes all
-        # the way have their hash stored (and not e.g. structs with var-
-        # sized arrays at the end).  'obj' must be the top_container.
-        TYPE = lltype.typeOf(obj)
-        if not isinstance(TYPE, lltype.GcStruct):
-            return None
-        if TYPE._is_varsize():
-            return None
-        return getattr(obj, '_hash_cache_', None)
-
     def get_finalizer_queue_index(self, hop):
         fq_tag = hop.spaceop.args[0].value
         assert 'FinalizerQueue TAG' in fq_tag.expr
diff --git a/rpython/memory/gctransform/refcounting.py b/rpython/memory/gctransform/refcounting.py
--- a/rpython/memory/gctransform/refcounting.py
+++ b/rpython/memory/gctransform/refcounting.py
@@ -18,8 +18,7 @@
 class RefcountingGCTransformer(GCTransformer):
     malloc_zero_filled = True
 
-    HDR = lltype.Struct("header", ("refcount", lltype.Signed),
-                                  ("hash", lltype.Signed))
+    HDR = lltype.Struct("header", ("refcount", lltype.Signed))
 
     def __init__(self, translator):
         super(RefcountingGCTransformer, self).__init__(translator, inline=True)
@@ -77,10 +76,7 @@
         ll_malloc_varsize = mh.ll_malloc_varsize
 
         def ll_identityhash(addr):
-            obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
-            h = obj.hash
-            if h == 0:
-                obj.hash = h = llmemory.cast_adr_to_int(addr)
+            h = llmemory.cast_adr_to_int(addr)
             return h
 
         if self.translator:
@@ -178,7 +174,6 @@
             if not self.gcheaderbuilder.get_header(p):
                 hdr = self.gcheaderbuilder.new_header(p)
                 hdr.refcount = sys.maxint // 2
-                hdr.hash = lltype.identityhash_nocache(p)
 
     def static_deallocation_funcptr_for_type(self, TYPE):
         if TYPE in self.static_deallocator_funcptrs:
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -374,9 +374,6 @@
         return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP),
                                         lltype.nullptr(ARRAY_TYPEID_MAP)))
 
-    def get_prebuilt_hash(self, obj):
-        return None
-
 
 class MinimalGCTransformer(BaseGCTransformer):
     def __init__(self, parenttransformer):
diff --git a/rpython/rlib/_rweakvaldict.py b/rpython/rlib/_rweakvaldict.py
--- a/rpython/rlib/_rweakvaldict.py
+++ b/rpython/rlib/_rweakvaldict.py
@@ -76,12 +76,16 @@
             bk = self.rtyper.annotator.bookkeeper
             classdef = bk.getuniqueclassdef(weakdict._valueclass)
             r_value = getinstancerepr(self.rtyper, classdef)
+            any_value = False
             for dictkey, dictvalue in weakdict._dict.items():
                 llkey = self.r_key.convert_const(dictkey)
                 llvalue = r_value.convert_const(dictvalue)
                 if llvalue:
                     llvalue = lltype.cast_pointer(rclass.OBJECTPTR, llvalue)
                     self.ll_set_nonnull(l_dict, llkey, llvalue)
+                    any_value = True
+            if any_value:
+                l_dict.resize_counter = -1
             return l_dict
 
     def rtype_method_get(self, hop):
@@ -114,6 +118,8 @@
 
     @jit.dont_look_inside
     def ll_get(self, d, llkey):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         #llop.debug_print(lltype.Void, i, 'get')
@@ -132,6 +138,8 @@
 
     @jit.dont_look_inside
     def ll_set_nonnull(self, d, llkey, llvalue):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         valueref = weakref_create(llvalue)    # GC effects here, before the rest
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
@@ -147,6 +155,8 @@
 
     @jit.dont_look_inside
     def ll_set_null(self, d, llkey):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         if d.entries.everused(i):
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -441,7 +441,7 @@
             except OSError as e:
                 os.write(2, "Could not start GDB: %s" % (
                     os.strerror(e.errno)))
-                raise SystemExit
+                os._exit(1)
         else:
             time.sleep(1)  # give the GDB time to attach
 
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -465,8 +465,14 @@
 
     Note that this can return 0 or -1 too.
 
-    It returns the same number, both before and after translation.
-    Dictionaries don't need to be rehashed after translation.
+    NOTE: It returns a different number before and after translation!
+    Dictionaries will be rehashed when the translated program starts.
+    Be careful about other places that store or depend on a hash value:
+    if such a place can exist before translation, you should add for
+    example a _cleanup_() method to clear this cache during translation.
+
+    (Nowadays we could completely remove compute_hash() and decide that
+    hash(x) is valid RPython instead, at least for the types listed here.)
     """
     if isinstance(x, (str, unicode)):
         return _hash_string(x)
@@ -484,17 +490,11 @@
     """RPython equivalent of object.__hash__(x).  This returns the
     so-called 'identity hash', which is the non-overridable default hash
     of Python.  Can be called for any RPython-level object that turns
-    into a GC object, but not NULL.  The value is not guaranteed to be the
-    same before and after translation, except for RPython instances on the
-    lltypesystem.
+    into a GC object, but not NULL.  The value will be different before
+    and after translation (WARNING: this is a change with older RPythons!)
     """
     assert x is not None
-    result = object.__hash__(x)
-    try:
-        x.__dict__['__precomputed_identity_hash'] = result
-    except (TypeError, AttributeError):
-        pass
-    return result
+    return object.__hash__(x)
 
 def compute_unique_id(x):
     """RPython equivalent of id(x).  The 'x' must be an RPython-level
@@ -519,21 +519,17 @@
 
 # ----------
 
-HASH_ALGORITHM = "rpython"  # XXX Is there a better name?
-HASH_ALGORITHM_FIXED = False
+def _hash_string(s):
+    """The default algorithm behind compute_hash() for a string or a unicode.
+    This is a modified Fowler-Noll-Vo (FNV) hash.  According to Wikipedia,
+    FNV needs carefully-computed constants called FNV primes and FNV offset
+    basis, which are absent from the present algorithm.  Nevertheless,
+    this matches CPython 2.7 without -R, which has proven a good hash in
+    practice (even if not crypographical nor randomizable).
 
- at not_rpython
-def set_hash_algorithm(algo):
-    """Must be called very early, before any string is hashed with
-    compute_hash()!"""
-    global HASH_ALGORITHM
-    if HASH_ALGORITHM != algo:
-        assert not HASH_ALGORITHM_FIXED, "compute_hash() already called!"
-        assert algo in ("rpython", "siphash24")
-        HASH_ALGORITHM = algo
-
-
-def _hash_string_rpython(s):
+    There is a mechanism to use another one in programs after translation.
+    See rsiphash.py, which implements the algorithm of CPython >= 3.4.
+    """
     from rpython.rlib.rarithmetic import intmask
 
     length = len(s)
@@ -547,100 +543,8 @@
     x ^= length
     return intmask(x)
 
-
- at not_rpython
-def _hash_string_siphash24(s):
-    """This version is called when untranslated only."""
-    import array
-    from rpython.rlib.rsiphash import siphash24
-    from rpython.rtyper.lltypesystem import lltype, rffi
-    from rpython.rlib.rarithmetic import intmask
-
-    if not isinstance(s, str):
-        if isinstance(s, unicode):
-            lst = map(ord, s)
-        else:
-            lst = map(ord, s.chars)    # for rstr.STR or UNICODE
-        # NOTE: a latin-1 unicode string must have the same hash as the
-        # corresponding byte string.
-        if all(n <= 0xFF for n in lst):
-            kind = "B"
-        elif rffi.sizeof(lltype.UniChar) == 4:
-            kind = "I"
-        else:
-            kind = "H"
-        s = array.array(kind, lst).tostring()
-    ptr = rffi.str2charp(s)
-    x = siphash24(ptr, len(s))
-    rffi.free_charp(ptr)
-    return intmask(x)
-
-def ll_hash_string_siphash24(ll_s):
-    """Called from lltypesystem/rstr.py.  'll_s' is a rstr.STR or UNICODE."""
-    from rpython.rlib.rsiphash import siphash24
-    from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
-    from rpython.rlib.rarithmetic import intmask
-
-    length = len(ll_s.chars)
-    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:
-        # no GC operation from here!
-        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
-    else:
-        # NOTE: a latin-1 unicode string must have the same hash as the
-        # corresponding byte string.  If the unicode is all within
-        # 0-255, then we need to allocate a byte buffer and copy the
-        # latin-1 encoding in it manually.
-        for i in range(length):
-            if ord(ll_s.chars[i]) > 0xFF:
-                # no GC operation from here!
-                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
-                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
-                break
-        else:
-            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
-            i = 0
-            while i < length:
-                p[i] = chr(ord(ll_s.chars[i]))
-                i += 1
-            x = siphash24(llmemory.cast_ptr_to_adr(p), length)
-            lltype.free(p, flavor='raw')
-            return intmask(x)
-    x = siphash24(addr, length)
-    keepalive_until_here(ll_s)
-    return intmask(x)
-ll_hash_string_siphash24._jit_look_inside_ = False
-
-
- at not_rpython
-def _hash_string(s):
-    """The algorithm behind compute_hash() for a string or a unicode.
-    This version is only for untranslated usage, and 's' is a str or unicode.
-    """
-    global HASH_ALGORITHM_FIXED
-    HASH_ALGORITHM_FIXED = True
-    if HASH_ALGORITHM == "rpython":
-        return _hash_string_rpython(s)
-    if HASH_ALGORITHM == "siphash24":
-        return _hash_string_siphash24(s)
-    raise NotImplementedError
-
 def ll_hash_string(ll_s):
-    """The algorithm behind compute_hash() for a string or a unicode.
-    This version is called from lltypesystem/rstr.py, and 'll_s' is a
-    rstr.STR or rstr.UNICODE.
-    """
-    if not we_are_translated():
-        global HASH_ALGORITHM_FIXED
-        HASH_ALGORITHM_FIXED = True
-    if HASH_ALGORITHM == "rpython":
-        return _hash_string_rpython(ll_s.chars)
-    if HASH_ALGORITHM == "siphash24":
-        if we_are_translated():
-            return ll_hash_string_siphash24(ll_s)
-        else:
-            return _hash_string_siphash24(ll_s)
-    raise NotImplementedError
-
+    return _hash_string(ll_s.chars)
 
 def _hash_float(f):
     """The algorithm behind compute_hash() for a float.
@@ -698,6 +602,21 @@
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
+    _about_ = ll_hash_string
+    # this is only used when annotating the code in rstr.py, and so
+    # it always occurs after the RPython program signalled its intent
+    # to use a different hash.  The code below overwrites the use of
+    # ll_hash_string() to make the annotator think a possibly different
+    # function was called.
+
+    def compute_annotation(self):
+        from rpython.annotator import model as annmodel
+        bk = self.bookkeeper
+        translator = bk.annotator.translator
+        fn = getattr(translator, 'll_hash_string', ll_hash_string)
+        return annmodel.SomePBC([bk.getdesc(fn)])
+
+class Entry(ExtRegistryEntry):
     _about_ = compute_identity_hash
 
     def compute_result_annotation(self, s_x):
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -1,12 +1,24 @@
-import sys, os, struct
+"""
+This module implements siphash-2-4, the hashing algorithm for strings
+and unicodes.  You can use it explicitly by calling siphash24() with
+a byte string, or you can use enable_siphash24() to enable the use
+of siphash-2-4 on all RPython strings and unicodes in your program
+after translation.
+"""
+import sys, os, errno
 from contextlib import contextmanager
-from rpython.rlib import rarithmetic
+from rpython.rlib import rarithmetic, rurandom
 from rpython.rlib.objectmodel import not_rpython, always_inline
-from rpython.rlib.rgc import no_collect
-from rpython.rlib.rarithmetic import r_uint64
+from rpython.rlib.objectmodel import we_are_translated, dont_inline
+from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib import rgc, jit, rposix
+from rpython.rlib.rarithmetic import r_uint64, r_uint32, r_uint
 from rpython.rlib.rawstorage import misaligned_is_fine
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rlib.nonconst import NonConstant
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.rtyper.annlowlevel import llhelper
 
 
 if sys.byteorder == 'little':
@@ -16,37 +28,166 @@
     _le64toh = rarithmetic.byteswap
 
 
-# Initialize the values of the secret seed: two 64-bit constants.
-# CPython picks a new seed every time 'python' starts.  PyPy cannot do
-# that as easily because many details may rely on getting the same hash
-# value before and after translation.  We can, however, pick a random
-# seed once per translation, which should already be quite good.
-#
-# XXX no, it is not: e.g. all Ubuntu installations of the same Ubuntu
-# would get the same seed.  That's not good enough.
+class Seed:
+    k0l = k1l = r_uint64(0)
+seed = Seed()
 
- at not_rpython
-def select_random_seed():
-    global k0, k1    # note: the globals k0, k1 are already byte-swapped
-    v0, v1 = struct.unpack("QQ", os.urandom(16))
-    k0 = r_uint64(v0)
-    k1 = r_uint64(v1)
 
-select_random_seed()
+def _decode64(s):
+    return (r_uint64(ord(s[0])) |
+            r_uint64(ord(s[1])) << 8 |
+            r_uint64(ord(s[2])) << 16 |
+            r_uint64(ord(s[3])) << 24 |
+            r_uint64(ord(s[4])) << 32 |
+            r_uint64(ord(s[5])) << 40 |
+            r_uint64(ord(s[6])) << 48 |
+            r_uint64(ord(s[7])) << 56)
+
+def select_random_seed(s):
+    """'s' is a string of length 16"""
+    seed.k0l = _decode64(s)
+    seed.k1l = _decode64(s[8:16])
+
+
+random_ctx = rurandom.init_urandom()
+strtoul = rffi.llexternal("strtoul", [rffi.CCHARP, rffi.CCHARPP, rffi.INT],
+                          rffi.ULONG, save_err=rffi.RFFI_SAVE_ERRNO)
+
+env_var_name = "PYTHONHASHSEED"
+
+def initialize_from_env():
+    # This uses the same algorithms as CPython 3.5.  The environment
+    # variable we read also defaults to "PYTHONHASHSEED".  If needed,
+    # a different RPython interpreter can patch the value of the
+    # global variable 'env_var_name', or just pass a different init
+    # function to enable_siphash24().
+    value = os.environ.get(env_var_name)
+    if value and value != "random":
+        with rffi.scoped_view_charp(value) as ptr:
+            with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as endptr:
+                endptr[0] = ptr
+                seed = strtoul(ptr, endptr, 10)
+                full = endptr[0][0] == '\x00'
+        seed = lltype.cast_primitive(lltype.Unsigned, seed)
+        if not full or seed > r_uint(4294967295) or (
+            rposix.get_saved_errno() == errno.ERANGE and
+            seed == lltype.cast_primitive(lltype.Unsigned,
+                                          rffi.cast(rffi.ULONG, -1))):
+            os.write(2,
+                "PYTHONHASHSEED must be \"random\" or an integer "
+                "in range [0; 4294967295]\n")
+            os._exit(1)
+        if not seed:
+            # disable the randomized hash
+            s = '\x00' * 16
+        else:
+            s = lcg_urandom(seed)
+    else:
+        try:
+            s = rurandom.urandom(random_ctx, 16)
+        except Exception as e:
+            os.write(2,
+                "%s: failed to get random numbers to initialize Python\n" %
+                (e.__class__.__name__,))
+            os._exit(1)
+            raise   # makes the annotator happy
+    select_random_seed(s)
+
+def lcg_urandom(x):
+    s = ''
+    for index in range(16):
+        x *= 214013
+        x += 2531011
+        s += chr((x >> 16) & 0xff)
+    return s
+
+
+_FUNC = lltype.Ptr(lltype.FuncType([], lltype.Void))
+
+def enable_siphash24(*init):
+    """
+    Enable the use of siphash-2-4 for all RPython strings and unicodes
+    in the translated program.  You must call this function anywhere
+    from your interpreter (from a place that is annotated).  Optionally,
+    you can pass a function to call to initialize the state; the default
+    is 'initialize_from_env' above.  Don't call this more than once.
+    """
+    _internal_enable_siphash24()
+    if init:
+        (init_func,) = init
+    else:
+        init_func = initialize_from_env
+    if NonConstant(0):
+        init_func()   # pre-annotate it
+    llop.call_at_startup(lltype.Void, llhelper(_FUNC, init_func))
+
+def _internal_enable_siphash24():
+    pass
+
+class Entry(ExtRegistryEntry):
+    _about_ = _internal_enable_siphash24
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        if hasattr(translator, 'll_hash_string'):
+            assert translator.ll_hash_string == ll_hash_string_siphash24
+        else:
+            translator.ll_hash_string = ll_hash_string_siphash24
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+
+ at rgc.no_collect
+def ll_hash_string_siphash24(ll_s):
+    """Called indirectly from lltypesystem/rstr.py, by redirection from
+    objectmodel.ll_string_hash().
+    """
+    from rpython.rlib.rarithmetic import intmask
+
+    # This function is entirely @rgc.no_collect.
+    length = len(ll_s.chars)
+    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:   # regular STR
+        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
+    else:
+        # NOTE: a latin-1 unicode string must have the same hash as the
+        # corresponding byte string.  If the unicode is all within
+        # 0-255, then we need to allocate a byte buffer and copy the
+        # latin-1 encoding in it manually.  Note also that we give a
+        # different hash result than CPython on ucs4 platforms, for
+        # unicode strings where CPython uses 2 bytes per character.
+        for i in range(length):
+            if ord(ll_s.chars[i]) > 0xFF:
+                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
+                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
+                break
+        else:
+            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
+            i = 0
+            while i < length:
+                p[i] = chr(ord(ll_s.chars[i]))
+                i += 1
+            x = _siphash24(llmemory.cast_ptr_to_adr(p), length)
+            lltype.free(p, flavor='raw')
+            return intmask(x)
+    x = _siphash24(addr, length)
+    keepalive_until_here(ll_s)
+    return intmask(x)
+
 
 @contextmanager
 def choosen_seed(new_k0, new_k1, test_misaligned_path=False):
-    global k0, k1, misaligned_is_fine
-    old = k0, k1, misaligned_is_fine
-    k0 = _le64toh(r_uint64(new_k0))
-    k1 = _le64toh(r_uint64(new_k1))
+    """For tests."""
+    global misaligned_is_fine
+    old = seed.k0l, seed.k1l, misaligned_is_fine
+    seed.k0l = _le64toh(r_uint64(new_k0))
+    seed.k1l = _le64toh(r_uint64(new_k1))
     if test_misaligned_path:
         misaligned_is_fine = False
     yield
-    k0, k1, misaligned_is_fine = old
+    seed.k0l, seed.k1l, misaligned_is_fine = old
 
 def get_current_seed():
-    return _le64toh(k0), _le64toh(k1)
+    return _le64toh(seed.k0l), _le64toh(seed.k1l)
 
 
 magic0 = r_uint64(0x736f6d6570736575)
@@ -77,20 +218,21 @@
     return v0, v1, v2, v3
 
 
- at no_collect
-def siphash24(addr_in, size):
+ at rgc.no_collect
+def _siphash24(addr_in, size):
     """Takes an address pointer and a size.  Returns the hash as a r_uint64,
     which can then be casted to the expected type."""
 
-    direct = (misaligned_is_fine or
-                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
-
+    k0 = seed.k0l
+    k1 = seed.k1l
     b = r_uint64(size) << 56
     v0 = k0 ^ magic0
     v1 = k1 ^ magic1
     v2 = k0 ^ magic2
     v3 = k1 ^ magic3
 
+    direct = (misaligned_is_fine or
+                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
     index = 0
     if direct:
         while size >= 8:
@@ -113,7 +255,6 @@
                 r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48 |
                 r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 7)) << 56
             )
-            mi = _le64toh(mi)
             size -= 8
             index += 8
             v3 ^= mi
@@ -158,3 +299,13 @@
     v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
 
     return (v0 ^ v1) ^ (v2 ^ v3)
+
+
+ at jit.dont_look_inside
+def siphash24(s):
+    """'s' is a normal string.  Returns its siphash-2-4 as a r_uint64.
+    Don't forget to cast the result to a regular integer if needed,
+    e.g. with rarithmetic.intmask().
+    """
+    with rffi.scoped_nonmovingbuffer(s) as p:
+        return _siphash24(llmemory.cast_ptr_to_adr(p), len(s))
diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -57,6 +57,8 @@
                              immortal=True, zero=True)
 
     def urandom(context, n, signal_checker=None):
+        # NOTE: no dictionaries here: rsiphash24 calls this to
+        # initialize the random seed of string hashes
         provider = context[0]
         if not provider:
             # This handle is never explicitly released. The operating
@@ -139,6 +141,8 @@
 
     def urandom(context, n, signal_checker=None):
         "Read n bytes from /dev/urandom."
+        # NOTE: no dictionaries here: rsiphash24 calls this to
+        # initialize the random seed of string hashes
         result = []
         if SYS_getrandom is not None:
             n = _getrandom(n, result, signal_checker)
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -166,7 +166,6 @@
     foo = Foo()
     h = compute_hash(foo)
     assert h == object.__hash__(foo)
-    assert h == getattr(foo, '__precomputed_identity_hash')
     assert compute_hash(None) == 0
 
 def test_compute_hash_float():
@@ -182,7 +181,6 @@
     foo = Foo()
     h = compute_identity_hash(foo)
     assert h == object.__hash__(foo)
-    assert h == getattr(foo, '__precomputed_identity_hash')
 
 def test_compute_unique_id():
     from rpython.rlib.rarithmetic import intmask
@@ -410,36 +408,6 @@
         res = self.interpret(f, [])
         assert res == 1
 
-    def test_compute_hash_across_translation(self):
-        class Foo(object):
-            pass
-        q = Foo()
-
-        def f(i):
-            assert compute_hash(None) == 0
-            assert compute_hash(i) == h_42
-            assert compute_hash(i + 1.0) == h_43_dot_0
-            assert compute_hash((i + 3) / 6.0) == h_7_dot_5
-            assert compute_hash("Hello" + str(i)) == h_Hello42
-            if i == 42:
-                p = None
-            else:
-                p = Foo()
-            assert compute_hash(p) == h_None
-            assert compute_hash(("world", None, i, 7.5)) == h_tuple
-            assert compute_hash(q) == h_q
-            return i * 2
-        h_42 = compute_hash(42)
-        h_43_dot_0 = compute_hash(43.0)
-        h_7_dot_5 = compute_hash(7.5)
-        h_Hello42 = compute_hash("Hello42")
-        h_None = compute_hash(None)
-        h_tuple = compute_hash(("world", None, 42, 7.5))
-        h_q = compute_hash(q)
-
-        res = self.interpret(f, [42])
-        assert res == 84
-
     def test_fetch_translated_config(self):
         assert fetch_translated_config() is None
         def f():
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
--- a/rpython/rlib/test/test_rsiphash.py
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -1,5 +1,10 @@
-from rpython.rlib.rsiphash import siphash24, choosen_seed
+import os
+from rpython.rlib.rsiphash import siphash24, _siphash24, choosen_seed
+from rpython.rlib.rsiphash import initialize_from_env, enable_siphash24
+from rpython.rlib.objectmodel import compute_hash
+from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.lltypesystem import llmemory, rffi
+from rpython.translator.c.test.test_genc import compile
 
 
 CASES = [
@@ -28,13 +33,11 @@
 ]
 
 def check(s):
-    p = rffi.str2charp(s)
     q = rffi.str2charp('?' + s)
     with choosen_seed(0x8a9f065a358479f4, 0x11cb1e9ee7f40e1f,
                       test_misaligned_path=True):
-        x = siphash24(llmemory.cast_ptr_to_adr(p), len(s))
-        y = siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
-    rffi.free_charp(p)
+        x = siphash24(s)
+        y = _siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
     rffi.free_charp(q)
     assert x == y
     return x
@@ -42,3 +45,104 @@
 def test_siphash24():
     for expected, string in CASES:
         assert check(string) == expected
+
+def test_fix_seed():
+    old_val = os.environ.get('PYTHONHASHSEED', None)
+    try:
+        os.environ['PYTHONHASHSEED'] = '0'
+        initialize_from_env()
+        assert siphash24("foo") == 15988776847138518036
+        # value checked with CPython 3.5
+
+        os.environ['PYTHONHASHSEED'] = '4000000000'
+        initialize_from_env()
+        assert siphash24("foo") == 13829150778707464258
+        # value checked with CPython 3.5
+
+        for env in ['', 'random']:
+            os.environ['PYTHONHASHSEED'] = env
+            initialize_from_env()
+            hash1 = siphash24("foo")
+            initialize_from_env()
+            hash2 = siphash24("foo")
+            assert hash1 != hash2     # extremely unlikely
+    finally:
+        if old_val is None:
+            del os.environ['PYTHONHASHSEED']
+        else:
+            os.environ['PYTHONHASHSEED'] = old_val
+
+def test_translated():
+    d1 = {"foo": 123}
+    d2 = {u"foo": 456, u"\u1234\u5678": 789}
+    class G:
+        pass
+    g = G()
+    g.v1 = d1.copy()
+    g.v2 = d2.copy()
+
+    def fetch(n):
+        if n == 0: return d1.get("foo", -1)
+        if n == 1: return g.v1.get("foo", -1)
+        if n == 2: return compute_hash("foo")
+        if n == 3: return d2.get(u"foo", -1)
+        if n == 4: return g.v2.get(u"foo", -1)
+        if n == 5: return compute_hash(u"foo")
+        if n == 6: return d2.get(u"\u1234\u5678", -1)
+        if n == 7: return g.v2.get(u"\u1234\u5678", -1)
+        if n == 8: return compute_hash(u"\u1234\u5678")
+        assert 0
+
+    def entrypoint(n):
+        enable_siphash24()
+        g.v1["bar"] = -2
+        g.v2[u"bar"] = -2
+        if n >= 0:    # get items one by one, because otherwise it may
+                      # be the case that one line influences the next
+            return str(fetch(n))
+        else:
+            # ...except in random mode, because we want all results
+            # to be computed with the same seed
+            return ' '.join([str(fetch(n)) for n in range(9)])
+
+    fn = compile(entrypoint, [int])
+
+    def getall():
+        return [int(fn(i)) for i in range(9)]
+
+    old_val = os.environ.get('PYTHONHASHSEED', None)
+    try:
+        os.environ['PYTHONHASHSEED'] = '0'
+        s1 = getall()
+        assert s1[:8] == [
+            123, 123, intmask(15988776847138518036),
+            456, 456, intmask(15988776847138518036),
+            789, 789]
+        assert s1[8] in [intmask(17593683438421985039),    # ucs2 mode
+                         intmask(94801584261658677)]       # ucs4 mode
+
+        os.environ['PYTHONHASHSEED'] = '3987654321'
+        s1 = getall()
+        assert s1[:8] == [
+            123, 123, intmask(5890804383681474441),
+            456, 456, intmask(5890804383681474441),
+            789, 789]
+        assert s1[8] in [intmask(4192582507672183374),     # ucs2 mode
+                         intmask(7179255293164649778)]     # ucs4 mode
+
+        for env in ['', 'random']:
+            os.environ['PYTHONHASHSEED'] = env
+            s1 = map(int, fn(-1).split())
+            s2 = map(int, fn(-1).split())
+            assert s1[0:2]+s1[3:5]+s1[6:8] == [123, 123, 456, 456, 789, 789]
+            assert s1[2] == s1[5]
+            assert s2[0:2]+s2[3:5]+s2[6:8] == [123, 123, 456, 456, 789, 789]
+            assert s2[2] == s2[5]
+            #
+            assert len(set([s1[2], s2[2], s1[8], s2[8]])) == 4
+
+    finally:
+        if old_val is None:
+            del os.environ['PYTHONHASHSEED']
+        else:
+            os.environ['PYTHONHASHSEED'] = old_val
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -1,8 +1,16 @@
+import sys, os
+
+if __name__ == '__main__':
+    # hack for test_translation_prebuilt_2()
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__),
+                                    '..', '..', '..'))
+
 import py
 from rpython.annotator.model import UnionError
-from rpython.rlib import rgc
+from rpython.rlib import rgc, nonconst
 from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rtyper.test.test_llinterp import interpret
+from rpython.translator.c.test.test_genc import compile
 
 class X(object):
     pass
@@ -213,3 +221,40 @@
         assert d.get(keys[3]) is None
     f()
     interpret(f, [])
+
+def test_translation_prebuilt_1():
+    class K:
+        pass
+    d = RWeakValueDictionary(K, X)
+    k1 = K(); k2 = K()
+    x1 = X(); x2 = X()
+    d.set(k1, x1)
+    d.set(k2, x2)
+    def f():
+        assert d.get(k1) is x1
+        assert d.get(k2) is x2
+    f()
+    fc = compile(f, [], gcpolicy="boehm", rweakref=True)
+    fc()
+
+def _test_translation_prebuilt_2():
+    from rpython.rlib import objectmodel
+    objectmodel.set_hash_algorithm("siphash24")
+    d = RWeakValueDictionary(str, X)
+    k1 = "key1"; k2 = "key2"
+    x1 = X(); x2 = X()
+    d.set(k1, x1)
+    d.set(k2, x2)
+    def f():
+        i = nonconst.NonConstant(1)
+        assert d.get("key%d" % (i,)) is x1
+        assert d.get("key%d" % (i+1,)) is x2
+    fc = compile(f, [], gcpolicy="boehm", rweakref=True)
+    fc()
+
+def test_translation_prebuilt_2():
+    import subprocess
+    subprocess.check_call([sys.executable, __file__])
+
+if __name__ == "__main__":
+    _test_translation_prebuilt_2()
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -513,6 +513,9 @@
     # __________________________________________________________
     # misc LL operation implementations
 
+    def op_call_at_startup(self, *args):
+        pass
+
     def op_debug_view(self, *ll_objects):
         from rpython.translator.tool.lltracker import track
         track(*ll_objects)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -539,6 +539,7 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
+    'call_at_startup':      LLOp(),
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1380,20 +1380,11 @@
             return callb(*args)
         raise TypeError("%r instance is not a function" % (self._T,))
 
-    def _identityhash(self, cache=True):
+    def _identityhash(self):
         p = normalizeptr(self)
-        try:
-            return p._obj._hash_cache_
-        except AttributeError:
-            assert self._T._gckind == 'gc'
-            assert self      # not for NULL
-            result = hash(p._obj)
-            if cache:
-                try:
-                    p._obj._hash_cache_ = result
-                except AttributeError:
-                    pass
-            return result
+        assert self._T._gckind == 'gc'
+        assert self      # not for NULL
+        return hash(p._obj)
 
 class _ptr(_abstract_ptr):
     __slots__ = ('_TYPE',
@@ -1759,7 +1750,7 @@
 class _struct(_parentable):
     _kind = "structure"
 
-    __slots__ = ('_hash_cache_', '_compilation_info')
+    __slots__ = ('_compilation_info',)
 
     def __new__(self, TYPE, n=None, initialization=None, parent=None,
                 parentindex=None):
@@ -2442,24 +2433,6 @@
     return SomeInteger()
 
 
-def identityhash_nocache(p):
-    """Version of identityhash() to use from backends that don't care about
-    caching."""
-    assert p
-    return p._identityhash(cache=False)
-
-def init_identity_hash(p, value):
-    """For a prebuilt object p, initialize its hash value to 'value'."""
-    assert isinstance(typeOf(p), Ptr)
-    p = normalizeptr(p)
-    if not p:
-        raise ValueError("cannot change hash(NULL)!")
-    if hasattr(p._obj, '_hash_cache_'):
-        raise ValueError("the hash of %r was already computed" % (p,))
-    if typeOf(p).TO._is_varsize():
-        raise ValueError("init_identity_hash(): not for varsized types")
-    p._obj._hash_cache_ = intmask(value)
-
 def isCompatibleType(TYPE1, TYPE2):
     return TYPE1._is_compatible(TYPE2)
 
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -236,21 +236,14 @@
                 if self.r_rdict_hashfn.lowleveltype != lltype.Void:
                     l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash)
                     l_dict.fnkeyhash = l_fn
-
-                for dictkeycontainer, dictvalue in dictobj._dict.items():
-                    llkey = r_key.convert_const(dictkeycontainer.key)
-                    llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        dictkeycontainer.hash)
-                return l_dict
-
+                any_items = dictobj._dict.items()
             else:
-                for dictkey, dictvalue in dictobj.items():
-                    llkey = r_key.convert_const(dictkey)
-                    llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        l_dict.keyhash(llkey))
-                return l_dict
+                any_items = dictobj.items()
+            if any_items:
+                raise TyperError("found a prebuilt, explicitly non-ordered, "
+                                 "non-empty dict.  it would require additional"
+                                 " support to rehash it at program start-up")
+            return l_dict
 
     def rtype_len(self, hop):
         v_dict, = hop.inputargs(self)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -829,7 +829,7 @@
         return assert_str0(charpsize2str(cp, size))
     charp2str._annenforceargs_ = [lltype.SomePtr(TYPEP)]
 
-    # str -> char*, bool, bool
+    # str -> char*, flag
     # Can't inline this because of the raw address manipulation.
     @jit.dont_look_inside
     def get_nonmovingbuffer(data):
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib import objectmodel, jit, rgc, types
 from rpython.rlib.signature import signature
-from rpython.rlib.objectmodel import specialize, likely
+from rpython.rlib.objectmodel import specialize, likely, not_rpython
 from rpython.rtyper.debug import ll_assert
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rtyper import rmodel
@@ -46,20 +46,23 @@
 @jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
 @jit.oopspec('ordereddict.lookup(d, key, hash, flag)')
 def ll_call_lookup_function(d, key, hash, flag):
-    fun = d.lookup_function_no & FUNC_MASK
-    # This likely() here forces gcc to compile the check for fun == FUNC_BYTE
-    # first.  Otherwise, this is a regular switch and gcc (at least 4.7)
-    # compiles this as a series of checks, with the FUNC_BYTE case last.
-    # It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
-    if likely(fun == FUNC_BYTE):
-        return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
-    elif fun == FUNC_SHORT:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
-    elif IS_64BIT and fun == FUNC_INT:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
-    elif fun == FUNC_LONG:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
-    assert False
+    while True:
+        fun = d.lookup_function_no & FUNC_MASK
+        # This likely() here forces gcc to compile the check for fun==FUNC_BYTE
+        # first.  Otherwise, this is a regular switch and gcc (at least 4.7)
+        # compiles this as a series of checks, with the FUNC_BYTE case last.
+        # It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
+        if likely(fun == FUNC_BYTE):
+            return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
+        elif fun == FUNC_SHORT:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
+        elif IS_64BIT and fun == FUNC_INT:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
+        elif fun == FUNC_LONG:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
+        else:
+            ll_dict_create_initial_index(d)
+            # then, retry
 
 def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
                 ll_fasthash_function=None, ll_hash_function=None,
@@ -235,6 +238,7 @@
             self.setup()
             self.setup_final()
             l_dict = ll_newdict_size(self.DICT, len(dictobj))
+            ll_no_initial_index(l_dict)
             self.dict_cache[key] = l_dict
             r_key = self.key_repr
             if r_key.lowleveltype == llmemory.Address:
@@ -252,16 +256,14 @@
                 for dictkeycontainer, dictvalue in dictobj._dict.items():
                     llkey = r_key.convert_const(dictkeycontainer.key)
                     llvalue = r_value.convert_const(dictvalue)
-                    _ll_dict_insertclean(l_dict, llkey, llvalue,
-                                         dictkeycontainer.hash)
+                    _ll_dict_insert_no_index(l_dict, llkey, llvalue)
                 return l_dict
 
             else:
                 for dictkey, dictvalue in dictobj.items():
                     llkey = r_key.convert_const(dictkey)
                     llvalue = r_value.convert_const(dictvalue)
-                    _ll_dict_insertclean(l_dict, llkey, llvalue,
-                                         l_dict.keyhash(llkey))
+                    _ll_dict_insert_no_index(l_dict, llkey, llvalue)
                 return l_dict
 
     def rtype_len(self, hop):
@@ -458,17 +460,30 @@
 
 IS_64BIT = sys.maxint != 2 ** 31 - 1
 
-FUNC_SHIFT = 2
-FUNC_MASK  = 0x03  # two bits
 if IS_64BIT:
-    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(4)
+    FUNC_SHIFT = 3
+    FUNC_MASK  = 0x07  # three bits
+    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG, FUNC_MUST_REINDEX = range(5)
 else:
-    FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(3)
+    FUNC_SHIFT = 2
+    FUNC_MASK  = 0x03  # two bits
+    FUNC_BYTE, FUNC_SHORT, FUNC_LONG, FUNC_MUST_REINDEX = range(4)
 TYPE_BYTE  = rffi.UCHAR
 TYPE_SHORT = rffi.USHORT
 TYPE_INT   = rffi.UINT
 TYPE_LONG  = lltype.Unsigned
 
+def ll_no_initial_index(d):
+    # Used when making new empty dicts, and when translating prebuilt dicts.
+    # Remove the index completely.  A dictionary must always have an
+    # index unless it is freshly created or freshly translated.  Most
+    # dict operations start with ll_call_lookup_function(), which will
+    # recompute the hashes and create the index.
+    ll_assert(d.num_live_items == d.num_ever_used_items, 
+         "ll_no_initial_index(): dict already in use")
+    d.lookup_function_no = FUNC_MUST_REINDEX
+    d.indexes = lltype.nullptr(llmemory.GCREF.TO)
+
 def ll_malloc_indexes_and_choose_lookup(d, n):
     # keep in sync with ll_clear_indexes() below
     if n <= 256:
@@ -518,6 +533,8 @@
     elif fun == FUNC_LONG:
         ll_dict_store_clean(d, hash, i, TYPE_LONG)
     else:
+        # can't be still FUNC_MUST_REINDEX here
+        ll_assert(False, "ll_call_insert_clean_function(): invalid lookup_fun")
         assert False
 
 def ll_call_delete_by_entry_index(d, hash, i):
@@ -531,6 +548,8 @@
     elif fun == FUNC_LONG:
         ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
     else:
+        # can't be still FUNC_MUST_REINDEX here
+        ll_assert(False, "ll_call_delete_by_entry_index(): invalid lookup_fun")
         assert False
 
 def ll_valid_from_flag(entries, i):
@@ -648,15 +667,14 @@
     ll_dict_reindex(d, _ll_len_of_d_indexes(d))
 _ll_dict_rescue._dont_inline_ = True
 
-def _ll_dict_insertclean(d, key, value, hash):
+ at not_rpython
+def _ll_dict_insert_no_index(d, key, value):
     # never translated
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
     entry = d.entries[d.num_ever_used_items]
     entry.key = key
     entry.value = value
-    if hasattr(ENTRY, 'f_hash'):
-        entry.f_hash = hash
+    # note that f_hash is left uninitialized in prebuilt dicts
     if hasattr(ENTRY, 'f_valid'):
         entry.f_valid = True
     d.num_ever_used_items += 1
@@ -811,12 +829,13 @@
         # also possible that there are more dead items immediately behind the
         # last one, we reclaim all the dead items at the end of the ordereditem
         # at the same point.
-        i = d.num_ever_used_items - 2
-        while i >= 0 and not d.entries.valid(i):
+        i = index
+        while True:
             i -= 1
-        j = i + 1
-        assert j >= 0
-        d.num_ever_used_items = j
+            assert i >= 0
+            if d.entries.valid(i):    # must be at least one
+                break
+        d.num_ever_used_items = i + 1
 
     # If the dictionary is at least 87.5% dead items, then consider shrinking
     # it.
@@ -844,6 +863,50 @@
     else:
         ll_dict_reindex(d, new_size)
 
+def ll_ensure_indexes(d):
+    num = d.lookup_function_no
+    if num == FUNC_MUST_REINDEX:
+        ll_dict_create_initial_index(d)
+    else:
+        ll_assert((num & FUNC_MASK) != FUNC_MUST_REINDEX,
+                  "bad combination in lookup_function_no")
+
+def ll_dict_create_initial_index(d):
+    """Create the initial index for a dictionary.  The common case is
+    that 'd' is empty.  The uncommon case is that it is a prebuilt
+    dictionary frozen by translation, in which case we must rehash all
+    entries.  The common case must be seen by the JIT.
+    """
+    if d.num_live_items == 0:
+        ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+        d.resize_counter = DICT_INITSIZE * 2
+    else:
+        ll_dict_rehash_after_translation(d)
+
+ at jit.dont_look_inside
+def ll_dict_rehash_after_translation(d):
+    assert d.num_live_items == d.num_ever_used_items
+    assert not d.indexes
+    #
+    # recompute all hashes.  Needed if they are stored in d.entries,
+    # but do it anyway: otherwise, e.g. a string-keyed dictionary
+    # won't have a fasthash on its strings if their hash is still
+    # uncomputed.
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    for i in range(d.num_ever_used_items):
+        assert d.entries.valid(i)
+        d_entry = d.entries[i]
+        h = d.keyhash(d_entry.key)
+        if hasattr(ENTRY, 'f_hash'):
+            d_entry.f_hash = h
+        #else: purely for the side-effect it can have on d_entry.key
+    #
+    # Use the smallest acceptable size for ll_dict_reindex
+    new_size = DICT_INITSIZE
+    while new_size * 2 - d.num_live_items * 3 <= 0:
+        new_size *= 2
+    ll_dict_reindex(d, new_size)
+
 def ll_dict_reindex(d, new_size):
     if bool(d.indexes) and _ll_len_of_d_indexes(d) == new_size:
         ll_clear_indexes(d, new_size)   # hack: we can reuse the same array
@@ -857,12 +920,33 @@
     entries = d.entries
     i = 0
     ibound = d.num_ever_used_items
-    while i < ibound:
-        if entries.valid(i):
-            hash = entries.hash(i)
-            ll_call_insert_clean_function(d, hash, i)
-        i += 1
-    #old_entries.delete() XXXX!
+    #
+    # Write four loops, moving the check for the value of 'fun' out of
+    # the loops.  A small speed-up over ll_call_insert_clean_function().
+    fun = d.lookup_function_no     # == lookup_function_no & FUNC_MASK
+    if fun == FUNC_BYTE:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_BYTE)
+            i += 1
+    elif fun == FUNC_SHORT:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_SHORT)
+            i += 1
+    elif IS_64BIT and fun == FUNC_INT:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_INT)
+            i += 1
+    elif fun == FUNC_LONG:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_LONG)
+            i += 1
+    else:
+        assert False
+
 
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
@@ -1013,10 +1097,11 @@
 def ll_newdict(DICT):
     d = DICT.allocate()
     d.entries = _ll_empty_array(DICT)
-    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    # Don't allocate an 'indexes' for empty dict.  It seems a typical
+    # program contains tons of empty dicts, so this might be a memory win.
     d.num_live_items = 0
     d.num_ever_used_items = 0
-    d.resize_counter = DICT_INITSIZE * 2
+    ll_no_initial_index(d)
     return d
 OrderedDictRepr.ll_newdict = staticmethod(ll_newdict)
 
@@ -1101,6 +1186,10 @@
                 # as soon as we do something like ll_dict_reindex().
                 if index == (dict.lookup_function_no >> FUNC_SHIFT):
                     dict.lookup_function_no += (1 << FUNC_SHIFT)
+                # note that we can't have modified a FUNC_MUST_REINDEX
+                # dict here because such dicts have no invalid entries
+                ll_assert((dict.lookup_function_no & FUNC_MASK) !=
+                      FUNC_MUST_REINDEX, "bad combination in _ll_dictnext")
             index = nextindex
         # clear the reference to the dict and prevent restarts
         iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
@@ -1146,6 +1235,8 @@
         return dict.entries[index].value
 
 def ll_dict_copy(dict):
+    ll_ensure_indexes(dict)
+
     DICT = lltype.typeOf(dict).TO
     newdict = DICT.allocate()
     newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
@@ -1180,6 +1271,10 @@
     DICT = lltype.typeOf(d).TO
     old_entries = d.entries
     d.entries = _ll_empty_array(DICT)
+    # note: we can't remove the index here, because it is possible that
+    # crazy Python code calls d.clear() from the method __eq__() called
+    # from ll_dict_lookup(d).  Instead, stick to the rule that once a
+    # dictionary has got an index, it will always have one.
     ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
     d.num_live_items = 0
     d.num_ever_used_items = 0
@@ -1190,6 +1285,7 @@
 def ll_dict_update(dic1, dic2):
     if dic1 == dic2:
         return
+    ll_ensure_indexes(dic2)    # needed for entries.hash() below
     ll_prepare_dict_update(dic1, dic2.num_live_items)
     i = 0
     while i < dic2.num_ever_used_items:
@@ -1216,6 +1312,7 @@
     # the case where dict.update() actually has a lot of collisions.
     # If num_extra is much greater than d.num_live_items the conditional_call
     # will trigger anyway, which is really the goal.
+    ll_ensure_indexes(d)
     x = num_extra - d.num_live_items
     jit.conditional_call(d.resize_counter <= x * 3,
                          _ll_dict_resize_to, d, num_extra)
@@ -1275,6 +1372,7 @@
     if dic.num_live_items == 0:
         raise KeyError
 
+    ll_ensure_indexes(dic)
     entries = dic.entries
 
     # find the last entry.  It's unclear if the loop below is still
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -1,9 +1,9 @@
 from weakref import WeakValueDictionary
 
 from rpython.annotator import model as annmodel
-from rpython.rlib import jit, types
+from rpython.rlib import jit, types, objectmodel
 from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated,
-    ll_hash_string, keepalive_until_here, specialize, enforceargs)
+    ll_hash_string, keepalive_until_here, specialize, enforceargs, dont_inline)
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.error import TyperError
@@ -383,6 +383,7 @@
             return 0
 
     @staticmethod
+    @dont_inline
     def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the
@@ -400,6 +401,7 @@
 
     @staticmethod
     def ll_strfasthash(s):
+        ll_assert(s.hash != 0, "ll_strfasthash: hash==0")
         return s.hash     # assumes that the hash is already computed
 
     @staticmethod
@@ -1258,7 +1260,8 @@
                               'gethash': LLHelpers.ll_strhash,
                               'length': LLHelpers.ll_length,
                               'find': LLHelpers.ll_find,
-                              'rfind': LLHelpers.ll_rfind}))
+                              'rfind': LLHelpers.ll_rfind},
+                    hints={'remove_hash': True}))
 UNICODE.become(GcStruct('rpy_unicode', ('hash', Signed),
                         ('chars', Array(UniChar, hints={'immutable': True})),
                         adtmeths={'malloc' : staticAdtMethod(mallocunicode),
@@ -1266,8 +1269,8 @@
                                   'copy_contents' : staticAdtMethod(copy_unicode_contents),
                                   'copy_contents_from_str' : staticAdtMethod(copy_unicode_contents),
                                   'gethash': LLHelpers.ll_strhash,
-                                  'length': LLHelpers.ll_length}
-                        ))
+                                  'length': LLHelpers.ll_length},
+                    hints={'remove_hash': True}))
 
 
 # TODO: make the public interface of the rstr module cleaner
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -749,22 +749,10 @@
     assert hash3 == identityhash(s3)
     assert hash3 == identityhash(s3.super)
     assert hash3 == identityhash(s3.super.super)
-    py.test.raises(ValueError, init_identity_hash, s3, hash3^1)
-    py.test.raises(ValueError, init_identity_hash, s3.super, hash3^4)
-    py.test.raises(ValueError, init_identity_hash, s3.super.super, hash3^9)
-
-    s3 = malloc(S3)
-    init_identity_hash(s3.super, -123)
-    assert -123 == identityhash(s3)
-    assert -123 == identityhash(s3.super)
-    assert -123 == identityhash(s3.super.super)
-    py.test.raises(ValueError, init_identity_hash, s3, 4313)
-    py.test.raises(ValueError, init_identity_hash, s3.super, 0)
-    py.test.raises(ValueError, init_identity_hash, s3.super.super, -124)
 
     from rpython.rtyper.lltypesystem import llmemory
     p3 = cast_opaque_ptr(llmemory.GCREF, s3)
-    assert -123 == identityhash(p3)
+    assert hash3 == identityhash(p3)
 
     A = GcArray(Signed)
     a = malloc(A, 3)
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -170,7 +170,6 @@
                             ('subclassrange_max', Signed),
                             ('rtti', Ptr(RuntimeTypeInfo)),
                             ('name', Ptr(rstr.STR)),
-                            ('hash', Signed),
                             ('instantiate', Ptr(FuncType([], OBJECTPTR))),
                             hints={'immutable': True}))
 # non-gc case
@@ -338,7 +337,6 @@
 
     def fill_vtable_root(self, vtable):
         """Initialize the head of the vtable."""
-        vtable.hash = hash(self)
         # initialize the 'subclassrange_*' and 'name' fields
         if self.classdef is not None:
             #vtable.parenttypeptr = self.rbase.getvtable()
@@ -785,7 +783,6 @@
     def initialize_prebuilt_instance(self, value, classdef, result):
         # must fill in the hash cache before the other ones
         # (see test_circular_hash_initialization)
-        self.initialize_prebuilt_hash(value, result)
         self._initialize_data_flattenrec(self.initialize_prebuilt_data,
                                          value, classdef, result)
 
@@ -943,11 +940,6 @@
             rclass = getclassrepr(self.rtyper, classdef)
             result.typeptr = rclass.getvtable()
 
-    def initialize_prebuilt_hash(self, value, result):
-        llattrvalue = getattr(value, '__precomputed_identity_hash', None)
-        if llattrvalue is not None:
-            lltype.init_identity_hash(result, llattrvalue)
-
     def getfieldrepr(self, attr):
         """Return the repr used for the given attribute."""
         if attr in self.fields:
diff --git a/rpython/rtyper/rfloat.py b/rpython/rtyper/rfloat.py
--- a/rpython/rtyper/rfloat.py
+++ b/rpython/rtyper/rfloat.py
@@ -26,6 +26,9 @@
     def get_ll_hash_function(self):
         return _hash_float
 
+    # no get_ll_fasthash_function: the hash is a bit slow, better cache
+    # it inside dict entries
+
     def rtype_bool(_, hop):

From pypy.commits at gmail.com  Mon Jan 30 12:35:11 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 30 Jan 2017 09:35:11 -0800 (PST)
Subject: [pypy-commit] pypy default: document cffi as a build-time
 requirement
Message-ID: <588f794f.2d88df0a.d95e3.1d3c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89847:31ec34667797
Date: 2017-01-30 17:34 +0000
http://bitbucket.org/pypy/pypy/changeset/31ec34667797/

Log:	document cffi as a build-time requirement

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -56,6 +56,9 @@
 .. _`arm document`: http://rpython.readthedocs.org/en/latest/arm.html
 .. _`RPython documentation`: http://rpython.readthedocs.org
 
+The host Python needs to have CFFI installed. If translating on PyPy, CFFI is
+already installed. If translating on CPython, you need to install it, e.g. 
+using ``pip install cffi``.
 
 To build PyPy on Unix using the C translation backend, you need at least a C
 compiler and ``make`` installed. Further, some optional modules have additional

From pypy.commits at gmail.com  Tue Jan 31 07:00:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 04:00:14 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: A more straightforward way to call
 initialize_from_env() from
Message-ID: <58907c4e.0a081c0a.f5bd4.611c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89848:38f25a06a8da
Date: 2017-01-31 11:00 +0100
http://bitbucket.org/pypy/pypy/changeset/38f25a06a8da/

Log:	A more straightforward way to call initialize_from_env() from
	RPython_StartupCode()

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -59,8 +59,8 @@
     # This uses the same algorithms as CPython 3.5.  The environment
     # variable we read also defaults to "PYTHONHASHSEED".  If needed,
     # a different RPython interpreter can patch the value of the
-    # global variable 'env_var_name', or just pass a different init
-    # function to enable_siphash24().
+    # global variable 'env_var_name', or just patch the whole
+    # initialize_from_env() function.
     value = os.environ.get(env_var_name)
     if value and value != "random":
         with rffi.scoped_view_charp(value) as ptr:
@@ -74,8 +74,8 @@
             seed == lltype.cast_primitive(lltype.Unsigned,
                                           rffi.cast(rffi.ULONG, -1))):
             os.write(2,
-                "PYTHONHASHSEED must be \"random\" or an integer "
-                "in range [0; 4294967295]\n")
+                "%s must be \"random\" or an integer "
+                "in range [0; 4294967295]\n" % (env_var_name,))
             os._exit(1)
         if not seed:
             # disable the randomized hash
@@ -104,28 +104,16 @@
 
 _FUNC = lltype.Ptr(lltype.FuncType([], lltype.Void))
 
-def enable_siphash24(*init):
+def enable_siphash24():
     """
     Enable the use of siphash-2-4 for all RPython strings and unicodes
     in the translated program.  You must call this function anywhere
-    from your interpreter (from a place that is annotated).  Optionally,
-    you can pass a function to call to initialize the state; the default
-    is 'initialize_from_env' above.  Don't call this more than once.
+    from your interpreter (from a place that is annotated).  Don't call
+    more than once.
     """
-    _internal_enable_siphash24()
-    if init:
-        (init_func,) = init
-    else:
-        init_func = initialize_from_env
-    if NonConstant(0):
-        init_func()   # pre-annotate it
-    llop.call_at_startup(lltype.Void, llhelper(_FUNC, init_func))
-
-def _internal_enable_siphash24():
-    pass
 
 class Entry(ExtRegistryEntry):
-    _about_ = _internal_enable_siphash24
+    _about_ = enable_siphash24
 
     def compute_result_annotation(self):
         translator = self.bookkeeper.annotator.translator
@@ -133,9 +121,19 @@
             assert translator.ll_hash_string == ll_hash_string_siphash24
         else:
             translator.ll_hash_string = ll_hash_string_siphash24
+        bk = self.bookkeeper
+        s_callable = bk.immutablevalue(initialize_from_env)
+        key = (enable_siphash24,)
+        bk.emulate_pbc_call(key, s_callable, [])
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+        bk = hop.rtyper.annotator.bookkeeper
+        s_callable = bk.immutablevalue(initialize_from_env)
+        r_callable = hop.rtyper.getrepr(s_callable)
+        ll_init = r_callable.get_unique_llfn().value
+        bk.annotator.translator._call_at_startup.append(ll_init)
+
 
 @rgc.no_collect
 def ll_hash_string_siphash24(ll_s):
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -156,6 +156,10 @@
         for obj in exports.EXPORTS_obj2name.keys():
             db.getcontainernode(obj)
         exports.clear()
+
+        for ll_func in db.translator._call_at_startup:
+            db.get(ll_func)
+
         db.complete()
 
         self.collect_compilation_info(db)
@@ -822,8 +826,8 @@
             for line in lines:
                 print >> f, '\t'+line
 
-    for extra in database.call_at_startup:
-        print >> f, '\t%s();\t/* call_at_startup */' % (extra,)
+    for ll_init in database.translator._call_at_startup:
+        print >> f, '\t%s();\t/* call_at_startup */' % (database.get(ll_init),)
 
     print >> f, '}'
 
diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py
--- a/rpython/translator/translator.py
+++ b/rpython/translator/translator.py
@@ -38,6 +38,7 @@
         self.graphs = []      # [graph]
         self.callgraph = {}   # {opaque_tag: (caller-graph, callee-graph)}
         self._prebuilt_graphs = {}   # only used by the pygame viewer
+        self._call_at_startup = []
 
     def buildflowgraph(self, func, mute_dot=False):
         """Get the flow graph for a function."""

From pypy.commits at gmail.com  Tue Jan 31 07:00:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 04:00:16 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Possible translation fix
Message-ID: <58907c50.b6a9df0a.3bf85.9c73@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89849:c115177531bb
Date: 2017-01-31 11:02 +0100
http://bitbucket.org/pypy/pypy/changeset/c115177531bb/

Log:	Possible translation fix

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -384,6 +384,7 @@
 
     @staticmethod
     @dont_inline
+    @jit.dont_look_inside
     def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the

From pypy.commits at gmail.com  Tue Jan 31 07:00:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 04:00:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-siphash24: hg merge rpython-hash
Message-ID: <58907c52.5190df0a.5d65e.97f2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-siphash24
Changeset: r89850:ecfcf2c386eb
Date: 2017-01-31 11:03 +0100
http://bitbucket.org/pypy/pypy/changeset/ecfcf2c386eb/

Log:	hg merge rpython-hash

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -59,8 +59,8 @@
     # This uses the same algorithms as CPython 3.5.  The environment
     # variable we read also defaults to "PYTHONHASHSEED".  If needed,
     # a different RPython interpreter can patch the value of the
-    # global variable 'env_var_name', or just pass a different init
-    # function to enable_siphash24().
+    # global variable 'env_var_name', or just patch the whole
+    # initialize_from_env() function.
     value = os.environ.get(env_var_name)
     if value and value != "random":
         with rffi.scoped_view_charp(value) as ptr:
@@ -74,8 +74,8 @@
             seed == lltype.cast_primitive(lltype.Unsigned,
                                           rffi.cast(rffi.ULONG, -1))):
             os.write(2,
-                "PYTHONHASHSEED must be \"random\" or an integer "
-                "in range [0; 4294967295]\n")
+                "%s must be \"random\" or an integer "
+                "in range [0; 4294967295]\n" % (env_var_name,))
             os._exit(1)
         if not seed:
             # disable the randomized hash
@@ -104,28 +104,16 @@
 
 _FUNC = lltype.Ptr(lltype.FuncType([], lltype.Void))
 
-def enable_siphash24(*init):
+def enable_siphash24():
     """
     Enable the use of siphash-2-4 for all RPython strings and unicodes
     in the translated program.  You must call this function anywhere
-    from your interpreter (from a place that is annotated).  Optionally,
-    you can pass a function to call to initialize the state; the default
-    is 'initialize_from_env' above.  Don't call this more than once.
+    from your interpreter (from a place that is annotated).  Don't call
+    more than once.
     """
-    _internal_enable_siphash24()
-    if init:
-        (init_func,) = init
-    else:
-        init_func = initialize_from_env
-    if NonConstant(0):
-        init_func()   # pre-annotate it
-    llop.call_at_startup(lltype.Void, llhelper(_FUNC, init_func))
-
-def _internal_enable_siphash24():
-    pass
 
 class Entry(ExtRegistryEntry):
-    _about_ = _internal_enable_siphash24
+    _about_ = enable_siphash24
 
     def compute_result_annotation(self):
         translator = self.bookkeeper.annotator.translator
@@ -133,9 +121,19 @@
             assert translator.ll_hash_string == ll_hash_string_siphash24
         else:
             translator.ll_hash_string = ll_hash_string_siphash24
+        bk = self.bookkeeper
+        s_callable = bk.immutablevalue(initialize_from_env)
+        key = (enable_siphash24,)
+        bk.emulate_pbc_call(key, s_callable, [])
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+        bk = hop.rtyper.annotator.bookkeeper
+        s_callable = bk.immutablevalue(initialize_from_env)
+        r_callable = hop.rtyper.getrepr(s_callable)
+        ll_init = r_callable.get_unique_llfn().value
+        bk.annotator.translator._call_at_startup.append(ll_init)
+
 
 @rgc.no_collect
 def ll_hash_string_siphash24(ll_s):
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -1,10 +1,3 @@
-import sys, os
-
-if __name__ == '__main__':
-    # hack for test_translation_prebuilt_2()
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__),
-                                    '..', '..', '..'))
-
 import py
 from rpython.annotator.model import UnionError
 from rpython.rlib import rgc, nonconst
@@ -238,23 +231,16 @@
     fc()
 
 def _test_translation_prebuilt_2():
-    from rpython.rlib import objectmodel
-    objectmodel.set_hash_algorithm("siphash24")
+    from rpython.rlib import rsiphash
     d = RWeakValueDictionary(str, X)
     k1 = "key1"; k2 = "key2"
     x1 = X(); x2 = X()
     d.set(k1, x1)
     d.set(k2, x2)
     def f():
+        rsiphash.enable_siphash24()
         i = nonconst.NonConstant(1)
         assert d.get("key%d" % (i,)) is x1
         assert d.get("key%d" % (i+1,)) is x2
     fc = compile(f, [], gcpolicy="boehm", rweakref=True)
     fc()
-
-def test_translation_prebuilt_2():
-    import subprocess
-    subprocess.check_call([sys.executable, __file__])
-
-if __name__ == "__main__":
-    _test_translation_prebuilt_2()
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -513,9 +513,6 @@
     # __________________________________________________________
     # misc LL operation implementations
 
-    def op_call_at_startup(self, *args):
-        pass
-
     def op_debug_view(self, *ll_objects):
         from rpython.translator.tool.lltracker import track
         track(*ll_objects)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -539,7 +539,7 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
-    'call_at_startup':      LLOp(),
+    'call_at_startup':      LLOp(canrun=True),
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -742,6 +742,9 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
+def op_call_at_startup(init_func):
+    pass    # do nothing
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -384,6 +384,7 @@
 
     @staticmethod
     @dont_inline
+    @jit.dont_look_inside
     def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -193,6 +193,7 @@
 
     def gen_block(self, block):
         if 1:      # (preserve indentation)
+            self._current_block = block
             myblocknum = self.blocknum[block]
             if block in self.inlinable_blocks:
                 # debug comment
@@ -942,7 +943,16 @@
                 self.expr(op.result))
 
     def OP_CALL_AT_STARTUP(self, op):
-        assert isinstance(op.args[0], Constant)
-        func = self.expr(op.args[0])
+        c = op.args[0]
+        if not isinstance(c, Constant):
+            # Bah, maybe it comes from a same_as(const) just before...
+            # Can occur if running without backendopts
+            for op1 in self._current_block.operations:
+                if op1.result is op.args[0]:
+                    assert op1.opname == "same_as"
+                    c = op1.args[0]
+                    break
+            assert isinstance(c, Constant)
+        func = self.expr(c)
         self.db.call_at_startup.add(func)
         return '/* call_at_startup %s */' % (func,)
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -156,6 +156,10 @@
         for obj in exports.EXPORTS_obj2name.keys():
             db.getcontainernode(obj)
         exports.clear()
+
+        for ll_func in db.translator._call_at_startup:
+            db.get(ll_func)
+
         db.complete()
 
         self.collect_compilation_info(db)
@@ -822,8 +826,8 @@
             for line in lines:
                 print >> f, '\t'+line
 
-    for extra in database.call_at_startup:
-        print >> f, '\t%s();\t/* call_at_startup */' % (extra,)
+    for ll_init in database.translator._call_at_startup:
+        print >> f, '\t%s();\t/* call_at_startup */' % (database.get(ll_init),)
 
     print >> f, '}'
 
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -2,11 +2,6 @@
 
 import math
 import sys, os
-
-if __name__ == '__main__':
-    # hack for test_hash_string_siphash24()
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__),
-                                    '..', '..', '..', '..'))
 import py
 
 from rpython.rlib.rstackovf import StackOverflow
@@ -604,8 +599,6 @@
         #    and not e.g. siphash24
 
     def _test_hash_string(self, algo):
-        from rpython.rlib import objectmodel
-        objectmodel.set_hash_algorithm(algo)
         s = "hello"
         u = u"world"
         v = u"\u1234\u2318+\u2bcd\u2102"
@@ -616,6 +609,9 @@
         assert hash_u == compute_hash("world")    #    a latin-1 unicode
         #
         def fn(length):
+            if algo == "siphash24":
+                from rpython.rlib import rsiphash
+                rsiphash.enable_siphash24()
             assert length >= 1
             return str((compute_hash(s),
                         compute_hash(u),
@@ -630,21 +626,23 @@
         f = self.getcompiled(fn, [int])
         res = f(5)
         res = [int(a) for a in res[1:-1].split(",")]
-        assert res[0] == hash_s
-        assert res[1] == hash_u
-        assert res[2] == hash_v
-        assert res[3] == hash_s
-        assert res[4] == hash_u
-        assert res[5] == hash_v
+        if algo == "fnv":
+            assert res[0] == hash_s
+            assert res[1] == hash_u
+            assert res[2] == hash_v
+        else:
+            assert res[0] != hash_s
+            assert res[1] != hash_u
+            assert res[2] != hash_v
+        assert res[3] == res[0]
+        assert res[4] == res[1]
+        assert res[5] == res[2]
 
-    def test_hash_string_rpython(self):
-        self._test_hash_string("rpython")
+    def test_hash_string_fnv(self):
+        self._test_hash_string("fnv")
 
     def test_hash_string_siphash24(self):
-        import subprocess
-        subprocess.check_call([sys.executable, __file__, "siphash24",
-                               self.__class__.__module__,
-                               self.__class__.__name__])
+        self._test_hash_string("siphash24")
 
     def test_list_basic_ops(self):
         def list_basic_ops(i, j):
@@ -945,11 +943,3 @@
         f = self.getcompiled(func, [int])
         res = f(2)
         assert res == 1     # and not 2
-
-
-if __name__ == '__main__':
-    # for test_hash_string_siphash24()
-    algo, clsmodule, clsname = sys.argv[1:]
-    mod = __import__(clsmodule, None, None, [clsname])
-    cls = getattr(mod, clsname)
-    cls()._test_hash_string(algo)
diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py
--- a/rpython/translator/translator.py
+++ b/rpython/translator/translator.py
@@ -38,6 +38,7 @@
         self.graphs = []      # [graph]
         self.callgraph = {}   # {opaque_tag: (caller-graph, callee-graph)}
         self._prebuilt_graphs = {}   # only used by the pygame viewer
+        self._call_at_startup = []
 
     def buildflowgraph(self, func, mute_dot=False):
         """Get the flow graph for a function."""

From pypy.commits at gmail.com  Tue Jan 31 07:00:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 04:00:20 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: translation fixes
Message-ID: <58907c54.545a1c0a.b4ae8.fc0c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89851:be1d112df191
Date: 2017-01-31 12:59 +0100
http://bitbucket.org/pypy/pypy/changeset/be1d112df191/

Log:	translation fixes

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -88,7 +88,7 @@
         except Exception as e:
             os.write(2,
                 "%s: failed to get random numbers to initialize Python\n" %
-                (e.__class__.__name__,))
+                (str(e),))
             os._exit(1)
             raise   # makes the annotator happy
     select_random_seed(s)
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -523,6 +523,7 @@
 
 @jit.dont_look_inside
 def ll_call_insert_clean_function(d, hash, i):
+    assert i >= 0
     fun = d.lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
         ll_dict_store_clean(d, hash, i, TYPE_BYTE)

From pypy.commits at gmail.com  Tue Jan 31 07:00:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 04:00:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-siphash24: hg merge rpython-hash
Message-ID: <58907c56.896f1c0a.a3a93.fca0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-siphash24
Changeset: r89852:abac58ea9c5b
Date: 2017-01-31 12:59 +0100
http://bitbucket.org/pypy/pypy/changeset/abac58ea9c5b/

Log:	hg merge rpython-hash

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -88,7 +88,7 @@
         except Exception as e:
             os.write(2,
                 "%s: failed to get random numbers to initialize Python\n" %
-                (e.__class__.__name__,))
+                (str(e),))
             os._exit(1)
             raise   # makes the annotator happy
     select_random_seed(s)
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -523,6 +523,7 @@
 
 @jit.dont_look_inside
 def ll_call_insert_clean_function(d, hash, i):
+    assert i >= 0
     fun = d.lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
         ll_dict_store_clean(d, hash, i, TYPE_BYTE)

From pypy.commits at gmail.com  Tue Jan 31 08:42:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 05:42:22 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: Test and fix. I'll switch to
 recommend 'rr' now...
Message-ID: <5890943e.496f1c0a.52e7f.1c65@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89853:e5916c9c44b3
Date: 2017-01-31 14:40 +0100
http://bitbucket.org/pypy/pypy/changeset/e5916c9c44b3/

Log:	Test and fix. I'll switch to recommend 'rr' now...

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -338,11 +338,15 @@
         return DictIteratorRepr(self, "items").newiter(hop)
 
     def rtype_method_iterkeys_with_hash(self, hop):
-        hop.exception_cannot_occur()
+        v_dic, = hop.inputargs(self)
+        hop.exception_is_here()
+        hop.gendirectcall(ll_ensure_indexes, v_dic)
         return DictIteratorRepr(self, "keys_with_hash").newiter(hop)
 
     def rtype_method_iteritems_with_hash(self, hop):
-        hop.exception_cannot_occur()
+        v_dic, = hop.inputargs(self)
+        hop.exception_is_here()
+        hop.gendirectcall(ll_ensure_indexes, v_dic)
         return DictIteratorRepr(self, "items_with_hash").newiter(hop)
 
     def rtype_method_clear(self, hop):
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -644,6 +644,25 @@
     def test_hash_string_siphash24(self):
         self._test_hash_string("siphash24")
 
+    def test_iterkeys_with_hash_on_prebuilt_dict(self):
+        from rpython.rlib import objectmodel
+        prebuilt_d = {"hello": 10, "world": 20}
+        #
+        def fn(n):
+            from rpython.rlib import rsiphash
+            rsiphash.enable_siphash24()
+            #assert str(n) not in prebuilt_d <- this made the test pass,
+            #       before the fix which was that iterkeys_with_hash()
+            #       didn't do the initial rehashing on its own
+            for key, h in objectmodel.iterkeys_with_hash(prebuilt_d):
+                print key, h
+                assert h == compute_hash(key)
+            return 42
+
+        f = self.getcompiled(fn, [int])
+        res = f(0)
+        assert res == 42
+
     def test_list_basic_ops(self):
         def list_basic_ops(i, j):
             l = [1, 2, 3]

From pypy.commits at gmail.com  Tue Jan 31 08:42:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 05:42:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-siphash24: hg merge rpython-hash
Message-ID: <58909440.8c7e1c0a.8fb6e.1aab@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-siphash24
Changeset: r89854:2c4a4e1bf21e
Date: 2017-01-31 14:41 +0100
http://bitbucket.org/pypy/pypy/changeset/2c4a4e1bf21e/

Log:	hg merge rpython-hash

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -338,11 +338,15 @@
         return DictIteratorRepr(self, "items").newiter(hop)
 
     def rtype_method_iterkeys_with_hash(self, hop):
-        hop.exception_cannot_occur()
+        v_dic, = hop.inputargs(self)
+        hop.exception_is_here()
+        hop.gendirectcall(ll_ensure_indexes, v_dic)
         return DictIteratorRepr(self, "keys_with_hash").newiter(hop)
 
     def rtype_method_iteritems_with_hash(self, hop):
-        hop.exception_cannot_occur()
+        v_dic, = hop.inputargs(self)
+        hop.exception_is_here()
+        hop.gendirectcall(ll_ensure_indexes, v_dic)
         return DictIteratorRepr(self, "items_with_hash").newiter(hop)
 
     def rtype_method_clear(self, hop):
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -644,6 +644,25 @@
     def test_hash_string_siphash24(self):
         self._test_hash_string("siphash24")
 
+    def test_iterkeys_with_hash_on_prebuilt_dict(self):
+        from rpython.rlib import objectmodel
+        prebuilt_d = {"hello": 10, "world": 20}
+        #
+        def fn(n):
+            from rpython.rlib import rsiphash
+            rsiphash.enable_siphash24()
+            #assert str(n) not in prebuilt_d <- this made the test pass,
+            #       before the fix which was that iterkeys_with_hash()
+            #       didn't do the initial rehashing on its own
+            for key, h in objectmodel.iterkeys_with_hash(prebuilt_d):
+                print key, h
+                assert h == compute_hash(key)
+            return 42
+
+        f = self.getcompiled(fn, [int])
+        res = f(0)
+        assert res == 42
+
     def test_list_basic_ops(self):
         def list_basic_ops(i, j):
             l = [1, 2, 3]

From pypy.commits at gmail.com  Tue Jan 31 08:51:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 05:51:47 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58909673.f0a6df0a.718f1.cef3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r850:91a9b611d3fc
Date: 2017-01-31 14:51 +0100
http://bitbucket.org/pypy/pypy.org/changeset/91a9b611d3fc/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66570 of $105000 (63.4%)
+   $66561 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2399 left
+    we have $2391 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Tue Jan 31 11:39:58 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 31 Jan 2017 08:39:58 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: register for sprint
Message-ID: <5890bdde.1282df0a.bb340.0d61@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5767:ec09ffa03480
Date: 2017-01-31 17:39 +0100
http://bitbucket.org/pypy/extradoc/changeset/ec09ffa03480/

Log:	register for sprint

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -10,6 +10,7 @@
     Name              Arrive/Depart     Accomodation 
 ==================== ============== =======================
 Armin Rigo                          private
+Richard Plangger     26.02/04.02    Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Tue Jan 31 11:44:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 08:44:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-siphash24: close branch, ready to merge
Message-ID: <5890bede.c3e31c0a.86816.7566@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-siphash24
Changeset: r89857:d7d9778b49d7
Date: 2017-01-31 17:37 +0100
http://bitbucket.org/pypy/pypy/changeset/d7d9778b49d7/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Tue Jan 31 11:44:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 08:44:13 -0800 (PST)
Subject: [pypy-commit] pypy default: hg merge rpython-hash
Message-ID: <5890bedd.28a8df0a.e6fc8.064a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89856:109091b4f12c
Date: 2017-01-31 17:36 +0100
http://bitbucket.org/pypy/pypy/changeset/109091b4f12c/

Log:	hg merge rpython-hash

	Support for choosing a different hash for strings and unicodes, with
	"siphash24" having a runtime randomized seed (the same as CPython
	3.5).

diff too long, truncating to 2000 out of 2907 lines

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -196,6 +196,13 @@
                default=False,
                requires=[("objspace.usemodules.cpyext", False)]),
 
+    ChoiceOption("hash",
+                 "The hash function to use for strings: fnv from CPython 2.7"
+                 " or siphash24 from CPython >= 3.4",
+                 ["fnv", "siphash24"],
+                 default="fnv",
+                 cmdline="--hash"),
+
     OptionDescription("std", "Standard Object Space Options", [
         BoolOption("withtproxy", "support transparent proxies",
                    default=True),
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -35,6 +35,7 @@
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
         w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+        hashfunc = space.config.objspace.hash
     else:
         w_initstdio = space.appexec([], """():
             return lambda unbuffered: None
@@ -45,6 +46,10 @@
             from rpython.jit.backend.hlinfo import highleveljitinfo
             highleveljitinfo.sys_executable = argv[0]
 
+        if hashfunc == "siphash24":
+            from rpython.rlib import rsiphash
+            rsiphash.enable_siphash24()
+
         #debug("entry point starting")
         #for arg in argv:
         #    debug(" argv -> " + arg)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -23,13 +23,34 @@
 # ____________________________________________________________
 
 class UniqueCache:
+    for_testing = False    # set to True on the class level in test_c.py
+
     def __init__(self, space):
         self.ctvoid = None      # Cache for the 'void' type
         self.ctvoidp = None     # Cache for the 'void *' type
         self.ctchara = None     # Cache for the 'char[]' type
         self.primitives = {}    # Cache for {name: primitive_type}
         self.functions = []     # see _new_function_type()
-        self.for_testing = False
+        self.functions_packed = None     # only across translation
+
+    def _cleanup_(self):
+        import gc
+        assert self.functions_packed is None
+        # Note: a full PyPy translation may still have
+        # 'self.functions == []' at this point, possibly depending
+        # on details.  Code tested directly in test_ffi_obj
+        gc.collect()
+        funcs = []
+        for weakdict in self.functions:
+            funcs += weakdict._dict.values()
+        del self.functions[:]
+        self.functions_packed = funcs if len(funcs) > 0 else None
+
+    def unpack_functions(self):
+        for fct in self.functions_packed:
+            _record_function_type(self, fct)
+        self.functions_packed = None
+
 
 def _clean_cache(space):
     "NOT_RPYTHON"
@@ -622,7 +643,7 @@
     for w_arg in fargs:
         y = compute_identity_hash(w_arg)
         x = intmask((1000003 * x) ^ y)
-    x ^= (ellipsis - abi)
+    x ^= ellipsis + 2 * abi
     if unique_cache.for_testing:    # constant-folded to False in translation;
         x &= 3                      # but for test, keep only 2 bits of hash
     return x
@@ -646,6 +667,8 @@
     # one such dict, but in case of hash collision, there might be
     # more.
     unique_cache = space.fromcache(UniqueCache)
+    if unique_cache.functions_packed is not None:
+        unique_cache.unpack_functions()
     func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
     for weakdict in unique_cache.functions:
         ctype = weakdict.get(func_hash)
@@ -674,13 +697,18 @@
     #
     fct = ctypefunc.W_CTypeFunc(space, fargs, fresult, ellipsis, abi)
     unique_cache = space.fromcache(UniqueCache)
-    func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
+    _record_function_type(unique_cache, fct)
+    return fct
+
+def _record_function_type(unique_cache, fct):
+    from pypy.module._cffi_backend import ctypefunc
+    #
+    func_hash = _func_key_hash(unique_cache, fct.fargs, fct.ctitem,
+                               fct.ellipsis, fct.abi)
     for weakdict in unique_cache.functions:
         if weakdict.get(func_hash) is None:
-            weakdict.set(func_hash, fct)
             break
     else:
         weakdict = rweakref.RWeakValueDictionary(int, ctypefunc.W_CTypeFunc)
         unique_cache.functions.append(weakdict)
-        weakdict.set(func_hash, fct)
-    return fct
+    weakdict.set(func_hash, fct)
diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py
--- a/pypy/module/_cffi_backend/test/test_c.py
+++ b/pypy/module/_cffi_backend/test/test_c.py
@@ -36,6 +36,7 @@
     def setup_class(cls):
         testfuncs_w = []
         keepalive_funcs = []
+        UniqueCache.for_testing = True
 
         def find_and_load_library_for_test(space, w_name, w_is_global=None):
             if w_is_global is None:
@@ -86,11 +87,12 @@
             _all_test_c.find_and_load_library = func
             _all_test_c._testfunc = testfunc
         """)
-        UniqueCache.for_testing = True
 
     def teardown_method(self, method):
+        _clean_cache(self.space)
+
+    def teardown_class(cls):
         UniqueCache.for_testing = False
-        _clean_cache(self.space)
 
 
 all_names = ', '.join(Module.interpleveldefs.keys())
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -1,5 +1,23 @@
+from pypy.module._cffi_backend import newtype
 from pypy.module._cffi_backend.newtype import _clean_cache
 
+
+class TestFFIObj:
+    spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
+
+    def teardown_method(self, meth):
+        _clean_cache(self.space)
+
+    def test_new_function_type_during_translation(self):
+        space = self.space
+        BInt = newtype.new_primitive_type(space, "int")
+        BFunc = newtype.new_function_type(space, space.wrap([BInt]), BInt)
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+        unique_cache = space.fromcache(newtype.UniqueCache)
+        unique_cache._cleanup_()
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+
+
 class AppTestFFIObj:
     spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
 
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -193,6 +193,15 @@
         W_WeakrefBase.__init__(self, space, w_obj, w_callable)
         self.w_hash = None
 
+    def _cleanup_(self):
+        # When a prebuilt weakref is frozen inside a translation, if
+        # this weakref has got an already-cached w_hash, then throw it
+        # away.  That's because the hash value will change after
+        # translation.  It will be recomputed the first time we ask for
+        # it.  Note that such a frozen weakref, if not dead, will point
+        # to a frozen object, so it will never die.
+        self.w_hash = None
+
     def descr__init__weakref(self, space, w_obj, w_callable=None,
                              __args__=None):
         if __args__.arguments_w:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1324,6 +1324,12 @@
         raise wrap_oserror(space, e)
     return space.wrap(res)
 
+class SigCheck:
+    pass
+_sigcheck = SigCheck()
+def _signal_checker():
+    _sigcheck.space.getexecutioncontext().checksignals()
+
 @unwrap_spec(n=int)
 def urandom(space, n):
     """urandom(n) -> str
@@ -1331,9 +1337,12 @@
     Return a string of n random bytes suitable for cryptographic use.
     """
     context = get(space).random_context
-    signal_checker = space.getexecutioncontext().checksignals
     try:
-        return space.wrap(rurandom.urandom(context, n, signal_checker))
+        # urandom() takes a final argument that should be a regular function,
+        # not a bound method like 'getexecutioncontext().checksignals'.
+        # Otherwise, we can't use it from several independent places.
+        _sigcheck.space = space
+        return space.wrap(rurandom.urandom(context, n, _signal_checker))
     except OSError as e:
         raise wrap_oserror(space, e)
 
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -576,6 +576,11 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
+    def _cleanup_(self):
+        # in case there are frozenset objects existing during
+        # translation, make sure we don't translate a cached hash
+        self.hash = 0
+
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_FrozensetObject):
             return False
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -287,7 +287,7 @@
                     for ek, ev in items:
                         result.dictdef.generalize_key(self.immutablevalue(ek))
                         result.dictdef.generalize_value(self.immutablevalue(ev))
-                        result.dictdef.seen_prebuilt_key(ek)
+                        #dictdef.seen_prebuilt_key(ek)---not needed any more
                     seen_elements = len(items)
                     # if the dictionary grew during the iteration,
                     # start over again
diff --git a/rpython/annotator/dictdef.py b/rpython/annotator/dictdef.py
--- a/rpython/annotator/dictdef.py
+++ b/rpython/annotator/dictdef.py
@@ -115,13 +115,5 @@
     def generalize_value(self, s_value):
         self.dictvalue.generalize(s_value)
 
-    def seen_prebuilt_key(self, x):
-        # In case we are an r_dict, we don't ask for the hash ourselves.
-        # Note that if the custom hashing function ends up asking for
-        # the hash of x, then it must use compute_hash() itself, so it
-        # works out.
-        if not self.dictkey.custom_eq_hash:
-            compute_hash(x)
-
     def __repr__(self):
         return '<{%r: %r}>' % (self.dictkey.s_value, self.dictvalue.s_value)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3704,25 +3704,6 @@
         s = a.build_types(f, [int])
         assert s.const == 0
 
-    def test_hash_sideeffect(self):
-        class X:
-            pass
-        x1 = X()
-        x2 = X()
-        x3 = X()
-        d = {(2, x1): 5, (3, x2): 7}
-        def f(n, m):
-            if   m == 1: x = x1
-            elif m == 2: x = x2
-            else:        x = x3
-            return d[n, x]
-        a = self.RPythonAnnotator()
-        s = a.build_types(f, [int, int])
-        assert s.knowntype == int
-        assert hasattr(x1, '__precomputed_identity_hash')
-        assert hasattr(x2, '__precomputed_identity_hash')
-        assert not hasattr(x3, '__precomputed_identity_hash')
-
     def test_contains_of_empty_dict(self):
         class A(object):
             def meth(self):
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -201,10 +201,6 @@
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
-    ChoiceOption("hash",
-                 "The hash to use for strings",
-                 ["rpython", "siphash24"],
-                 default="rpython", cmdline="--hash"),
 
     OptionDescription("backendopt", "Backend Optimization Options", [
         # control inlining
@@ -394,12 +390,6 @@
         if sys.platform == "darwin" or sys.platform =="win32":
             raise ConfigError("'asmgcc' not supported on this platform")
 
-def apply_extra_settings(config):
-    # make the setting of config.hash definitive
-    from rpython.rlib.objectmodel import set_hash_algorithm
-    config.translation.hash = config.translation.hash
-    set_hash_algorithm(config.translation.hash)
-
 # ----------------------------------------------------------------
 
 def set_platform(config):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -117,9 +117,7 @@
 
 # The following flag is set on nursery objects of which we asked the id
 # or the identityhash.  It means that a space of the size of the object
-# has already been allocated in the nonmovable part.  The same flag is
-# abused to mark prebuilt objects whose hash has been taken during
-# translation and is statically recorded.
+# has already been allocated in the nonmovable part.
 GCFLAG_HAS_SHADOW   = first_gcflag << 3
 
 # The following flag is set temporarily on some objects during a major
@@ -208,10 +206,6 @@
     # by GCFLAG_xxx above.
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
-    # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
-    #     then they are one word longer, the extra word storing the hash.
-
 
     # During a minor collection, the objects in the nursery that are
     # moved outside are changed in-place: their header is replaced with
@@ -2640,40 +2634,22 @@
         return shadow
     _find_shadow._dont_inline_ = True
 
-    @specialize.arg(2)
-    def id_or_identityhash(self, gcobj, is_hash):
+    def id_or_identityhash(self, gcobj):
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
         obj = llmemory.cast_ptr_to_adr(gcobj)
-        #
         if self.is_valid_gc_object(obj):
             if self.is_in_nursery(obj):
                 obj = self._find_shadow(obj)
-            elif is_hash:
-                if self.header(obj).tid & GCFLAG_HAS_SHADOW:
-                    #
-                    # For identityhash(), we need a special case for some
-                    # prebuilt objects: their hash must be the same before
-                    # and after translation.  It is stored as an extra word
-                    # after the object.  But we cannot use it for id()
-                    # because the stored value might clash with a real one.
-                    size = self.get_size(obj)
-                    i = (obj + size).signed[0]
-                    # Important: the returned value is not mangle_hash()ed!
-                    return i
-        #
-        i = llmemory.cast_adr_to_int(obj)
-        if is_hash:
-            i = mangle_hash(i)
-        return i
+        return llmemory.cast_adr_to_int(obj)
     id_or_identityhash._always_inline_ = True
 
     def id(self, gcobj):
-        return self.id_or_identityhash(gcobj, False)
+        return self.id_or_identityhash(gcobj)
 
     def identityhash(self, gcobj):
-        return self.id_or_identityhash(gcobj, True)
+        return mangle_hash(self.id_or_identityhash(gcobj))
 
     # ----------
     # Finalizers
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -104,9 +104,7 @@
 
 # The following flag is set on nursery objects of which we asked the id
 # or the identityhash.  It means that a space of the size of the object
-# has already been allocated in the nonmovable part.  The same flag is
-# abused to mark prebuilt objects whose hash has been taken during
-# translation and is statically recorded.
+# has already been allocated in the nonmovable part.
 GCFLAG_HAS_SHADOW   = first_gcflag << 3
 
 # The following flag is set temporarily on some objects during a major
@@ -149,9 +147,6 @@
     # by GCFLAG_xxx above.
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
-    # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
-    #     then they are one word longer, the extra word storing the hash.
 
     _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True})
 
@@ -1868,40 +1863,22 @@
         return shadow
     _find_shadow._dont_inline_ = True
 
-    @specialize.arg(2)
-    def id_or_identityhash(self, gcobj, is_hash):
+    def id_or_identityhash(self, gcobj):
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
         obj = llmemory.cast_ptr_to_adr(gcobj)
-        #
         if self.is_valid_gc_object(obj):
             if self.is_in_nursery(obj):
                 obj = self._find_shadow(obj)
-            elif is_hash:
-                if self.header(obj).tid & GCFLAG_HAS_SHADOW:
-                    #
-                    # For identityhash(), we need a special case for some
-                    # prebuilt objects: their hash must be the same before
-                    # and after translation.  It is stored as an extra word
-                    # after the object.  But we cannot use it for id()
-                    # because the stored value might clash with a real one.
-                    size = self.get_size(obj)
-                    i = (obj + size).signed[0]
-                    # Important: the returned value is not mangle_hash()ed!
-                    return i
-        #
-        i = llmemory.cast_adr_to_int(obj)
-        if is_hash:
-            i = mangle_hash(i)
-        return i
+        return llmemory.cast_adr_to_int(obj)
     id_or_identityhash._always_inline_ = True
 
     def id(self, gcobj):
-        return self.id_or_identityhash(gcobj, False)
+        return self.id_or_identityhash(gcobj)
 
     def identityhash(self, gcobj):
-        return self.id_or_identityhash(gcobj, True)
+        return mangle_hash(self.id_or_identityhash(gcobj))
 
     # ----------
     # Finalizers
diff --git a/rpython/memory/gc/semispace.py b/rpython/memory/gc/semispace.py
--- a/rpython/memory/gc/semispace.py
+++ b/rpython/memory/gc/semispace.py
@@ -48,9 +48,6 @@
 
     HDR = lltype.Struct('header', ('tid', lltype.Signed))   # XXX or rffi.INT?
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', _GCFLAG_HASH_BASE * 0x2
-    # ^^^ prebuilt objects either have GC_HASH_TAKEN_ADDR or they
-    #     have GC_HASH_HASFIELD (and then they are one word longer).
     FORWARDSTUB = lltype.GcStruct('forwarding_stub',
                                   ('forw', llmemory.Address))
     FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -11,7 +11,7 @@
 class BoehmGCTransformer(GCTransformer):
     malloc_zero_filled = True
     FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
-    HDR = lltype.Struct("header", ("hash", lltype.Signed))
+    NO_HEADER = True
 
     def __init__(self, translator, inline=False):
         super(BoehmGCTransformer, self).__init__(translator, inline=inline)
@@ -29,13 +29,8 @@
         ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length
         ll_malloc_varsize = mh.ll_malloc_varsize
 
-        HDRPTR = lltype.Ptr(self.HDR)
-
         def ll_identityhash(addr):
-            obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
-            h = obj.hash
-            if h == 0:
-                obj.hash = h = ~llmemory.cast_adr_to_int(addr)
+            h = ~llmemory.cast_adr_to_int(addr)
             return h
 
         if self.translator:
@@ -194,11 +189,6 @@
                           resulttype = lltype.Signed)
         hop.genop('int_invert', [v_int], resultvar=hop.spaceop.result)
 
-    def gcheader_initdata(self, obj):
-        hdr = lltype.malloc(self.HDR, immortal=True)
-        hdr.hash = lltype.identityhash_nocache(obj._as_ptr())
-        return hdr._obj
-
 
 ########## weakrefs ##########
 # Boehm: weakref objects are small structures containing only a Boehm
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -610,25 +610,6 @@
     def special_funcptr_for_type(self, TYPE):
         return self.layoutbuilder.special_funcptr_for_type(TYPE)
 
-    def gc_header_for(self, obj, needs_hash=False):
-        hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj)
-        withhash, flag = self.gcdata.gc.withhash_flag_is_in_field
-        x = getattr(hdr, withhash)
-        TYPE = lltype.typeOf(x)
-        x = lltype.cast_primitive(lltype.Signed, x)
-        if needs_hash:
-            x |= flag       # set the flag in the header
-        else:
-            x &= ~flag      # clear the flag in the header
-        x = lltype.cast_primitive(TYPE, x)
-        setattr(hdr, withhash, x)
-        return hdr
-
-    def get_hash_offset(self, T):
-        type_id = self.get_type_id(T)
-        assert not self.gcdata.q_is_varsize(type_id)
-        return self.gcdata.q_fixed_size(type_id)
-
     def finish_tables(self):
         group = self.layoutbuilder.close_table()
         log.info("assigned %s typeids" % (len(group.members), ))
@@ -1514,22 +1495,9 @@
 
     def gcheader_initdata(self, obj):
         o = lltype.top_container(obj)
-        needs_hash = self.get_prebuilt_hash(o) is not None
-        hdr = self.gc_header_for(o, needs_hash)
+        hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(o)
         return hdr._obj
 
-    def get_prebuilt_hash(self, obj):
-        # for prebuilt objects that need to have their hash stored and
-        # restored.  Note that only structures that are StructNodes all
-        # the way have their hash stored (and not e.g. structs with var-
-        # sized arrays at the end).  'obj' must be the top_container.
-        TYPE = lltype.typeOf(obj)
-        if not isinstance(TYPE, lltype.GcStruct):
-            return None
-        if TYPE._is_varsize():
-            return None
-        return getattr(obj, '_hash_cache_', None)
-
     def get_finalizer_queue_index(self, hop):
         fq_tag = hop.spaceop.args[0].value
         assert 'FinalizerQueue TAG' in fq_tag.expr
diff --git a/rpython/memory/gctransform/refcounting.py b/rpython/memory/gctransform/refcounting.py
--- a/rpython/memory/gctransform/refcounting.py
+++ b/rpython/memory/gctransform/refcounting.py
@@ -18,8 +18,7 @@
 class RefcountingGCTransformer(GCTransformer):
     malloc_zero_filled = True
 
-    HDR = lltype.Struct("header", ("refcount", lltype.Signed),
-                                  ("hash", lltype.Signed))
+    HDR = lltype.Struct("header", ("refcount", lltype.Signed))
 
     def __init__(self, translator):
         super(RefcountingGCTransformer, self).__init__(translator, inline=True)
@@ -77,10 +76,7 @@
         ll_malloc_varsize = mh.ll_malloc_varsize
 
         def ll_identityhash(addr):
-            obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
-            h = obj.hash
-            if h == 0:
-                obj.hash = h = llmemory.cast_adr_to_int(addr)
+            h = llmemory.cast_adr_to_int(addr)
             return h
 
         if self.translator:
@@ -178,7 +174,6 @@
             if not self.gcheaderbuilder.get_header(p):
                 hdr = self.gcheaderbuilder.new_header(p)
                 hdr.refcount = sys.maxint // 2
-                hdr.hash = lltype.identityhash_nocache(p)
 
     def static_deallocation_funcptr_for_type(self, TYPE):
         if TYPE in self.static_deallocator_funcptrs:
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -374,9 +374,6 @@
         return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP),
                                         lltype.nullptr(ARRAY_TYPEID_MAP)))
 
-    def get_prebuilt_hash(self, obj):
-        return None
-
 
 class MinimalGCTransformer(BaseGCTransformer):
     def __init__(self, parenttransformer):
diff --git a/rpython/rlib/_rweakvaldict.py b/rpython/rlib/_rweakvaldict.py
--- a/rpython/rlib/_rweakvaldict.py
+++ b/rpython/rlib/_rweakvaldict.py
@@ -76,12 +76,16 @@
             bk = self.rtyper.annotator.bookkeeper
             classdef = bk.getuniqueclassdef(weakdict._valueclass)
             r_value = getinstancerepr(self.rtyper, classdef)
+            any_value = False
             for dictkey, dictvalue in weakdict._dict.items():
                 llkey = self.r_key.convert_const(dictkey)
                 llvalue = r_value.convert_const(dictvalue)
                 if llvalue:
                     llvalue = lltype.cast_pointer(rclass.OBJECTPTR, llvalue)
                     self.ll_set_nonnull(l_dict, llkey, llvalue)
+                    any_value = True
+            if any_value:
+                l_dict.resize_counter = -1
             return l_dict
 
     def rtype_method_get(self, hop):
@@ -114,6 +118,8 @@
 
     @jit.dont_look_inside
     def ll_get(self, d, llkey):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         #llop.debug_print(lltype.Void, i, 'get')
@@ -132,6 +138,8 @@
 
     @jit.dont_look_inside
     def ll_set_nonnull(self, d, llkey, llvalue):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         valueref = weakref_create(llvalue)    # GC effects here, before the rest
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
@@ -147,6 +155,8 @@
 
     @jit.dont_look_inside
     def ll_set_null(self, d, llkey):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         if d.entries.everused(i):
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -441,7 +441,7 @@
             except OSError as e:
                 os.write(2, "Could not start GDB: %s" % (
                     os.strerror(e.errno)))
-                raise SystemExit
+                os._exit(1)
         else:
             time.sleep(1)  # give the GDB time to attach
 
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -465,8 +465,14 @@
 
     Note that this can return 0 or -1 too.
 
-    It returns the same number, both before and after translation.
-    Dictionaries don't need to be rehashed after translation.
+    NOTE: It returns a different number before and after translation!
+    Dictionaries will be rehashed when the translated program starts.
+    Be careful about other places that store or depend on a hash value:
+    if such a place can exist before translation, you should add for
+    example a _cleanup_() method to clear this cache during translation.
+
+    (Nowadays we could completely remove compute_hash() and decide that
+    hash(x) is valid RPython instead, at least for the types listed here.)
     """
     if isinstance(x, (str, unicode)):
         return _hash_string(x)
@@ -484,17 +490,11 @@
     """RPython equivalent of object.__hash__(x).  This returns the
     so-called 'identity hash', which is the non-overridable default hash
     of Python.  Can be called for any RPython-level object that turns
-    into a GC object, but not NULL.  The value is not guaranteed to be the
-    same before and after translation, except for RPython instances on the
-    lltypesystem.
+    into a GC object, but not NULL.  The value will be different before
+    and after translation (WARNING: this is a change with older RPythons!)
     """
     assert x is not None
-    result = object.__hash__(x)
-    try:
-        x.__dict__['__precomputed_identity_hash'] = result
-    except (TypeError, AttributeError):
-        pass
-    return result
+    return object.__hash__(x)
 
 def compute_unique_id(x):
     """RPython equivalent of id(x).  The 'x' must be an RPython-level
@@ -519,21 +519,17 @@
 
 # ----------
 
-HASH_ALGORITHM = "rpython"  # XXX Is there a better name?
-HASH_ALGORITHM_FIXED = False
+def _hash_string(s):
+    """The default algorithm behind compute_hash() for a string or a unicode.
+    This is a modified Fowler-Noll-Vo (FNV) hash.  According to Wikipedia,
+    FNV needs carefully-computed constants called FNV primes and FNV offset
+    basis, which are absent from the present algorithm.  Nevertheless,
+    this matches CPython 2.7 without -R, which has proven a good hash in
+    practice (even if not crypographical nor randomizable).
 
- at not_rpython
-def set_hash_algorithm(algo):
-    """Must be called very early, before any string is hashed with
-    compute_hash()!"""
-    global HASH_ALGORITHM
-    if HASH_ALGORITHM != algo:
-        assert not HASH_ALGORITHM_FIXED, "compute_hash() already called!"
-        assert algo in ("rpython", "siphash24")
-        HASH_ALGORITHM = algo
-
-
-def _hash_string_rpython(s):
+    There is a mechanism to use another one in programs after translation.
+    See rsiphash.py, which implements the algorithm of CPython >= 3.4.
+    """
     from rpython.rlib.rarithmetic import intmask
 
     length = len(s)
@@ -547,100 +543,8 @@
     x ^= length
     return intmask(x)
 
-
- at not_rpython
-def _hash_string_siphash24(s):
-    """This version is called when untranslated only."""
-    import array
-    from rpython.rlib.rsiphash import siphash24
-    from rpython.rtyper.lltypesystem import lltype, rffi
-    from rpython.rlib.rarithmetic import intmask
-
-    if not isinstance(s, str):
-        if isinstance(s, unicode):
-            lst = map(ord, s)
-        else:
-            lst = map(ord, s.chars)    # for rstr.STR or UNICODE
-        # NOTE: a latin-1 unicode string must have the same hash as the
-        # corresponding byte string.
-        if all(n <= 0xFF for n in lst):
-            kind = "B"
-        elif rffi.sizeof(lltype.UniChar) == 4:
-            kind = "I"
-        else:
-            kind = "H"
-        s = array.array(kind, lst).tostring()
-    ptr = rffi.str2charp(s)
-    x = siphash24(ptr, len(s))
-    rffi.free_charp(ptr)
-    return intmask(x)
-
-def ll_hash_string_siphash24(ll_s):
-    """Called from lltypesystem/rstr.py.  'll_s' is a rstr.STR or UNICODE."""
-    from rpython.rlib.rsiphash import siphash24
-    from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
-    from rpython.rlib.rarithmetic import intmask
-
-    length = len(ll_s.chars)
-    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:
-        # no GC operation from here!
-        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
-    else:
-        # NOTE: a latin-1 unicode string must have the same hash as the
-        # corresponding byte string.  If the unicode is all within
-        # 0-255, then we need to allocate a byte buffer and copy the
-        # latin-1 encoding in it manually.
-        for i in range(length):
-            if ord(ll_s.chars[i]) > 0xFF:
-                # no GC operation from here!
-                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
-                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
-                break
-        else:
-            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
-            i = 0
-            while i < length:
-                p[i] = chr(ord(ll_s.chars[i]))
-                i += 1
-            x = siphash24(llmemory.cast_ptr_to_adr(p), length)
-            lltype.free(p, flavor='raw')
-            return intmask(x)
-    x = siphash24(addr, length)
-    keepalive_until_here(ll_s)
-    return intmask(x)
-ll_hash_string_siphash24._jit_look_inside_ = False
-
-
- at not_rpython
-def _hash_string(s):
-    """The algorithm behind compute_hash() for a string or a unicode.
-    This version is only for untranslated usage, and 's' is a str or unicode.
-    """
-    global HASH_ALGORITHM_FIXED
-    HASH_ALGORITHM_FIXED = True
-    if HASH_ALGORITHM == "rpython":
-        return _hash_string_rpython(s)
-    if HASH_ALGORITHM == "siphash24":
-        return _hash_string_siphash24(s)
-    raise NotImplementedError
-
 def ll_hash_string(ll_s):
-    """The algorithm behind compute_hash() for a string or a unicode.
-    This version is called from lltypesystem/rstr.py, and 'll_s' is a
-    rstr.STR or rstr.UNICODE.
-    """
-    if not we_are_translated():
-        global HASH_ALGORITHM_FIXED
-        HASH_ALGORITHM_FIXED = True
-    if HASH_ALGORITHM == "rpython":
-        return _hash_string_rpython(ll_s.chars)
-    if HASH_ALGORITHM == "siphash24":
-        if we_are_translated():
-            return ll_hash_string_siphash24(ll_s)
-        else:
-            return _hash_string_siphash24(ll_s)
-    raise NotImplementedError
-
+    return _hash_string(ll_s.chars)
 
 def _hash_float(f):
     """The algorithm behind compute_hash() for a float.
@@ -698,6 +602,21 @@
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
+    _about_ = ll_hash_string
+    # this is only used when annotating the code in rstr.py, and so
+    # it always occurs after the RPython program signalled its intent
+    # to use a different hash.  The code below overwrites the use of
+    # ll_hash_string() to make the annotator think a possibly different
+    # function was called.
+
+    def compute_annotation(self):
+        from rpython.annotator import model as annmodel
+        bk = self.bookkeeper
+        translator = bk.annotator.translator
+        fn = getattr(translator, 'll_hash_string', ll_hash_string)
+        return annmodel.SomePBC([bk.getdesc(fn)])
+
+class Entry(ExtRegistryEntry):
     _about_ = compute_identity_hash
 
     def compute_result_annotation(self, s_x):
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -1,12 +1,24 @@
-import sys, os, struct
+"""
+This module implements siphash-2-4, the hashing algorithm for strings
+and unicodes.  You can use it explicitly by calling siphash24() with
+a byte string, or you can use enable_siphash24() to enable the use
+of siphash-2-4 on all RPython strings and unicodes in your program
+after translation.
+"""
+import sys, os, errno
 from contextlib import contextmanager
-from rpython.rlib import rarithmetic
+from rpython.rlib import rarithmetic, rurandom
 from rpython.rlib.objectmodel import not_rpython, always_inline
-from rpython.rlib.rgc import no_collect
-from rpython.rlib.rarithmetic import r_uint64
+from rpython.rlib.objectmodel import we_are_translated, dont_inline
+from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib import rgc, jit, rposix
+from rpython.rlib.rarithmetic import r_uint64, r_uint32, r_uint
 from rpython.rlib.rawstorage import misaligned_is_fine
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rlib.nonconst import NonConstant
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.rtyper.annlowlevel import llhelper
 
 
 if sys.byteorder == 'little':
@@ -16,37 +28,164 @@
     _le64toh = rarithmetic.byteswap
 
 
-# Initialize the values of the secret seed: two 64-bit constants.
-# CPython picks a new seed every time 'python' starts.  PyPy cannot do
-# that as easily because many details may rely on getting the same hash
-# value before and after translation.  We can, however, pick a random
-# seed once per translation, which should already be quite good.
-#
-# XXX no, it is not: e.g. all Ubuntu installations of the same Ubuntu
-# would get the same seed.  That's not good enough.
+class Seed:
+    k0l = k1l = r_uint64(0)
+seed = Seed()
 
- at not_rpython
-def select_random_seed():
-    global k0, k1    # note: the globals k0, k1 are already byte-swapped
-    v0, v1 = struct.unpack("QQ", os.urandom(16))
-    k0 = r_uint64(v0)
-    k1 = r_uint64(v1)
 
-select_random_seed()
+def _decode64(s):
+    return (r_uint64(ord(s[0])) |
+            r_uint64(ord(s[1])) << 8 |
+            r_uint64(ord(s[2])) << 16 |
+            r_uint64(ord(s[3])) << 24 |
+            r_uint64(ord(s[4])) << 32 |
+            r_uint64(ord(s[5])) << 40 |
+            r_uint64(ord(s[6])) << 48 |
+            r_uint64(ord(s[7])) << 56)
+
+def select_random_seed(s):
+    """'s' is a string of length 16"""
+    seed.k0l = _decode64(s)
+    seed.k1l = _decode64(s[8:16])
+
+
+random_ctx = rurandom.init_urandom()
+strtoul = rffi.llexternal("strtoul", [rffi.CCHARP, rffi.CCHARPP, rffi.INT],
+                          rffi.ULONG, save_err=rffi.RFFI_SAVE_ERRNO)
+
+env_var_name = "PYTHONHASHSEED"
+
+def initialize_from_env():
+    # This uses the same algorithms as CPython 3.5.  The environment
+    # variable we read also defaults to "PYTHONHASHSEED".  If needed,
+    # a different RPython interpreter can patch the value of the
+    # global variable 'env_var_name', or just patch the whole
+    # initialize_from_env() function.
+    value = os.environ.get(env_var_name)
+    if value and value != "random":
+        with rffi.scoped_view_charp(value) as ptr:
+            with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as endptr:
+                endptr[0] = ptr
+                seed = strtoul(ptr, endptr, 10)
+                full = endptr[0][0] == '\x00'
+        seed = lltype.cast_primitive(lltype.Unsigned, seed)
+        if not full or seed > r_uint(4294967295) or (
+            rposix.get_saved_errno() == errno.ERANGE and
+            seed == lltype.cast_primitive(lltype.Unsigned,
+                                          rffi.cast(rffi.ULONG, -1))):
+            os.write(2,
+                "%s must be \"random\" or an integer "
+                "in range [0; 4294967295]\n" % (env_var_name,))
+            os._exit(1)
+        if not seed:
+            # disable the randomized hash
+            s = '\x00' * 16
+        else:
+            s = lcg_urandom(seed)
+    else:
+        try:
+            s = rurandom.urandom(random_ctx, 16)
+        except Exception as e:
+            os.write(2,
+                "%s: failed to get random numbers to initialize Python\n" %
+                (str(e),))
+            os._exit(1)
+            raise   # makes the annotator happy
+    select_random_seed(s)
+
+def lcg_urandom(x):
+    s = ''
+    for index in range(16):
+        x *= 214013
+        x += 2531011
+        s += chr((x >> 16) & 0xff)
+    return s
+
+
+_FUNC = lltype.Ptr(lltype.FuncType([], lltype.Void))
+
+def enable_siphash24():
+    """
+    Enable the use of siphash-2-4 for all RPython strings and unicodes
+    in the translated program.  You must call this function anywhere
+    from your interpreter (from a place that is annotated).  Don't call
+    more than once.
+    """
+
+class Entry(ExtRegistryEntry):
+    _about_ = enable_siphash24
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        if hasattr(translator, 'll_hash_string'):
+            assert translator.ll_hash_string == ll_hash_string_siphash24
+        else:
+            translator.ll_hash_string = ll_hash_string_siphash24
+        bk = self.bookkeeper
+        s_callable = bk.immutablevalue(initialize_from_env)
+        key = (enable_siphash24,)
+        bk.emulate_pbc_call(key, s_callable, [])
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        bk = hop.rtyper.annotator.bookkeeper
+        s_callable = bk.immutablevalue(initialize_from_env)
+        r_callable = hop.rtyper.getrepr(s_callable)
+        ll_init = r_callable.get_unique_llfn().value
+        bk.annotator.translator._call_at_startup.append(ll_init)
+
+
+ at rgc.no_collect
+def ll_hash_string_siphash24(ll_s):
+    """Called indirectly from lltypesystem/rstr.py, by redirection from
+    objectmodel.ll_string_hash().
+    """
+    from rpython.rlib.rarithmetic import intmask
+
+    # This function is entirely @rgc.no_collect.
+    length = len(ll_s.chars)
+    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:   # regular STR
+        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
+    else:
+        # NOTE: a latin-1 unicode string must have the same hash as the
+        # corresponding byte string.  If the unicode is all within
+        # 0-255, then we need to allocate a byte buffer and copy the
+        # latin-1 encoding in it manually.  Note also that we give a
+        # different hash result than CPython on ucs4 platforms, for
+        # unicode strings where CPython uses 2 bytes per character.
+        for i in range(length):
+            if ord(ll_s.chars[i]) > 0xFF:
+                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
+                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
+                break
+        else:
+            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
+            i = 0
+            while i < length:
+                p[i] = chr(ord(ll_s.chars[i]))
+                i += 1
+            x = _siphash24(llmemory.cast_ptr_to_adr(p), length)
+            lltype.free(p, flavor='raw')
+            return intmask(x)
+    x = _siphash24(addr, length)
+    keepalive_until_here(ll_s)
+    return intmask(x)
+
 
 @contextmanager
 def choosen_seed(new_k0, new_k1, test_misaligned_path=False):
-    global k0, k1, misaligned_is_fine
-    old = k0, k1, misaligned_is_fine
-    k0 = _le64toh(r_uint64(new_k0))
-    k1 = _le64toh(r_uint64(new_k1))
+    """For tests."""
+    global misaligned_is_fine
+    old = seed.k0l, seed.k1l, misaligned_is_fine
+    seed.k0l = _le64toh(r_uint64(new_k0))
+    seed.k1l = _le64toh(r_uint64(new_k1))
     if test_misaligned_path:
         misaligned_is_fine = False
     yield
-    k0, k1, misaligned_is_fine = old
+    seed.k0l, seed.k1l, misaligned_is_fine = old
 
 def get_current_seed():
-    return _le64toh(k0), _le64toh(k1)
+    return _le64toh(seed.k0l), _le64toh(seed.k1l)
 
 
 magic0 = r_uint64(0x736f6d6570736575)
@@ -77,20 +216,21 @@
     return v0, v1, v2, v3
 
 
- at no_collect
-def siphash24(addr_in, size):
+ at rgc.no_collect
+def _siphash24(addr_in, size):
     """Takes an address pointer and a size.  Returns the hash as a r_uint64,
     which can then be casted to the expected type."""
 
-    direct = (misaligned_is_fine or
-                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
-
+    k0 = seed.k0l
+    k1 = seed.k1l
     b = r_uint64(size) << 56
     v0 = k0 ^ magic0
     v1 = k1 ^ magic1
     v2 = k0 ^ magic2
     v3 = k1 ^ magic3
 
+    direct = (misaligned_is_fine or
+                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
     index = 0
     if direct:
         while size >= 8:
@@ -113,7 +253,6 @@
                 r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48 |
                 r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 7)) << 56
             )
-            mi = _le64toh(mi)
             size -= 8
             index += 8
             v3 ^= mi
@@ -158,3 +297,13 @@
     v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
 
     return (v0 ^ v1) ^ (v2 ^ v3)
+
+
+ at jit.dont_look_inside
+def siphash24(s):
+    """'s' is a normal string.  Returns its siphash-2-4 as a r_uint64.
+    Don't forget to cast the result to a regular integer if needed,
+    e.g. with rarithmetic.intmask().
+    """
+    with rffi.scoped_nonmovingbuffer(s) as p:
+        return _siphash24(llmemory.cast_ptr_to_adr(p), len(s))
diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -57,6 +57,8 @@
                              immortal=True, zero=True)
 
     def urandom(context, n, signal_checker=None):
+        # NOTE: no dictionaries here: rsiphash24 calls this to
+        # initialize the random seed of string hashes
         provider = context[0]
         if not provider:
             # This handle is never explicitly released. The operating
@@ -139,6 +141,8 @@
 
     def urandom(context, n, signal_checker=None):
         "Read n bytes from /dev/urandom."
+        # NOTE: no dictionaries here: rsiphash24 calls this to
+        # initialize the random seed of string hashes
         result = []
         if SYS_getrandom is not None:
             n = _getrandom(n, result, signal_checker)
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -166,7 +166,6 @@
     foo = Foo()
     h = compute_hash(foo)
     assert h == object.__hash__(foo)
-    assert h == getattr(foo, '__precomputed_identity_hash')
     assert compute_hash(None) == 0
 
 def test_compute_hash_float():
@@ -182,7 +181,6 @@
     foo = Foo()
     h = compute_identity_hash(foo)
     assert h == object.__hash__(foo)
-    assert h == getattr(foo, '__precomputed_identity_hash')
 
 def test_compute_unique_id():
     from rpython.rlib.rarithmetic import intmask
@@ -410,36 +408,6 @@
         res = self.interpret(f, [])
         assert res == 1
 
-    def test_compute_hash_across_translation(self):
-        class Foo(object):
-            pass
-        q = Foo()
-
-        def f(i):
-            assert compute_hash(None) == 0
-            assert compute_hash(i) == h_42
-            assert compute_hash(i + 1.0) == h_43_dot_0
-            assert compute_hash((i + 3) / 6.0) == h_7_dot_5
-            assert compute_hash("Hello" + str(i)) == h_Hello42
-            if i == 42:
-                p = None
-            else:
-                p = Foo()
-            assert compute_hash(p) == h_None
-            assert compute_hash(("world", None, i, 7.5)) == h_tuple
-            assert compute_hash(q) == h_q
-            return i * 2
-        h_42 = compute_hash(42)
-        h_43_dot_0 = compute_hash(43.0)
-        h_7_dot_5 = compute_hash(7.5)
-        h_Hello42 = compute_hash("Hello42")
-        h_None = compute_hash(None)
-        h_tuple = compute_hash(("world", None, 42, 7.5))
-        h_q = compute_hash(q)
-
-        res = self.interpret(f, [42])
-        assert res == 84
-
     def test_fetch_translated_config(self):
         assert fetch_translated_config() is None
         def f():
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
--- a/rpython/rlib/test/test_rsiphash.py
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -1,5 +1,10 @@
-from rpython.rlib.rsiphash import siphash24, choosen_seed
+import os
+from rpython.rlib.rsiphash import siphash24, _siphash24, choosen_seed
+from rpython.rlib.rsiphash import initialize_from_env, enable_siphash24
+from rpython.rlib.objectmodel import compute_hash
+from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.lltypesystem import llmemory, rffi
+from rpython.translator.c.test.test_genc import compile
 
 
 CASES = [
@@ -28,13 +33,11 @@
 ]
 
 def check(s):
-    p = rffi.str2charp(s)
     q = rffi.str2charp('?' + s)
     with choosen_seed(0x8a9f065a358479f4, 0x11cb1e9ee7f40e1f,
                       test_misaligned_path=True):
-        x = siphash24(llmemory.cast_ptr_to_adr(p), len(s))
-        y = siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
-    rffi.free_charp(p)
+        x = siphash24(s)
+        y = _siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
     rffi.free_charp(q)
     assert x == y
     return x
@@ -42,3 +45,104 @@
 def test_siphash24():
     for expected, string in CASES:
         assert check(string) == expected
+
+def test_fix_seed():
+    old_val = os.environ.get('PYTHONHASHSEED', None)
+    try:
+        os.environ['PYTHONHASHSEED'] = '0'
+        initialize_from_env()
+        assert siphash24("foo") == 15988776847138518036
+        # value checked with CPython 3.5
+
+        os.environ['PYTHONHASHSEED'] = '4000000000'
+        initialize_from_env()
+        assert siphash24("foo") == 13829150778707464258
+        # value checked with CPython 3.5
+
+        for env in ['', 'random']:
+            os.environ['PYTHONHASHSEED'] = env
+            initialize_from_env()
+            hash1 = siphash24("foo")
+            initialize_from_env()
+            hash2 = siphash24("foo")
+            assert hash1 != hash2     # extremely unlikely
+    finally:
+        if old_val is None:
+            del os.environ['PYTHONHASHSEED']
+        else:
+            os.environ['PYTHONHASHSEED'] = old_val
+
+def test_translated():
+    d1 = {"foo": 123}
+    d2 = {u"foo": 456, u"\u1234\u5678": 789}
+    class G:
+        pass
+    g = G()
+    g.v1 = d1.copy()
+    g.v2 = d2.copy()
+
+    def fetch(n):
+        if n == 0: return d1.get("foo", -1)
+        if n == 1: return g.v1.get("foo", -1)
+        if n == 2: return compute_hash("foo")
+        if n == 3: return d2.get(u"foo", -1)
+        if n == 4: return g.v2.get(u"foo", -1)
+        if n == 5: return compute_hash(u"foo")
+        if n == 6: return d2.get(u"\u1234\u5678", -1)
+        if n == 7: return g.v2.get(u"\u1234\u5678", -1)
+        if n == 8: return compute_hash(u"\u1234\u5678")
+        assert 0
+
+    def entrypoint(n):
+        enable_siphash24()
+        g.v1["bar"] = -2
+        g.v2[u"bar"] = -2
+        if n >= 0:    # get items one by one, because otherwise it may
+                      # be the case that one line influences the next
+            return str(fetch(n))
+        else:
+            # ...except in random mode, because we want all results
+            # to be computed with the same seed
+            return ' '.join([str(fetch(n)) for n in range(9)])
+
+    fn = compile(entrypoint, [int])
+
+    def getall():
+        return [int(fn(i)) for i in range(9)]
+
+    old_val = os.environ.get('PYTHONHASHSEED', None)
+    try:
+        os.environ['PYTHONHASHSEED'] = '0'
+        s1 = getall()
+        assert s1[:8] == [
+            123, 123, intmask(15988776847138518036),
+            456, 456, intmask(15988776847138518036),
+            789, 789]
+        assert s1[8] in [intmask(17593683438421985039),    # ucs2 mode
+                         intmask(94801584261658677)]       # ucs4 mode
+
+        os.environ['PYTHONHASHSEED'] = '3987654321'
+        s1 = getall()
+        assert s1[:8] == [
+            123, 123, intmask(5890804383681474441),
+            456, 456, intmask(5890804383681474441),
+            789, 789]
+        assert s1[8] in [intmask(4192582507672183374),     # ucs2 mode
+                         intmask(7179255293164649778)]     # ucs4 mode
+
+        for env in ['', 'random']:
+            os.environ['PYTHONHASHSEED'] = env
+            s1 = map(int, fn(-1).split())
+            s2 = map(int, fn(-1).split())
+            assert s1[0:2]+s1[3:5]+s1[6:8] == [123, 123, 456, 456, 789, 789]
+            assert s1[2] == s1[5]
+            assert s2[0:2]+s2[3:5]+s2[6:8] == [123, 123, 456, 456, 789, 789]
+            assert s2[2] == s2[5]
+            #
+            assert len(set([s1[2], s2[2], s1[8], s2[8]])) == 4
+
+    finally:
+        if old_val is None:
+            del os.environ['PYTHONHASHSEED']
+        else:
+            os.environ['PYTHONHASHSEED'] = old_val
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -1,8 +1,9 @@
 import py
 from rpython.annotator.model import UnionError
-from rpython.rlib import rgc
+from rpython.rlib import rgc, nonconst
 from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rtyper.test.test_llinterp import interpret
+from rpython.translator.c.test.test_genc import compile
 
 class X(object):
     pass
@@ -213,3 +214,33 @@
         assert d.get(keys[3]) is None
     f()
     interpret(f, [])
+
+def test_translation_prebuilt_1():
+    class K:
+        pass
+    d = RWeakValueDictionary(K, X)
+    k1 = K(); k2 = K()
+    x1 = X(); x2 = X()
+    d.set(k1, x1)
+    d.set(k2, x2)
+    def f():
+        assert d.get(k1) is x1
+        assert d.get(k2) is x2
+    f()
+    fc = compile(f, [], gcpolicy="boehm", rweakref=True)
+    fc()
+
+def _test_translation_prebuilt_2():
+    from rpython.rlib import rsiphash
+    d = RWeakValueDictionary(str, X)
+    k1 = "key1"; k2 = "key2"
+    x1 = X(); x2 = X()
+    d.set(k1, x1)
+    d.set(k2, x2)
+    def f():
+        rsiphash.enable_siphash24()
+        i = nonconst.NonConstant(1)
+        assert d.get("key%d" % (i,)) is x1
+        assert d.get("key%d" % (i+1,)) is x2
+    fc = compile(f, [], gcpolicy="boehm", rweakref=True)
+    fc()
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -539,6 +539,7 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
+    'call_at_startup':      LLOp(canrun=True),
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1380,20 +1380,11 @@
             return callb(*args)
         raise TypeError("%r instance is not a function" % (self._T,))
 
-    def _identityhash(self, cache=True):
+    def _identityhash(self):
         p = normalizeptr(self)
-        try:
-            return p._obj._hash_cache_
-        except AttributeError:
-            assert self._T._gckind == 'gc'
-            assert self      # not for NULL
-            result = hash(p._obj)
-            if cache:
-                try:
-                    p._obj._hash_cache_ = result
-                except AttributeError:
-                    pass
-            return result
+        assert self._T._gckind == 'gc'
+        assert self      # not for NULL
+        return hash(p._obj)
 
 class _ptr(_abstract_ptr):
     __slots__ = ('_TYPE',
@@ -1759,7 +1750,7 @@
 class _struct(_parentable):
     _kind = "structure"
 
-    __slots__ = ('_hash_cache_', '_compilation_info')
+    __slots__ = ('_compilation_info',)
 
     def __new__(self, TYPE, n=None, initialization=None, parent=None,
                 parentindex=None):
@@ -2442,24 +2433,6 @@
     return SomeInteger()
 
 
-def identityhash_nocache(p):
-    """Version of identityhash() to use from backends that don't care about
-    caching."""
-    assert p
-    return p._identityhash(cache=False)
-
-def init_identity_hash(p, value):
-    """For a prebuilt object p, initialize its hash value to 'value'."""
-    assert isinstance(typeOf(p), Ptr)
-    p = normalizeptr(p)
-    if not p:
-        raise ValueError("cannot change hash(NULL)!")
-    if hasattr(p._obj, '_hash_cache_'):
-        raise ValueError("the hash of %r was already computed" % (p,))
-    if typeOf(p).TO._is_varsize():
-        raise ValueError("init_identity_hash(): not for varsized types")
-    p._obj._hash_cache_ = intmask(value)
-
 def isCompatibleType(TYPE1, TYPE2):
     return TYPE1._is_compatible(TYPE2)
 
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -742,6 +742,9 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
+def op_call_at_startup(init_func):
+    pass    # do nothing
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -236,21 +236,14 @@
                 if self.r_rdict_hashfn.lowleveltype != lltype.Void:
                     l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash)
                     l_dict.fnkeyhash = l_fn
-
-                for dictkeycontainer, dictvalue in dictobj._dict.items():
-                    llkey = r_key.convert_const(dictkeycontainer.key)
-                    llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        dictkeycontainer.hash)
-                return l_dict
-
+                any_items = dictobj._dict.items()
             else:
-                for dictkey, dictvalue in dictobj.items():
-                    llkey = r_key.convert_const(dictkey)
-                    llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        l_dict.keyhash(llkey))
-                return l_dict
+                any_items = dictobj.items()
+            if any_items:
+                raise TyperError("found a prebuilt, explicitly non-ordered, "
+                                 "non-empty dict.  it would require additional"
+                                 " support to rehash it at program start-up")
+            return l_dict
 
     def rtype_len(self, hop):
         v_dict, = hop.inputargs(self)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -829,7 +829,7 @@
         return assert_str0(charpsize2str(cp, size))
     charp2str._annenforceargs_ = [lltype.SomePtr(TYPEP)]
 
-    # str -> char*, bool, bool
+    # str -> char*, flag
     # Can't inline this because of the raw address manipulation.
     @jit.dont_look_inside
     def get_nonmovingbuffer(data):
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib import objectmodel, jit, rgc, types
 from rpython.rlib.signature import signature
-from rpython.rlib.objectmodel import specialize, likely
+from rpython.rlib.objectmodel import specialize, likely, not_rpython
 from rpython.rtyper.debug import ll_assert
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rtyper import rmodel
@@ -46,20 +46,23 @@
 @jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
 @jit.oopspec('ordereddict.lookup(d, key, hash, flag)')
 def ll_call_lookup_function(d, key, hash, flag):
-    fun = d.lookup_function_no & FUNC_MASK
-    # This likely() here forces gcc to compile the check for fun == FUNC_BYTE
-    # first.  Otherwise, this is a regular switch and gcc (at least 4.7)
-    # compiles this as a series of checks, with the FUNC_BYTE case last.
-    # It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
-    if likely(fun == FUNC_BYTE):
-        return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
-    elif fun == FUNC_SHORT:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
-    elif IS_64BIT and fun == FUNC_INT:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
-    elif fun == FUNC_LONG:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
-    assert False
+    while True:
+        fun = d.lookup_function_no & FUNC_MASK
+        # This likely() here forces gcc to compile the check for fun==FUNC_BYTE
+        # first.  Otherwise, this is a regular switch and gcc (at least 4.7)
+        # compiles this as a series of checks, with the FUNC_BYTE case last.
+        # It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
+        if likely(fun == FUNC_BYTE):
+            return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
+        elif fun == FUNC_SHORT:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
+        elif IS_64BIT and fun == FUNC_INT:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
+        elif fun == FUNC_LONG:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
+        else:
+            ll_dict_create_initial_index(d)
+            # then, retry
 
 def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
                 ll_fasthash_function=None, ll_hash_function=None,
@@ -235,6 +238,7 @@
             self.setup()
             self.setup_final()
             l_dict = ll_newdict_size(self.DICT, len(dictobj))
+            ll_no_initial_index(l_dict)
             self.dict_cache[key] = l_dict
             r_key = self.key_repr
             if r_key.lowleveltype == llmemory.Address:
@@ -252,16 +256,14 @@
                 for dictkeycontainer, dictvalue in dictobj._dict.items():
                     llkey = r_key.convert_const(dictkeycontainer.key)
                     llvalue = r_value.convert_const(dictvalue)
-                    _ll_dict_insertclean(l_dict, llkey, llvalue,
-                                         dictkeycontainer.hash)
+                    _ll_dict_insert_no_index(l_dict, llkey, llvalue)
                 return l_dict
 
             else:
                 for dictkey, dictvalue in dictobj.items():
                     llkey = r_key.convert_const(dictkey)
                     llvalue = r_value.convert_const(dictvalue)
-                    _ll_dict_insertclean(l_dict, llkey, llvalue,
-                                         l_dict.keyhash(llkey))
+                    _ll_dict_insert_no_index(l_dict, llkey, llvalue)
                 return l_dict
 
     def rtype_len(self, hop):
@@ -336,11 +338,15 @@
         return DictIteratorRepr(self, "items").newiter(hop)
 
     def rtype_method_iterkeys_with_hash(self, hop):
-        hop.exception_cannot_occur()
+        v_dic, = hop.inputargs(self)
+        hop.exception_is_here()
+        hop.gendirectcall(ll_ensure_indexes, v_dic)
         return DictIteratorRepr(self, "keys_with_hash").newiter(hop)
 
     def rtype_method_iteritems_with_hash(self, hop):
-        hop.exception_cannot_occur()
+        v_dic, = hop.inputargs(self)
+        hop.exception_is_here()
+        hop.gendirectcall(ll_ensure_indexes, v_dic)
         return DictIteratorRepr(self, "items_with_hash").newiter(hop)
 
     def rtype_method_clear(self, hop):
@@ -458,17 +464,30 @@
 
 IS_64BIT = sys.maxint != 2 ** 31 - 1
 
-FUNC_SHIFT = 2
-FUNC_MASK  = 0x03  # two bits
 if IS_64BIT:
-    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(4)
+    FUNC_SHIFT = 3
+    FUNC_MASK  = 0x07  # three bits
+    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG, FUNC_MUST_REINDEX = range(5)
 else:
-    FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(3)
+    FUNC_SHIFT = 2
+    FUNC_MASK  = 0x03  # two bits
+    FUNC_BYTE, FUNC_SHORT, FUNC_LONG, FUNC_MUST_REINDEX = range(4)
 TYPE_BYTE  = rffi.UCHAR
 TYPE_SHORT = rffi.USHORT
 TYPE_INT   = rffi.UINT
 TYPE_LONG  = lltype.Unsigned
 
+def ll_no_initial_index(d):
+    # Used when making new empty dicts, and when translating prebuilt dicts.
+    # Remove the index completely.  A dictionary must always have an
+    # index unless it is freshly created or freshly translated.  Most
+    # dict operations start with ll_call_lookup_function(), which will
+    # recompute the hashes and create the index.
+    ll_assert(d.num_live_items == d.num_ever_used_items, 
+         "ll_no_initial_index(): dict already in use")
+    d.lookup_function_no = FUNC_MUST_REINDEX
+    d.indexes = lltype.nullptr(llmemory.GCREF.TO)
+
 def ll_malloc_indexes_and_choose_lookup(d, n):
     # keep in sync with ll_clear_indexes() below
     if n <= 256:
@@ -508,6 +527,7 @@
 
 @jit.dont_look_inside
 def ll_call_insert_clean_function(d, hash, i):
+    assert i >= 0
     fun = d.lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
         ll_dict_store_clean(d, hash, i, TYPE_BYTE)
@@ -518,6 +538,8 @@
     elif fun == FUNC_LONG:
         ll_dict_store_clean(d, hash, i, TYPE_LONG)
     else:
+        # can't be still FUNC_MUST_REINDEX here
+        ll_assert(False, "ll_call_insert_clean_function(): invalid lookup_fun")
         assert False
 
 def ll_call_delete_by_entry_index(d, hash, i):
@@ -531,6 +553,8 @@
     elif fun == FUNC_LONG:
         ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
     else:
+        # can't be still FUNC_MUST_REINDEX here
+        ll_assert(False, "ll_call_delete_by_entry_index(): invalid lookup_fun")
         assert False
 
 def ll_valid_from_flag(entries, i):
@@ -648,15 +672,14 @@
     ll_dict_reindex(d, _ll_len_of_d_indexes(d))
 _ll_dict_rescue._dont_inline_ = True
 
-def _ll_dict_insertclean(d, key, value, hash):
+ at not_rpython
+def _ll_dict_insert_no_index(d, key, value):
     # never translated
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
     entry = d.entries[d.num_ever_used_items]
     entry.key = key
     entry.value = value
-    if hasattr(ENTRY, 'f_hash'):
-        entry.f_hash = hash
+    # note that f_hash is left uninitialized in prebuilt dicts
     if hasattr(ENTRY, 'f_valid'):
         entry.f_valid = True
     d.num_ever_used_items += 1
@@ -811,12 +834,13 @@
         # also possible that there are more dead items immediately behind the
         # last one, we reclaim all the dead items at the end of the ordereditem
         # at the same point.
-        i = d.num_ever_used_items - 2
-        while i >= 0 and not d.entries.valid(i):
+        i = index
+        while True:
             i -= 1
-        j = i + 1
-        assert j >= 0
-        d.num_ever_used_items = j
+            assert i >= 0
+            if d.entries.valid(i):    # must be at least one
+                break
+        d.num_ever_used_items = i + 1
 
     # If the dictionary is at least 87.5% dead items, then consider shrinking
     # it.
@@ -844,6 +868,50 @@
     else:
         ll_dict_reindex(d, new_size)
 
+def ll_ensure_indexes(d):
+    num = d.lookup_function_no
+    if num == FUNC_MUST_REINDEX:
+        ll_dict_create_initial_index(d)
+    else:
+        ll_assert((num & FUNC_MASK) != FUNC_MUST_REINDEX,
+                  "bad combination in lookup_function_no")
+
+def ll_dict_create_initial_index(d):
+    """Create the initial index for a dictionary.  The common case is
+    that 'd' is empty.  The uncommon case is that it is a prebuilt
+    dictionary frozen by translation, in which case we must rehash all
+    entries.  The common case must be seen by the JIT.
+    """
+    if d.num_live_items == 0:
+        ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+        d.resize_counter = DICT_INITSIZE * 2
+    else:
+        ll_dict_rehash_after_translation(d)
+
+ at jit.dont_look_inside
+def ll_dict_rehash_after_translation(d):
+    assert d.num_live_items == d.num_ever_used_items
+    assert not d.indexes
+    #
+    # recompute all hashes.  Needed if they are stored in d.entries,
+    # but do it anyway: otherwise, e.g. a string-keyed dictionary
+    # won't have a fasthash on its strings if their hash is still
+    # uncomputed.
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    for i in range(d.num_ever_used_items):
+        assert d.entries.valid(i)
+        d_entry = d.entries[i]
+        h = d.keyhash(d_entry.key)
+        if hasattr(ENTRY, 'f_hash'):
+            d_entry.f_hash = h
+        #else: purely for the side-effect it can have on d_entry.key
+    #
+    # Use the smallest acceptable size for ll_dict_reindex
+    new_size = DICT_INITSIZE
+    while new_size * 2 - d.num_live_items * 3 <= 0:
+        new_size *= 2
+    ll_dict_reindex(d, new_size)
+
 def ll_dict_reindex(d, new_size):
     if bool(d.indexes) and _ll_len_of_d_indexes(d) == new_size:
         ll_clear_indexes(d, new_size)   # hack: we can reuse the same array
@@ -857,12 +925,33 @@
     entries = d.entries
     i = 0
     ibound = d.num_ever_used_items
-    while i < ibound:
-        if entries.valid(i):
-            hash = entries.hash(i)
-            ll_call_insert_clean_function(d, hash, i)
-        i += 1
-    #old_entries.delete() XXXX!
+    #
+    # Write four loops, moving the check for the value of 'fun' out of
+    # the loops.  A small speed-up over ll_call_insert_clean_function().
+    fun = d.lookup_function_no     # == lookup_function_no & FUNC_MASK
+    if fun == FUNC_BYTE:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_BYTE)
+            i += 1
+    elif fun == FUNC_SHORT:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_SHORT)
+            i += 1
+    elif IS_64BIT and fun == FUNC_INT:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_INT)
+            i += 1
+    elif fun == FUNC_LONG:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_LONG)
+            i += 1
+    else:
+        assert False
+
 
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
@@ -1013,10 +1102,11 @@
 def ll_newdict(DICT):
     d = DICT.allocate()
     d.entries = _ll_empty_array(DICT)
-    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    # Don't allocate an 'indexes' for empty dict.  It seems a typical
+    # program contains tons of empty dicts, so this might be a memory win.
     d.num_live_items = 0
     d.num_ever_used_items = 0
-    d.resize_counter = DICT_INITSIZE * 2
+    ll_no_initial_index(d)
     return d
 OrderedDictRepr.ll_newdict = staticmethod(ll_newdict)
 
@@ -1101,6 +1191,10 @@
                 # as soon as we do something like ll_dict_reindex().
                 if index == (dict.lookup_function_no >> FUNC_SHIFT):
                     dict.lookup_function_no += (1 << FUNC_SHIFT)
+                # note that we can't have modified a FUNC_MUST_REINDEX
+                # dict here because such dicts have no invalid entries
+                ll_assert((dict.lookup_function_no & FUNC_MASK) !=
+                      FUNC_MUST_REINDEX, "bad combination in _ll_dictnext")
             index = nextindex
         # clear the reference to the dict and prevent restarts
         iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
@@ -1146,6 +1240,8 @@
         return dict.entries[index].value
 
 def ll_dict_copy(dict):
+    ll_ensure_indexes(dict)
+
     DICT = lltype.typeOf(dict).TO
     newdict = DICT.allocate()
     newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
@@ -1180,6 +1276,10 @@
     DICT = lltype.typeOf(d).TO
     old_entries = d.entries
     d.entries = _ll_empty_array(DICT)
+    # note: we can't remove the index here, because it is possible that
+    # crazy Python code calls d.clear() from the method __eq__() called
+    # from ll_dict_lookup(d).  Instead, stick to the rule that once a
+    # dictionary has got an index, it will always have one.
     ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
     d.num_live_items = 0
     d.num_ever_used_items = 0
@@ -1190,6 +1290,7 @@
 def ll_dict_update(dic1, dic2):
     if dic1 == dic2:
         return
+    ll_ensure_indexes(dic2)    # needed for entries.hash() below
     ll_prepare_dict_update(dic1, dic2.num_live_items)
     i = 0
     while i < dic2.num_ever_used_items:
@@ -1216,6 +1317,7 @@
     # the case where dict.update() actually has a lot of collisions.
     # If num_extra is much greater than d.num_live_items the conditional_call
     # will trigger anyway, which is really the goal.
+    ll_ensure_indexes(d)
     x = num_extra - d.num_live_items
     jit.conditional_call(d.resize_counter <= x * 3,
                          _ll_dict_resize_to, d, num_extra)
@@ -1275,6 +1377,7 @@
     if dic.num_live_items == 0:
         raise KeyError
 
+    ll_ensure_indexes(dic)
     entries = dic.entries
 
     # find the last entry.  It's unclear if the loop below is still
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -1,9 +1,9 @@
 from weakref import WeakValueDictionary
 
 from rpython.annotator import model as annmodel
-from rpython.rlib import jit, types
+from rpython.rlib import jit, types, objectmodel
 from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated,
-    ll_hash_string, keepalive_until_here, specialize, enforceargs)
+    ll_hash_string, keepalive_until_here, specialize, enforceargs, dont_inline)
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.error import TyperError
@@ -383,6 +383,8 @@
             return 0
 
     @staticmethod
+    @dont_inline
+    @jit.dont_look_inside
     def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the
@@ -400,6 +402,7 @@
 
     @staticmethod
     def ll_strfasthash(s):
+        ll_assert(s.hash != 0, "ll_strfasthash: hash==0")
         return s.hash     # assumes that the hash is already computed
 
     @staticmethod
@@ -1258,7 +1261,8 @@
                               'gethash': LLHelpers.ll_strhash,
                               'length': LLHelpers.ll_length,
                               'find': LLHelpers.ll_find,
-                              'rfind': LLHelpers.ll_rfind}))
+                              'rfind': LLHelpers.ll_rfind},
+                    hints={'remove_hash': True}))
 UNICODE.become(GcStruct('rpy_unicode', ('hash', Signed),
                         ('chars', Array(UniChar, hints={'immutable': True})),
                         adtmeths={'malloc' : staticAdtMethod(mallocunicode),
@@ -1266,8 +1270,8 @@
                                   'copy_contents' : staticAdtMethod(copy_unicode_contents),
                                   'copy_contents_from_str' : staticAdtMethod(copy_unicode_contents),
                                   'gethash': LLHelpers.ll_strhash,
-                                  'length': LLHelpers.ll_length}
-                        ))
+                                  'length': LLHelpers.ll_length},
+                    hints={'remove_hash': True}))
 
 
 # TODO: make the public interface of the rstr module cleaner
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -749,22 +749,10 @@
     assert hash3 == identityhash(s3)
     assert hash3 == identityhash(s3.super)
     assert hash3 == identityhash(s3.super.super)
-    py.test.raises(ValueError, init_identity_hash, s3, hash3^1)
-    py.test.raises(ValueError, init_identity_hash, s3.super, hash3^4)
-    py.test.raises(ValueError, init_identity_hash, s3.super.super, hash3^9)
-
-    s3 = malloc(S3)
-    init_identity_hash(s3.super, -123)
-    assert -123 == identityhash(s3)
-    assert -123 == identityhash(s3.super)
-    assert -123 == identityhash(s3.super.super)
-    py.test.raises(ValueError, init_identity_hash, s3, 4313)
-    py.test.raises(ValueError, init_identity_hash, s3.super, 0)
-    py.test.raises(ValueError, init_identity_hash, s3.super.super, -124)
 
     from rpython.rtyper.lltypesystem import llmemory
     p3 = cast_opaque_ptr(llmemory.GCREF, s3)
-    assert -123 == identityhash(p3)
+    assert hash3 == identityhash(p3)
 
     A = GcArray(Signed)
     a = malloc(A, 3)
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -170,7 +170,6 @@
                             ('subclassrange_max', Signed),
                             ('rtti', Ptr(RuntimeTypeInfo)),
                             ('name', Ptr(rstr.STR)),
-                            ('hash', Signed),
                             ('instantiate', Ptr(FuncType([], OBJECTPTR))),
                             hints={'immutable': True}))
 # non-gc case
@@ -338,7 +337,6 @@
 
     def fill_vtable_root(self, vtable):
         """Initialize the head of the vtable."""
-        vtable.hash = hash(self)
         # initialize the 'subclassrange_*' and 'name' fields
         if self.classdef is not None:
             #vtable.parenttypeptr = self.rbase.getvtable()
@@ -785,7 +783,6 @@
     def initialize_prebuilt_instance(self, value, classdef, result):
         # must fill in the hash cache before the other ones
         # (see test_circular_hash_initialization)
-        self.initialize_prebuilt_hash(value, result)
         self._initialize_data_flattenrec(self.initialize_prebuilt_data,
                                          value, classdef, result)
 
@@ -943,11 +940,6 @@
             rclass = getclassrepr(self.rtyper, classdef)
             result.typeptr = rclass.getvtable()
 
-    def initialize_prebuilt_hash(self, value, result):
-        llattrvalue = getattr(value, '__precomputed_identity_hash', None)
-        if llattrvalue is not None:
-            lltype.init_identity_hash(result, llattrvalue)
-
     def getfieldrepr(self, attr):
         """Return the repr used for the given attribute."""
         if attr in self.fields:

From pypy.commits at gmail.com  Tue Jan 31 11:44:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 08:44:10 -0800 (PST)
Subject: [pypy-commit] pypy rpython-hash: close branch, ready to merge
Message-ID: <5890beda.8dae1c0a.87f4a.723e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-hash
Changeset: r89855:4ca0b56e089d
Date: 2017-01-31 17:35 +0100
http://bitbucket.org/pypy/pypy/changeset/4ca0b56e089d/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Tue Jan 31 11:44:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 08:44:19 -0800 (PST)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <5890bee3.cf0d1c0a.8fab2.6e8e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89859:3b15f02f0f87
Date: 2017-01-31 17:42 +0100
http://bitbucket.org/pypy/pypy/changeset/3b15f02f0f87/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -125,3 +125,12 @@
 rffi structures in cpyext can now be created by parsing simple C headers.
 Additionally, the cts object that holds the parsed information can act like
 cffi's ffi objects, with the methods cts.cast() and cts.gettype().
+
+.. branch: rpython-hash
+
+Don't freeze hashes in the translated pypy.  In practice, that means
+that we can now translate PyPy with the option --hash=siphash24 and get
+the same hashes as CPython 3.5, which can be randomized (in a
+crypographically good way).  It is the default in PyPy3.  The default of
+PyPy2 remains unchanged: there are user programs out there that depend
+on constant hashes (or even sometimes on specific hash results).

From pypy.commits at gmail.com  Tue Jan 31 11:44:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 08:44:20 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5890bee4.896f1c0a.a3a93.7267@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89860:33b30ce4c76c
Date: 2017-01-31 17:43 +0100
http://bitbucket.org/pypy/pypy/changeset/33b30ce4c76c/

Log:	hg merge default

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -56,6 +56,9 @@
 .. _`arm document`: http://rpython.readthedocs.org/en/latest/arm.html
 .. _`RPython documentation`: http://rpython.readthedocs.org
 
+The host Python needs to have CFFI installed. If translating on PyPy, CFFI is
+already installed. If translating on CPython, you need to install it, e.g. 
+using ``pip install cffi``.
 
 To build PyPy on Unix using the C translation backend, you need at least a C
 compiler and ``make`` installed. Further, some optional modules have additional
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -134,3 +134,12 @@
 rffi structures in cpyext can now be created by parsing simple C headers.
 Additionally, the cts object that holds the parsed information can act like
 cffi's ffi objects, with the methods cts.cast() and cts.gettype().
+
+.. branch: rpython-hash
+
+Don't freeze hashes in the translated pypy.  In practice, that means
+that we can now translate PyPy with the option --hash=siphash24 and get
+the same hashes as CPython 3.5, which can be randomized (in a
+crypographically good way).  It is the default in PyPy3.  The default of
+PyPy2 remains unchanged: there are user programs out there that depend
+on constant hashes (or even sometimes on specific hash results).

From pypy.commits at gmail.com  Tue Jan 31 11:44:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 08:44:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge py3.5-siphash24
Message-ID: <5890bee1.4a86df0a.172f0.fad9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89858:94325768daae
Date: 2017-01-31 17:37 +0100
http://bitbucket.org/pypy/pypy/changeset/94325768daae/

Log:	hg merge py3.5-siphash24

diff too long, truncating to 2000 out of 2956 lines

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -201,6 +201,13 @@
                "issue, you can disable them here",
                default=True),
 
+    ChoiceOption("hash",
+                 "The hash function to use for strings: fnv from CPython 2.7"
+                 " or siphash24 from CPython >= 3.4",
+                 ["fnv", "siphash24"],
+                 default="siphash24",
+                 cmdline="--hash"),
+
     OptionDescription("std", "Standard Object Space Options", [
         BoolOption("withtproxy", "support transparent proxies",
                    default=True),
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -36,6 +36,7 @@
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
         w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+        hashfunc = space.config.objspace.hash
     else:
         w_initstdio = space.appexec([], """():
             return lambda unbuffered: None
@@ -46,6 +47,10 @@
             from rpython.jit.backend.hlinfo import highleveljitinfo
             highleveljitinfo.sys_executable = argv[0]
 
+        if hashfunc == "siphash24":
+            from rpython.rlib import rsiphash
+            rsiphash.enable_siphash24()
+
         #debug("entry point starting")
         #for arg in argv:
         #    debug(" argv -> " + arg)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -23,13 +23,34 @@
 # ____________________________________________________________
 
 class UniqueCache:
+    for_testing = False    # set to True on the class level in test_c.py
+
     def __init__(self, space):
         self.ctvoid = None      # Cache for the 'void' type
         self.ctvoidp = None     # Cache for the 'void *' type
         self.ctchara = None     # Cache for the 'char[]' type
         self.primitives = {}    # Cache for {name: primitive_type}
         self.functions = []     # see _new_function_type()
-        self.for_testing = False
+        self.functions_packed = None     # only across translation
+
+    def _cleanup_(self):
+        import gc
+        assert self.functions_packed is None
+        # Note: a full PyPy translation may still have
+        # 'self.functions == []' at this point, possibly depending
+        # on details.  Code tested directly in test_ffi_obj
+        gc.collect()
+        funcs = []
+        for weakdict in self.functions:
+            funcs += weakdict._dict.values()
+        del self.functions[:]
+        self.functions_packed = funcs if len(funcs) > 0 else None
+
+    def unpack_functions(self):
+        for fct in self.functions_packed:
+            _record_function_type(self, fct)
+        self.functions_packed = None
+
 
 def _clean_cache(space):
     "NOT_RPYTHON"
@@ -622,7 +643,7 @@
     for w_arg in fargs:
         y = compute_identity_hash(w_arg)
         x = intmask((1000003 * x) ^ y)
-    x ^= (ellipsis - abi)
+    x ^= ellipsis + 2 * abi
     if unique_cache.for_testing:    # constant-folded to False in translation;
         x &= 3                      # but for test, keep only 2 bits of hash
     return x
@@ -646,6 +667,8 @@
     # one such dict, but in case of hash collision, there might be
     # more.
     unique_cache = space.fromcache(UniqueCache)
+    if unique_cache.functions_packed is not None:
+        unique_cache.unpack_functions()
     func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
     for weakdict in unique_cache.functions:
         ctype = weakdict.get(func_hash)
@@ -674,13 +697,18 @@
     #
     fct = ctypefunc.W_CTypeFunc(space, fargs, fresult, ellipsis, abi)
     unique_cache = space.fromcache(UniqueCache)
-    func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
+    _record_function_type(unique_cache, fct)
+    return fct
+
+def _record_function_type(unique_cache, fct):
+    from pypy.module._cffi_backend import ctypefunc
+    #
+    func_hash = _func_key_hash(unique_cache, fct.fargs, fct.ctitem,
+                               fct.ellipsis, fct.abi)
     for weakdict in unique_cache.functions:
         if weakdict.get(func_hash) is None:
-            weakdict.set(func_hash, fct)
             break
     else:
         weakdict = rweakref.RWeakValueDictionary(int, ctypefunc.W_CTypeFunc)
         unique_cache.functions.append(weakdict)
-        weakdict.set(func_hash, fct)
-    return fct
+    weakdict.set(func_hash, fct)
diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py
--- a/pypy/module/_cffi_backend/test/test_c.py
+++ b/pypy/module/_cffi_backend/test/test_c.py
@@ -37,6 +37,7 @@
     def setup_class(cls):
         testfuncs_w = []
         keepalive_funcs = []
+        UniqueCache.for_testing = True
 
         test_lib_c = tmpdir.join('_test_lib.c')
         src_test_lib_c = py.path.local(__file__).dirpath().join('_test_lib.c')
@@ -100,11 +101,12 @@
             _all_test_c.find_and_load_library = func
             _all_test_c._testfunc = testfunc
         """)
-        UniqueCache.for_testing = True
 
     def teardown_method(self, method):
+        _clean_cache(self.space)
+
+    def teardown_class(cls):
         UniqueCache.for_testing = False
-        _clean_cache(self.space)
 
 
 all_names = ', '.join(Module.interpleveldefs.keys())
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -1,5 +1,23 @@
+from pypy.module._cffi_backend import newtype
 from pypy.module._cffi_backend.newtype import _clean_cache
 
+
+class TestFFIObj:
+    spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
+
+    def teardown_method(self, meth):
+        _clean_cache(self.space)
+
+    def test_new_function_type_during_translation(self):
+        space = self.space
+        BInt = newtype.new_primitive_type(space, "int")
+        BFunc = newtype.new_function_type(space, space.wrap([BInt]), BInt)
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+        unique_cache = space.fromcache(newtype.UniqueCache)
+        unique_cache._cleanup_()
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+
+
 class AppTestFFIObj:
     spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
 
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -194,6 +194,15 @@
         W_WeakrefBase.__init__(self, space, w_obj, w_callable)
         self.w_hash = None
 
+    def _cleanup_(self):
+        # When a prebuilt weakref is frozen inside a translation, if
+        # this weakref has got an already-cached w_hash, then throw it
+        # away.  That's because the hash value will change after
+        # translation.  It will be recomputed the first time we ask for
+        # it.  Note that such a frozen weakref, if not dead, will point
+        # to a frozen object, so it will never die.
+        self.w_hash = None
+
     def descr__init__weakref(self, space, w_obj, w_callable=None,
                              __args__=None):
         if __args__.arguments_w:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2127,6 +2127,12 @@
         raise wrap_oserror(space, e, eintr_retry=False)
     return space.wrap(res)
 
+class SigCheck:
+    pass
+_sigcheck = SigCheck()
+def _signal_checker():
+    _sigcheck.space.getexecutioncontext().checksignals()
+
 @unwrap_spec(size=int)
 def urandom(space, size):
     """urandom(size) -> str
@@ -2134,9 +2140,12 @@
     Return a string of 'size' random bytes suitable for cryptographic use.
     """
     context = get(space).random_context
-    signal_checker = space.getexecutioncontext().checksignals
     try:
-        return space.newbytes(rurandom.urandom(context, n, signal_checker))
+        # urandom() takes a final argument that should be a regular function,
+        # not a bound method like 'getexecutioncontext().checksignals'.
+        # Otherwise, we can't use it from several independent places.
+        _sigcheck.space = space
+        return space.newbytes(rurandom.urandom(context, n, _signal_checker))
     except OSError as e:
         # 'rurandom' should catch and retry internally if it gets EINTR
         # (at least in os.read(), which is probably enough in practice)
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -5,7 +5,6 @@
 from pypy.objspace.std.complexobject import HASH_IMAG
 from pypy.objspace.std.floatobject import HASH_INF, HASH_NAN
 from pypy.objspace.std.intobject import HASH_MODULUS
-from pypy.objspace.std.bytesobject import HASH_ALGORITHM
 from pypy.interpreter import gateway
 from rpython.rlib import rbigint, rfloat
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -79,11 +78,22 @@
     return space.call_function(w_int_info, space.newtuple(info_w))
 
 def get_hash_info(space):
-    HASH_HASH_BITS = 8 * rffi.sizeof(lltype.Signed)
-    HASH_SEED_BITS = 0    # XXX don't know what this is supposed to be
+    HASH_ALGORITHM = space.config.objspace.hash
+    if space.config.objspace.hash == "fnv":
+        HASH_HASH_BITS = 8 * rffi.sizeof(lltype.Signed)
+        HASH_SEED_BITS = 0
+        #   CPython has  ^ > 0  here, but the seed of "fnv" is of limited
+        #   use, so we don't implement it
+    elif space.config.objspace.hash == "siphash24":
+        HASH_HASH_BITS = 64
+        HASH_SEED_BITS = 128
+    else:
+        assert 0, "please add the parameters for this different hash function"
+
+    HASH_WIDTH = 8 * rffi.sizeof(lltype.Signed)
     HASH_CUTOFF = 0
     info_w = [
-        space.wrap(8 * rffi.sizeof(lltype.Signed)),
+        space.wrap(HASH_WIDTH),
         space.wrap(HASH_MODULUS),
         space.wrap(HASH_INF),
         space.wrap(HASH_NAN),
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -3,7 +3,7 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin, newlist_hint,
-    resizelist_hint, HASH_ALGORITHM)
+    resizelist_hint)
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder
 
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -563,6 +563,11 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
+    def _cleanup_(self):
+        # in case there are frozenset objects existing during
+        # translation, make sure we don't translate a cached hash
+        self.hash = 0
+
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_FrozensetObject):
             return False
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -287,7 +287,7 @@
                     for ek, ev in items:
                         result.dictdef.generalize_key(self.immutablevalue(ek))
                         result.dictdef.generalize_value(self.immutablevalue(ev))
-                        result.dictdef.seen_prebuilt_key(ek)
+                        #dictdef.seen_prebuilt_key(ek)---not needed any more
                     seen_elements = len(items)
                     # if the dictionary grew during the iteration,
                     # start over again
diff --git a/rpython/annotator/dictdef.py b/rpython/annotator/dictdef.py
--- a/rpython/annotator/dictdef.py
+++ b/rpython/annotator/dictdef.py
@@ -115,13 +115,5 @@
     def generalize_value(self, s_value):
         self.dictvalue.generalize(s_value)
 
-    def seen_prebuilt_key(self, x):
-        # In case we are an r_dict, we don't ask for the hash ourselves.
-        # Note that if the custom hashing function ends up asking for
-        # the hash of x, then it must use compute_hash() itself, so it
-        # works out.
-        if not self.dictkey.custom_eq_hash:
-            compute_hash(x)
-
     def __repr__(self):
         return '<{%r: %r}>' % (self.dictkey.s_value, self.dictvalue.s_value)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3704,25 +3704,6 @@
         s = a.build_types(f, [int])
         assert s.const == 0
 
-    def test_hash_sideeffect(self):
-        class X:
-            pass
-        x1 = X()
-        x2 = X()
-        x3 = X()
-        d = {(2, x1): 5, (3, x2): 7}
-        def f(n, m):
-            if   m == 1: x = x1
-            elif m == 2: x = x2
-            else:        x = x3
-            return d[n, x]
-        a = self.RPythonAnnotator()
-        s = a.build_types(f, [int, int])
-        assert s.knowntype == int
-        assert hasattr(x1, '__precomputed_identity_hash')
-        assert hasattr(x2, '__precomputed_identity_hash')
-        assert not hasattr(x3, '__precomputed_identity_hash')
-
     def test_contains_of_empty_dict(self):
         class A(object):
             def meth(self):
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -201,10 +201,6 @@
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
-    ChoiceOption("hash",
-                 "The hash to use for strings",
-                 ["rpython", "siphash24"],
-                 default="rpython", cmdline="--hash"),
 
     OptionDescription("backendopt", "Backend Optimization Options", [
         # control inlining
@@ -394,12 +390,6 @@
         if sys.platform == "darwin" or sys.platform =="win32":
             raise ConfigError("'asmgcc' not supported on this platform")
 
-def apply_extra_settings(config):
-    # make the setting of config.hash definitive
-    from rpython.rlib.objectmodel import set_hash_algorithm
-    config.translation.hash = config.translation.hash
-    set_hash_algorithm(config.translation.hash)
-
 # ----------------------------------------------------------------
 
 def set_platform(config):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -117,9 +117,7 @@
 
 # The following flag is set on nursery objects of which we asked the id
 # or the identityhash.  It means that a space of the size of the object
-# has already been allocated in the nonmovable part.  The same flag is
-# abused to mark prebuilt objects whose hash has been taken during
-# translation and is statically recorded.
+# has already been allocated in the nonmovable part.
 GCFLAG_HAS_SHADOW   = first_gcflag << 3
 
 # The following flag is set temporarily on some objects during a major
@@ -208,10 +206,6 @@
     # by GCFLAG_xxx above.
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
-    # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
-    #     then they are one word longer, the extra word storing the hash.
-
 
     # During a minor collection, the objects in the nursery that are
     # moved outside are changed in-place: their header is replaced with
@@ -2640,40 +2634,22 @@
         return shadow
     _find_shadow._dont_inline_ = True
 
-    @specialize.arg(2)
-    def id_or_identityhash(self, gcobj, is_hash):
+    def id_or_identityhash(self, gcobj):
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
         obj = llmemory.cast_ptr_to_adr(gcobj)
-        #
         if self.is_valid_gc_object(obj):
             if self.is_in_nursery(obj):
                 obj = self._find_shadow(obj)
-            elif is_hash:
-                if self.header(obj).tid & GCFLAG_HAS_SHADOW:
-                    #
-                    # For identityhash(), we need a special case for some
-                    # prebuilt objects: their hash must be the same before
-                    # and after translation.  It is stored as an extra word
-                    # after the object.  But we cannot use it for id()
-                    # because the stored value might clash with a real one.
-                    size = self.get_size(obj)
-                    i = (obj + size).signed[0]
-                    # Important: the returned value is not mangle_hash()ed!
-                    return i
-        #
-        i = llmemory.cast_adr_to_int(obj)
-        if is_hash:
-            i = mangle_hash(i)
-        return i
+        return llmemory.cast_adr_to_int(obj)
     id_or_identityhash._always_inline_ = True
 
     def id(self, gcobj):
-        return self.id_or_identityhash(gcobj, False)
+        return self.id_or_identityhash(gcobj)
 
     def identityhash(self, gcobj):
-        return self.id_or_identityhash(gcobj, True)
+        return mangle_hash(self.id_or_identityhash(gcobj))
 
     # ----------
     # Finalizers
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -104,9 +104,7 @@
 
 # The following flag is set on nursery objects of which we asked the id
 # or the identityhash.  It means that a space of the size of the object
-# has already been allocated in the nonmovable part.  The same flag is
-# abused to mark prebuilt objects whose hash has been taken during
-# translation and is statically recorded.
+# has already been allocated in the nonmovable part.
 GCFLAG_HAS_SHADOW   = first_gcflag << 3
 
 # The following flag is set temporarily on some objects during a major
@@ -149,9 +147,6 @@
     # by GCFLAG_xxx above.
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
-    # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
-    #     then they are one word longer, the extra word storing the hash.
 
     _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True})
 
@@ -1868,40 +1863,22 @@
         return shadow
     _find_shadow._dont_inline_ = True
 
-    @specialize.arg(2)
-    def id_or_identityhash(self, gcobj, is_hash):
+    def id_or_identityhash(self, gcobj):
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
         obj = llmemory.cast_ptr_to_adr(gcobj)
-        #
         if self.is_valid_gc_object(obj):
             if self.is_in_nursery(obj):
                 obj = self._find_shadow(obj)
-            elif is_hash:
-                if self.header(obj).tid & GCFLAG_HAS_SHADOW:
-                    #
-                    # For identityhash(), we need a special case for some
-                    # prebuilt objects: their hash must be the same before
-                    # and after translation.  It is stored as an extra word
-                    # after the object.  But we cannot use it for id()
-                    # because the stored value might clash with a real one.
-                    size = self.get_size(obj)
-                    i = (obj + size).signed[0]
-                    # Important: the returned value is not mangle_hash()ed!
-                    return i
-        #
-        i = llmemory.cast_adr_to_int(obj)
-        if is_hash:
-            i = mangle_hash(i)
-        return i
+        return llmemory.cast_adr_to_int(obj)
     id_or_identityhash._always_inline_ = True
 
     def id(self, gcobj):
-        return self.id_or_identityhash(gcobj, False)
+        return self.id_or_identityhash(gcobj)
 
     def identityhash(self, gcobj):
-        return self.id_or_identityhash(gcobj, True)
+        return mangle_hash(self.id_or_identityhash(gcobj))
 
     # ----------
     # Finalizers
diff --git a/rpython/memory/gc/semispace.py b/rpython/memory/gc/semispace.py
--- a/rpython/memory/gc/semispace.py
+++ b/rpython/memory/gc/semispace.py
@@ -48,9 +48,6 @@
 
     HDR = lltype.Struct('header', ('tid', lltype.Signed))   # XXX or rffi.INT?
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', _GCFLAG_HASH_BASE * 0x2
-    # ^^^ prebuilt objects either have GC_HASH_TAKEN_ADDR or they
-    #     have GC_HASH_HASFIELD (and then they are one word longer).
     FORWARDSTUB = lltype.GcStruct('forwarding_stub',
                                   ('forw', llmemory.Address))
     FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -11,7 +11,7 @@
 class BoehmGCTransformer(GCTransformer):
     malloc_zero_filled = True
     FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
-    HDR = lltype.Struct("header", ("hash", lltype.Signed))
+    NO_HEADER = True
 
     def __init__(self, translator, inline=False):
         super(BoehmGCTransformer, self).__init__(translator, inline=inline)
@@ -29,13 +29,8 @@
         ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length
         ll_malloc_varsize = mh.ll_malloc_varsize
 
-        HDRPTR = lltype.Ptr(self.HDR)
-
         def ll_identityhash(addr):
-            obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
-            h = obj.hash
-            if h == 0:
-                obj.hash = h = ~llmemory.cast_adr_to_int(addr)
+            h = ~llmemory.cast_adr_to_int(addr)
             return h
 
         if self.translator:
@@ -194,11 +189,6 @@
                           resulttype = lltype.Signed)
         hop.genop('int_invert', [v_int], resultvar=hop.spaceop.result)
 
-    def gcheader_initdata(self, obj):
-        hdr = lltype.malloc(self.HDR, immortal=True)
-        hdr.hash = lltype.identityhash_nocache(obj._as_ptr())
-        return hdr._obj
-
 
 ########## weakrefs ##########
 # Boehm: weakref objects are small structures containing only a Boehm
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -610,25 +610,6 @@
     def special_funcptr_for_type(self, TYPE):
         return self.layoutbuilder.special_funcptr_for_type(TYPE)
 
-    def gc_header_for(self, obj, needs_hash=False):
-        hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj)
-        withhash, flag = self.gcdata.gc.withhash_flag_is_in_field
-        x = getattr(hdr, withhash)
-        TYPE = lltype.typeOf(x)
-        x = lltype.cast_primitive(lltype.Signed, x)
-        if needs_hash:
-            x |= flag       # set the flag in the header
-        else:
-            x &= ~flag      # clear the flag in the header
-        x = lltype.cast_primitive(TYPE, x)
-        setattr(hdr, withhash, x)
-        return hdr
-
-    def get_hash_offset(self, T):
-        type_id = self.get_type_id(T)
-        assert not self.gcdata.q_is_varsize(type_id)
-        return self.gcdata.q_fixed_size(type_id)
-
     def finish_tables(self):
         group = self.layoutbuilder.close_table()
         log.info("assigned %s typeids" % (len(group.members), ))
@@ -1514,22 +1495,9 @@
 
     def gcheader_initdata(self, obj):
         o = lltype.top_container(obj)
-        needs_hash = self.get_prebuilt_hash(o) is not None
-        hdr = self.gc_header_for(o, needs_hash)
+        hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(o)
         return hdr._obj
 
-    def get_prebuilt_hash(self, obj):
-        # for prebuilt objects that need to have their hash stored and
-        # restored.  Note that only structures that are StructNodes all
-        # the way have their hash stored (and not e.g. structs with var-
-        # sized arrays at the end).  'obj' must be the top_container.
-        TYPE = lltype.typeOf(obj)
-        if not isinstance(TYPE, lltype.GcStruct):
-            return None
-        if TYPE._is_varsize():
-            return None
-        return getattr(obj, '_hash_cache_', None)
-
     def get_finalizer_queue_index(self, hop):
         fq_tag = hop.spaceop.args[0].value
         assert 'FinalizerQueue TAG' in fq_tag.expr
diff --git a/rpython/memory/gctransform/refcounting.py b/rpython/memory/gctransform/refcounting.py
--- a/rpython/memory/gctransform/refcounting.py
+++ b/rpython/memory/gctransform/refcounting.py
@@ -18,8 +18,7 @@
 class RefcountingGCTransformer(GCTransformer):
     malloc_zero_filled = True
 
-    HDR = lltype.Struct("header", ("refcount", lltype.Signed),
-                                  ("hash", lltype.Signed))
+    HDR = lltype.Struct("header", ("refcount", lltype.Signed))
 
     def __init__(self, translator):
         super(RefcountingGCTransformer, self).__init__(translator, inline=True)
@@ -77,10 +76,7 @@
         ll_malloc_varsize = mh.ll_malloc_varsize
 
         def ll_identityhash(addr):
-            obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
-            h = obj.hash
-            if h == 0:
-                obj.hash = h = llmemory.cast_adr_to_int(addr)
+            h = llmemory.cast_adr_to_int(addr)
             return h
 
         if self.translator:
@@ -178,7 +174,6 @@
             if not self.gcheaderbuilder.get_header(p):
                 hdr = self.gcheaderbuilder.new_header(p)
                 hdr.refcount = sys.maxint // 2
-                hdr.hash = lltype.identityhash_nocache(p)
 
     def static_deallocation_funcptr_for_type(self, TYPE):
         if TYPE in self.static_deallocator_funcptrs:
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -374,9 +374,6 @@
         return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP),
                                         lltype.nullptr(ARRAY_TYPEID_MAP)))
 
-    def get_prebuilt_hash(self, obj):
-        return None
-
 
 class MinimalGCTransformer(BaseGCTransformer):
     def __init__(self, parenttransformer):
diff --git a/rpython/rlib/_rweakvaldict.py b/rpython/rlib/_rweakvaldict.py
--- a/rpython/rlib/_rweakvaldict.py
+++ b/rpython/rlib/_rweakvaldict.py
@@ -76,12 +76,16 @@
             bk = self.rtyper.annotator.bookkeeper
             classdef = bk.getuniqueclassdef(weakdict._valueclass)
             r_value = getinstancerepr(self.rtyper, classdef)
+            any_value = False
             for dictkey, dictvalue in weakdict._dict.items():
                 llkey = self.r_key.convert_const(dictkey)
                 llvalue = r_value.convert_const(dictvalue)
                 if llvalue:
                     llvalue = lltype.cast_pointer(rclass.OBJECTPTR, llvalue)
                     self.ll_set_nonnull(l_dict, llkey, llvalue)
+                    any_value = True
+            if any_value:
+                l_dict.resize_counter = -1
             return l_dict
 
     def rtype_method_get(self, hop):
@@ -114,6 +118,8 @@
 
     @jit.dont_look_inside
     def ll_get(self, d, llkey):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         #llop.debug_print(lltype.Void, i, 'get')
@@ -132,6 +138,8 @@
 
     @jit.dont_look_inside
     def ll_set_nonnull(self, d, llkey, llvalue):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         valueref = weakref_create(llvalue)    # GC effects here, before the rest
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
@@ -147,6 +155,8 @@
 
     @jit.dont_look_inside
     def ll_set_null(self, d, llkey):
+        if d.resize_counter < 0:
+            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         if d.entries.everused(i):
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -441,7 +441,7 @@
             except OSError as e:
                 os.write(2, "Could not start GDB: %s" % (
                     os.strerror(e.errno)))
-                raise SystemExit
+                os._exit(1)
         else:
             time.sleep(1)  # give the GDB time to attach
 
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -465,8 +465,14 @@
 
     Note that this can return 0 or -1 too.
 
-    It returns the same number, both before and after translation.
-    Dictionaries don't need to be rehashed after translation.
+    NOTE: It returns a different number before and after translation!
+    Dictionaries will be rehashed when the translated program starts.
+    Be careful about other places that store or depend on a hash value:
+    if such a place can exist before translation, you should add for
+    example a _cleanup_() method to clear this cache during translation.
+
+    (Nowadays we could completely remove compute_hash() and decide that
+    hash(x) is valid RPython instead, at least for the types listed here.)
     """
     if isinstance(x, (str, unicode)):
         return _hash_string(x)
@@ -484,17 +490,11 @@
     """RPython equivalent of object.__hash__(x).  This returns the
     so-called 'identity hash', which is the non-overridable default hash
     of Python.  Can be called for any RPython-level object that turns
-    into a GC object, but not NULL.  The value is not guaranteed to be the
-    same before and after translation, except for RPython instances on the
-    lltypesystem.
+    into a GC object, but not NULL.  The value will be different before
+    and after translation (WARNING: this is a change with older RPythons!)
     """
     assert x is not None
-    result = object.__hash__(x)
-    try:
-        x.__dict__['__precomputed_identity_hash'] = result
-    except (TypeError, AttributeError):
-        pass
-    return result
+    return object.__hash__(x)
 
 def compute_unique_id(x):
     """RPython equivalent of id(x).  The 'x' must be an RPython-level
@@ -519,21 +519,17 @@
 
 # ----------
 
-HASH_ALGORITHM = "rpython"  # XXX Is there a better name?
-HASH_ALGORITHM_FIXED = False
+def _hash_string(s):
+    """The default algorithm behind compute_hash() for a string or a unicode.
+    This is a modified Fowler-Noll-Vo (FNV) hash.  According to Wikipedia,
+    FNV needs carefully-computed constants called FNV primes and FNV offset
+    basis, which are absent from the present algorithm.  Nevertheless,
+    this matches CPython 2.7 without -R, which has proven a good hash in
+    practice (even if not crypographical nor randomizable).
 
- at not_rpython
-def set_hash_algorithm(algo):
-    """Must be called very early, before any string is hashed with
-    compute_hash()!"""
-    global HASH_ALGORITHM
-    if HASH_ALGORITHM != algo:
-        assert not HASH_ALGORITHM_FIXED, "compute_hash() already called!"
-        assert algo in ("rpython", "siphash24")
-        HASH_ALGORITHM = algo
-
-
-def _hash_string_rpython(s):
+    There is a mechanism to use another one in programs after translation.
+    See rsiphash.py, which implements the algorithm of CPython >= 3.4.
+    """
     from rpython.rlib.rarithmetic import intmask
 
     length = len(s)
@@ -547,100 +543,8 @@
     x ^= length
     return intmask(x)
 
-
- at not_rpython
-def _hash_string_siphash24(s):
-    """This version is called when untranslated only."""
-    import array
-    from rpython.rlib.rsiphash import siphash24
-    from rpython.rtyper.lltypesystem import lltype, rffi
-    from rpython.rlib.rarithmetic import intmask
-
-    if not isinstance(s, str):
-        if isinstance(s, unicode):
-            lst = map(ord, s)
-        else:
-            lst = map(ord, s.chars)    # for rstr.STR or UNICODE
-        # NOTE: a latin-1 unicode string must have the same hash as the
-        # corresponding byte string.
-        if all(n <= 0xFF for n in lst):
-            kind = "B"
-        elif rffi.sizeof(lltype.UniChar) == 4:
-            kind = "I"
-        else:
-            kind = "H"
-        s = array.array(kind, lst).tostring()
-    ptr = rffi.str2charp(s)
-    x = siphash24(ptr, len(s))
-    rffi.free_charp(ptr)
-    return intmask(x)
-
-def ll_hash_string_siphash24(ll_s):
-    """Called from lltypesystem/rstr.py.  'll_s' is a rstr.STR or UNICODE."""
-    from rpython.rlib.rsiphash import siphash24
-    from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
-    from rpython.rlib.rarithmetic import intmask
-
-    length = len(ll_s.chars)
-    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:
-        # no GC operation from here!
-        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
-    else:
-        # NOTE: a latin-1 unicode string must have the same hash as the
-        # corresponding byte string.  If the unicode is all within
-        # 0-255, then we need to allocate a byte buffer and copy the
-        # latin-1 encoding in it manually.
-        for i in range(length):
-            if ord(ll_s.chars[i]) > 0xFF:
-                # no GC operation from here!
-                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
-                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
-                break
-        else:
-            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
-            i = 0
-            while i < length:
-                p[i] = chr(ord(ll_s.chars[i]))
-                i += 1
-            x = siphash24(llmemory.cast_ptr_to_adr(p), length)
-            lltype.free(p, flavor='raw')
-            return intmask(x)
-    x = siphash24(addr, length)
-    keepalive_until_here(ll_s)
-    return intmask(x)
-ll_hash_string_siphash24._jit_look_inside_ = False
-
-
- at not_rpython
-def _hash_string(s):
-    """The algorithm behind compute_hash() for a string or a unicode.
-    This version is only for untranslated usage, and 's' is a str or unicode.
-    """
-    global HASH_ALGORITHM_FIXED
-    HASH_ALGORITHM_FIXED = True
-    if HASH_ALGORITHM == "rpython":
-        return _hash_string_rpython(s)
-    if HASH_ALGORITHM == "siphash24":
-        return _hash_string_siphash24(s)
-    raise NotImplementedError
-
 def ll_hash_string(ll_s):
-    """The algorithm behind compute_hash() for a string or a unicode.
-    This version is called from lltypesystem/rstr.py, and 'll_s' is a
-    rstr.STR or rstr.UNICODE.
-    """
-    if not we_are_translated():
-        global HASH_ALGORITHM_FIXED
-        HASH_ALGORITHM_FIXED = True
-    if HASH_ALGORITHM == "rpython":
-        return _hash_string_rpython(ll_s.chars)
-    if HASH_ALGORITHM == "siphash24":
-        if we_are_translated():
-            return ll_hash_string_siphash24(ll_s)
-        else:
-            return _hash_string_siphash24(ll_s)
-    raise NotImplementedError
-
+    return _hash_string(ll_s.chars)
 
 def _hash_float(f):
     """The algorithm behind compute_hash() for a float.
@@ -698,6 +602,21 @@
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
+    _about_ = ll_hash_string
+    # this is only used when annotating the code in rstr.py, and so
+    # it always occurs after the RPython program signalled its intent
+    # to use a different hash.  The code below overwrites the use of
+    # ll_hash_string() to make the annotator think a possibly different
+    # function was called.
+
+    def compute_annotation(self):
+        from rpython.annotator import model as annmodel
+        bk = self.bookkeeper
+        translator = bk.annotator.translator
+        fn = getattr(translator, 'll_hash_string', ll_hash_string)
+        return annmodel.SomePBC([bk.getdesc(fn)])
+
+class Entry(ExtRegistryEntry):
     _about_ = compute_identity_hash
 
     def compute_result_annotation(self, s_x):
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -1,12 +1,24 @@
-import sys, os, struct
+"""
+This module implements siphash-2-4, the hashing algorithm for strings
+and unicodes.  You can use it explicitly by calling siphash24() with
+a byte string, or you can use enable_siphash24() to enable the use
+of siphash-2-4 on all RPython strings and unicodes in your program
+after translation.
+"""
+import sys, os, errno
 from contextlib import contextmanager
-from rpython.rlib import rarithmetic
+from rpython.rlib import rarithmetic, rurandom
 from rpython.rlib.objectmodel import not_rpython, always_inline
-from rpython.rlib.rgc import no_collect
-from rpython.rlib.rarithmetic import r_uint64
+from rpython.rlib.objectmodel import we_are_translated, dont_inline
+from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib import rgc, jit, rposix
+from rpython.rlib.rarithmetic import r_uint64, r_uint32, r_uint
 from rpython.rlib.rawstorage import misaligned_is_fine
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rlib.nonconst import NonConstant
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.rtyper.annlowlevel import llhelper
 
 
 if sys.byteorder == 'little':
@@ -16,37 +28,164 @@
     _le64toh = rarithmetic.byteswap
 
 
-# Initialize the values of the secret seed: two 64-bit constants.
-# CPython picks a new seed every time 'python' starts.  PyPy cannot do
-# that as easily because many details may rely on getting the same hash
-# value before and after translation.  We can, however, pick a random
-# seed once per translation, which should already be quite good.
-#
-# XXX no, it is not: e.g. all Ubuntu installations of the same Ubuntu
-# would get the same seed.  That's not good enough.
+class Seed:
+    k0l = k1l = r_uint64(0)
+seed = Seed()
 
- at not_rpython
-def select_random_seed():
-    global k0, k1    # note: the globals k0, k1 are already byte-swapped
-    v0, v1 = struct.unpack("QQ", os.urandom(16))
-    k0 = r_uint64(v0)
-    k1 = r_uint64(v1)
 
-select_random_seed()
+def _decode64(s):
+    return (r_uint64(ord(s[0])) |
+            r_uint64(ord(s[1])) << 8 |
+            r_uint64(ord(s[2])) << 16 |
+            r_uint64(ord(s[3])) << 24 |
+            r_uint64(ord(s[4])) << 32 |
+            r_uint64(ord(s[5])) << 40 |
+            r_uint64(ord(s[6])) << 48 |
+            r_uint64(ord(s[7])) << 56)
+
+def select_random_seed(s):
+    """'s' is a string of length 16"""
+    seed.k0l = _decode64(s)
+    seed.k1l = _decode64(s[8:16])
+
+
+random_ctx = rurandom.init_urandom()
+strtoul = rffi.llexternal("strtoul", [rffi.CCHARP, rffi.CCHARPP, rffi.INT],
+                          rffi.ULONG, save_err=rffi.RFFI_SAVE_ERRNO)
+
+env_var_name = "PYTHONHASHSEED"
+
+def initialize_from_env():
+    # This uses the same algorithms as CPython 3.5.  The environment
+    # variable we read also defaults to "PYTHONHASHSEED".  If needed,
+    # a different RPython interpreter can patch the value of the
+    # global variable 'env_var_name', or just patch the whole
+    # initialize_from_env() function.
+    value = os.environ.get(env_var_name)
+    if value and value != "random":
+        with rffi.scoped_view_charp(value) as ptr:
+            with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as endptr:
+                endptr[0] = ptr
+                seed = strtoul(ptr, endptr, 10)
+                full = endptr[0][0] == '\x00'
+        seed = lltype.cast_primitive(lltype.Unsigned, seed)
+        if not full or seed > r_uint(4294967295) or (
+            rposix.get_saved_errno() == errno.ERANGE and
+            seed == lltype.cast_primitive(lltype.Unsigned,
+                                          rffi.cast(rffi.ULONG, -1))):
+            os.write(2,
+                "%s must be \"random\" or an integer "
+                "in range [0; 4294967295]\n" % (env_var_name,))
+            os._exit(1)
+        if not seed:
+            # disable the randomized hash
+            s = '\x00' * 16
+        else:
+            s = lcg_urandom(seed)
+    else:
+        try:
+            s = rurandom.urandom(random_ctx, 16)
+        except Exception as e:
+            os.write(2,
+                "%s: failed to get random numbers to initialize Python\n" %
+                (str(e),))
+            os._exit(1)
+            raise   # makes the annotator happy
+    select_random_seed(s)
+
+def lcg_urandom(x):
+    s = ''
+    for index in range(16):
+        x *= 214013
+        x += 2531011
+        s += chr((x >> 16) & 0xff)
+    return s
+
+
+_FUNC = lltype.Ptr(lltype.FuncType([], lltype.Void))
+
+def enable_siphash24():
+    """
+    Enable the use of siphash-2-4 for all RPython strings and unicodes
+    in the translated program.  You must call this function anywhere
+    from your interpreter (from a place that is annotated).  Don't call
+    more than once.
+    """
+
+class Entry(ExtRegistryEntry):
+    _about_ = enable_siphash24
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        if hasattr(translator, 'll_hash_string'):
+            assert translator.ll_hash_string == ll_hash_string_siphash24
+        else:
+            translator.ll_hash_string = ll_hash_string_siphash24
+        bk = self.bookkeeper
+        s_callable = bk.immutablevalue(initialize_from_env)
+        key = (enable_siphash24,)
+        bk.emulate_pbc_call(key, s_callable, [])
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        bk = hop.rtyper.annotator.bookkeeper
+        s_callable = bk.immutablevalue(initialize_from_env)
+        r_callable = hop.rtyper.getrepr(s_callable)
+        ll_init = r_callable.get_unique_llfn().value
+        bk.annotator.translator._call_at_startup.append(ll_init)
+
+
+ at rgc.no_collect
+def ll_hash_string_siphash24(ll_s):
+    """Called indirectly from lltypesystem/rstr.py, by redirection from
+    objectmodel.ll_string_hash().
+    """
+    from rpython.rlib.rarithmetic import intmask
+
+    # This function is entirely @rgc.no_collect.
+    length = len(ll_s.chars)
+    if lltype.typeOf(ll_s).TO.chars.OF == lltype.Char:   # regular STR
+        addr = rstr._get_raw_buf_string(rstr.STR, ll_s, 0)
+    else:
+        # NOTE: a latin-1 unicode string must have the same hash as the
+        # corresponding byte string.  If the unicode is all within
+        # 0-255, then we need to allocate a byte buffer and copy the
+        # latin-1 encoding in it manually.  Note also that we give a
+        # different hash result than CPython on ucs4 platforms, for
+        # unicode strings where CPython uses 2 bytes per character.
+        for i in range(length):
+            if ord(ll_s.chars[i]) > 0xFF:
+                addr = rstr._get_raw_buf_unicode(rstr.UNICODE, ll_s, 0)
+                length *= rffi.sizeof(rstr.UNICODE.chars.OF)
+                break
+        else:
+            p = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
+            i = 0
+            while i < length:
+                p[i] = chr(ord(ll_s.chars[i]))
+                i += 1
+            x = _siphash24(llmemory.cast_ptr_to_adr(p), length)
+            lltype.free(p, flavor='raw')
+            return intmask(x)
+    x = _siphash24(addr, length)
+    keepalive_until_here(ll_s)
+    return intmask(x)
+
 
 @contextmanager
 def choosen_seed(new_k0, new_k1, test_misaligned_path=False):
-    global k0, k1, misaligned_is_fine
-    old = k0, k1, misaligned_is_fine
-    k0 = _le64toh(r_uint64(new_k0))
-    k1 = _le64toh(r_uint64(new_k1))
+    """For tests."""
+    global misaligned_is_fine
+    old = seed.k0l, seed.k1l, misaligned_is_fine
+    seed.k0l = _le64toh(r_uint64(new_k0))
+    seed.k1l = _le64toh(r_uint64(new_k1))
     if test_misaligned_path:
         misaligned_is_fine = False
     yield
-    k0, k1, misaligned_is_fine = old
+    seed.k0l, seed.k1l, misaligned_is_fine = old
 
 def get_current_seed():
-    return _le64toh(k0), _le64toh(k1)
+    return _le64toh(seed.k0l), _le64toh(seed.k1l)
 
 
 magic0 = r_uint64(0x736f6d6570736575)
@@ -77,20 +216,21 @@
     return v0, v1, v2, v3
 
 
- at no_collect
-def siphash24(addr_in, size):
+ at rgc.no_collect
+def _siphash24(addr_in, size):
     """Takes an address pointer and a size.  Returns the hash as a r_uint64,
     which can then be casted to the expected type."""
 
-    direct = (misaligned_is_fine or
-                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
-
+    k0 = seed.k0l
+    k1 = seed.k1l
     b = r_uint64(size) << 56
     v0 = k0 ^ magic0
     v1 = k1 ^ magic1
     v2 = k0 ^ magic2
     v3 = k1 ^ magic3
 
+    direct = (misaligned_is_fine or
+                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
     index = 0
     if direct:
         while size >= 8:
@@ -113,7 +253,6 @@
                 r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48 |
                 r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 7)) << 56
             )
-            mi = _le64toh(mi)
             size -= 8
             index += 8
             v3 ^= mi
@@ -158,3 +297,13 @@
     v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
 
     return (v0 ^ v1) ^ (v2 ^ v3)
+
+
+ at jit.dont_look_inside
+def siphash24(s):
+    """'s' is a normal string.  Returns its siphash-2-4 as a r_uint64.
+    Don't forget to cast the result to a regular integer if needed,
+    e.g. with rarithmetic.intmask().
+    """
+    with rffi.scoped_nonmovingbuffer(s) as p:
+        return _siphash24(llmemory.cast_ptr_to_adr(p), len(s))
diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -57,6 +57,8 @@
                              immortal=True, zero=True)
 
     def urandom(context, n, signal_checker=None):
+        # NOTE: no dictionaries here: rsiphash24 calls this to
+        # initialize the random seed of string hashes
         provider = context[0]
         if not provider:
             # This handle is never explicitly released. The operating
@@ -139,6 +141,8 @@
 
     def urandom(context, n, signal_checker=None):
         "Read n bytes from /dev/urandom."
+        # NOTE: no dictionaries here: rsiphash24 calls this to
+        # initialize the random seed of string hashes
         result = []
         if SYS_getrandom is not None:
             n = _getrandom(n, result, signal_checker)
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -166,7 +166,6 @@
     foo = Foo()
     h = compute_hash(foo)
     assert h == object.__hash__(foo)
-    assert h == getattr(foo, '__precomputed_identity_hash')
     assert compute_hash(None) == 0
 
 def test_compute_hash_float():
@@ -182,7 +181,6 @@
     foo = Foo()
     h = compute_identity_hash(foo)
     assert h == object.__hash__(foo)
-    assert h == getattr(foo, '__precomputed_identity_hash')
 
 def test_compute_unique_id():
     from rpython.rlib.rarithmetic import intmask
@@ -410,36 +408,6 @@
         res = self.interpret(f, [])
         assert res == 1
 
-    def test_compute_hash_across_translation(self):
-        class Foo(object):
-            pass
-        q = Foo()
-
-        def f(i):
-            assert compute_hash(None) == 0
-            assert compute_hash(i) == h_42
-            assert compute_hash(i + 1.0) == h_43_dot_0
-            assert compute_hash((i + 3) / 6.0) == h_7_dot_5
-            assert compute_hash("Hello" + str(i)) == h_Hello42
-            if i == 42:
-                p = None
-            else:
-                p = Foo()
-            assert compute_hash(p) == h_None
-            assert compute_hash(("world", None, i, 7.5)) == h_tuple
-            assert compute_hash(q) == h_q
-            return i * 2
-        h_42 = compute_hash(42)
-        h_43_dot_0 = compute_hash(43.0)
-        h_7_dot_5 = compute_hash(7.5)
-        h_Hello42 = compute_hash("Hello42")
-        h_None = compute_hash(None)
-        h_tuple = compute_hash(("world", None, 42, 7.5))
-        h_q = compute_hash(q)
-
-        res = self.interpret(f, [42])
-        assert res == 84
-
     def test_fetch_translated_config(self):
         assert fetch_translated_config() is None
         def f():
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
--- a/rpython/rlib/test/test_rsiphash.py
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -1,5 +1,10 @@
-from rpython.rlib.rsiphash import siphash24, choosen_seed
+import os
+from rpython.rlib.rsiphash import siphash24, _siphash24, choosen_seed
+from rpython.rlib.rsiphash import initialize_from_env, enable_siphash24
+from rpython.rlib.objectmodel import compute_hash
+from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.lltypesystem import llmemory, rffi
+from rpython.translator.c.test.test_genc import compile
 
 
 CASES = [
@@ -28,13 +33,11 @@
 ]
 
 def check(s):
-    p = rffi.str2charp(s)
     q = rffi.str2charp('?' + s)
     with choosen_seed(0x8a9f065a358479f4, 0x11cb1e9ee7f40e1f,
                       test_misaligned_path=True):
-        x = siphash24(llmemory.cast_ptr_to_adr(p), len(s))
-        y = siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
-    rffi.free_charp(p)
+        x = siphash24(s)
+        y = _siphash24(llmemory.cast_ptr_to_adr(rffi.ptradd(q, 1)), len(s))
     rffi.free_charp(q)
     assert x == y
     return x
@@ -42,3 +45,104 @@
 def test_siphash24():
     for expected, string in CASES:
         assert check(string) == expected
+
+def test_fix_seed():
+    old_val = os.environ.get('PYTHONHASHSEED', None)
+    try:
+        os.environ['PYTHONHASHSEED'] = '0'
+        initialize_from_env()
+        assert siphash24("foo") == 15988776847138518036
+        # value checked with CPython 3.5
+
+        os.environ['PYTHONHASHSEED'] = '4000000000'
+        initialize_from_env()
+        assert siphash24("foo") == 13829150778707464258
+        # value checked with CPython 3.5
+
+        for env in ['', 'random']:
+            os.environ['PYTHONHASHSEED'] = env
+            initialize_from_env()
+            hash1 = siphash24("foo")
+            initialize_from_env()
+            hash2 = siphash24("foo")
+            assert hash1 != hash2     # extremely unlikely
+    finally:
+        if old_val is None:
+            del os.environ['PYTHONHASHSEED']
+        else:
+            os.environ['PYTHONHASHSEED'] = old_val
+
+def test_translated():
+    d1 = {"foo": 123}
+    d2 = {u"foo": 456, u"\u1234\u5678": 789}
+    class G:
+        pass
+    g = G()
+    g.v1 = d1.copy()
+    g.v2 = d2.copy()
+
+    def fetch(n):
+        if n == 0: return d1.get("foo", -1)
+        if n == 1: return g.v1.get("foo", -1)
+        if n == 2: return compute_hash("foo")
+        if n == 3: return d2.get(u"foo", -1)
+        if n == 4: return g.v2.get(u"foo", -1)
+        if n == 5: return compute_hash(u"foo")
+        if n == 6: return d2.get(u"\u1234\u5678", -1)
+        if n == 7: return g.v2.get(u"\u1234\u5678", -1)
+        if n == 8: return compute_hash(u"\u1234\u5678")
+        assert 0
+
+    def entrypoint(n):
+        enable_siphash24()
+        g.v1["bar"] = -2
+        g.v2[u"bar"] = -2
+        if n >= 0:    # get items one by one, because otherwise it may
+                      # be the case that one line influences the next
+            return str(fetch(n))
+        else:
+            # ...except in random mode, because we want all results
+            # to be computed with the same seed
+            return ' '.join([str(fetch(n)) for n in range(9)])
+
+    fn = compile(entrypoint, [int])
+
+    def getall():
+        return [int(fn(i)) for i in range(9)]
+
+    old_val = os.environ.get('PYTHONHASHSEED', None)
+    try:
+        os.environ['PYTHONHASHSEED'] = '0'
+        s1 = getall()
+        assert s1[:8] == [
+            123, 123, intmask(15988776847138518036),
+            456, 456, intmask(15988776847138518036),
+            789, 789]
+        assert s1[8] in [intmask(17593683438421985039),    # ucs2 mode
+                         intmask(94801584261658677)]       # ucs4 mode
+
+        os.environ['PYTHONHASHSEED'] = '3987654321'
+        s1 = getall()
+        assert s1[:8] == [
+            123, 123, intmask(5890804383681474441),
+            456, 456, intmask(5890804383681474441),
+            789, 789]
+        assert s1[8] in [intmask(4192582507672183374),     # ucs2 mode
+                         intmask(7179255293164649778)]     # ucs4 mode
+
+        for env in ['', 'random']:
+            os.environ['PYTHONHASHSEED'] = env
+            s1 = map(int, fn(-1).split())
+            s2 = map(int, fn(-1).split())
+            assert s1[0:2]+s1[3:5]+s1[6:8] == [123, 123, 456, 456, 789, 789]
+            assert s1[2] == s1[5]
+            assert s2[0:2]+s2[3:5]+s2[6:8] == [123, 123, 456, 456, 789, 789]
+            assert s2[2] == s2[5]
+            #
+            assert len(set([s1[2], s2[2], s1[8], s2[8]])) == 4
+
+    finally:
+        if old_val is None:
+            del os.environ['PYTHONHASHSEED']
+        else:
+            os.environ['PYTHONHASHSEED'] = old_val
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -1,8 +1,9 @@
 import py
 from rpython.annotator.model import UnionError
-from rpython.rlib import rgc
+from rpython.rlib import rgc, nonconst
 from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rtyper.test.test_llinterp import interpret
+from rpython.translator.c.test.test_genc import compile
 
 class X(object):
     pass
@@ -213,3 +214,33 @@
         assert d.get(keys[3]) is None
     f()
     interpret(f, [])
+
+def test_translation_prebuilt_1():
+    class K:
+        pass
+    d = RWeakValueDictionary(K, X)
+    k1 = K(); k2 = K()
+    x1 = X(); x2 = X()
+    d.set(k1, x1)
+    d.set(k2, x2)
+    def f():
+        assert d.get(k1) is x1
+        assert d.get(k2) is x2
+    f()
+    fc = compile(f, [], gcpolicy="boehm", rweakref=True)
+    fc()
+
+def _test_translation_prebuilt_2():
+    from rpython.rlib import rsiphash
+    d = RWeakValueDictionary(str, X)
+    k1 = "key1"; k2 = "key2"
+    x1 = X(); x2 = X()
+    d.set(k1, x1)
+    d.set(k2, x2)
+    def f():
+        rsiphash.enable_siphash24()
+        i = nonconst.NonConstant(1)
+        assert d.get("key%d" % (i,)) is x1
+        assert d.get("key%d" % (i+1,)) is x2
+    fc = compile(f, [], gcpolicy="boehm", rweakref=True)
+    fc()
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -539,6 +539,7 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
+    'call_at_startup':      LLOp(canrun=True),
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1380,20 +1380,11 @@
             return callb(*args)
         raise TypeError("%r instance is not a function" % (self._T,))
 
-    def _identityhash(self, cache=True):
+    def _identityhash(self):
         p = normalizeptr(self)
-        try:
-            return p._obj._hash_cache_
-        except AttributeError:
-            assert self._T._gckind == 'gc'
-            assert self      # not for NULL
-            result = hash(p._obj)
-            if cache:
-                try:
-                    p._obj._hash_cache_ = result
-                except AttributeError:
-                    pass
-            return result
+        assert self._T._gckind == 'gc'
+        assert self      # not for NULL
+        return hash(p._obj)
 
 class _ptr(_abstract_ptr):
     __slots__ = ('_TYPE',
@@ -1759,7 +1750,7 @@
 class _struct(_parentable):
     _kind = "structure"
 
-    __slots__ = ('_hash_cache_', '_compilation_info')
+    __slots__ = ('_compilation_info',)
 
     def __new__(self, TYPE, n=None, initialization=None, parent=None,
                 parentindex=None):
@@ -2442,24 +2433,6 @@
     return SomeInteger()
 
 
-def identityhash_nocache(p):
-    """Version of identityhash() to use from backends that don't care about
-    caching."""
-    assert p
-    return p._identityhash(cache=False)
-
-def init_identity_hash(p, value):
-    """For a prebuilt object p, initialize its hash value to 'value'."""
-    assert isinstance(typeOf(p), Ptr)
-    p = normalizeptr(p)
-    if not p:
-        raise ValueError("cannot change hash(NULL)!")
-    if hasattr(p._obj, '_hash_cache_'):
-        raise ValueError("the hash of %r was already computed" % (p,))
-    if typeOf(p).TO._is_varsize():
-        raise ValueError("init_identity_hash(): not for varsized types")
-    p._obj._hash_cache_ = intmask(value)
-
 def isCompatibleType(TYPE1, TYPE2):
     return TYPE1._is_compatible(TYPE2)
 
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -742,6 +742,9 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
+def op_call_at_startup(init_func):
+    pass    # do nothing
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -236,21 +236,14 @@
                 if self.r_rdict_hashfn.lowleveltype != lltype.Void:
                     l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash)
                     l_dict.fnkeyhash = l_fn
-
-                for dictkeycontainer, dictvalue in dictobj._dict.items():
-                    llkey = r_key.convert_const(dictkeycontainer.key)
-                    llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        dictkeycontainer.hash)
-                return l_dict
-
+                any_items = dictobj._dict.items()
             else:
-                for dictkey, dictvalue in dictobj.items():
-                    llkey = r_key.convert_const(dictkey)
-                    llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        l_dict.keyhash(llkey))
-                return l_dict
+                any_items = dictobj.items()
+            if any_items:
+                raise TyperError("found a prebuilt, explicitly non-ordered, "
+                                 "non-empty dict.  it would require additional"
+                                 " support to rehash it at program start-up")
+            return l_dict
 
     def rtype_len(self, hop):
         v_dict, = hop.inputargs(self)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -829,7 +829,7 @@
         return assert_str0(charpsize2str(cp, size))
     charp2str._annenforceargs_ = [lltype.SomePtr(TYPEP)]
 
-    # str -> char*, bool, bool
+    # str -> char*, flag
     # Can't inline this because of the raw address manipulation.
     @jit.dont_look_inside
     def get_nonmovingbuffer(data):
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib import objectmodel, jit, rgc, types
 from rpython.rlib.signature import signature
-from rpython.rlib.objectmodel import specialize, likely
+from rpython.rlib.objectmodel import specialize, likely, not_rpython
 from rpython.rtyper.debug import ll_assert
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rtyper import rmodel
@@ -46,20 +46,23 @@
 @jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
 @jit.oopspec('ordereddict.lookup(d, key, hash, flag)')
 def ll_call_lookup_function(d, key, hash, flag):
-    fun = d.lookup_function_no & FUNC_MASK
-    # This likely() here forces gcc to compile the check for fun == FUNC_BYTE
-    # first.  Otherwise, this is a regular switch and gcc (at least 4.7)
-    # compiles this as a series of checks, with the FUNC_BYTE case last.
-    # It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
-    if likely(fun == FUNC_BYTE):
-        return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
-    elif fun == FUNC_SHORT:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
-    elif IS_64BIT and fun == FUNC_INT:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
-    elif fun == FUNC_LONG:
-        return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
-    assert False
+    while True:
+        fun = d.lookup_function_no & FUNC_MASK
+        # This likely() here forces gcc to compile the check for fun==FUNC_BYTE
+        # first.  Otherwise, this is a regular switch and gcc (at least 4.7)
+        # compiles this as a series of checks, with the FUNC_BYTE case last.
+        # It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
+        if likely(fun == FUNC_BYTE):
+            return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
+        elif fun == FUNC_SHORT:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
+        elif IS_64BIT and fun == FUNC_INT:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
+        elif fun == FUNC_LONG:
+            return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
+        else:
+            ll_dict_create_initial_index(d)
+            # then, retry
 
 def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
                 ll_fasthash_function=None, ll_hash_function=None,
@@ -235,6 +238,7 @@
             self.setup()
             self.setup_final()
             l_dict = ll_newdict_size(self.DICT, len(dictobj))
+            ll_no_initial_index(l_dict)
             self.dict_cache[key] = l_dict
             r_key = self.key_repr
             if r_key.lowleveltype == llmemory.Address:
@@ -252,16 +256,14 @@
                 for dictkeycontainer, dictvalue in dictobj._dict.items():
                     llkey = r_key.convert_const(dictkeycontainer.key)
                     llvalue = r_value.convert_const(dictvalue)
-                    _ll_dict_insertclean(l_dict, llkey, llvalue,
-                                         dictkeycontainer.hash)
+                    _ll_dict_insert_no_index(l_dict, llkey, llvalue)
                 return l_dict
 
             else:
                 for dictkey, dictvalue in dictobj.items():
                     llkey = r_key.convert_const(dictkey)
                     llvalue = r_value.convert_const(dictvalue)
-                    _ll_dict_insertclean(l_dict, llkey, llvalue,
-                                         l_dict.keyhash(llkey))
+                    _ll_dict_insert_no_index(l_dict, llkey, llvalue)
                 return l_dict
 
     def rtype_len(self, hop):
@@ -336,11 +338,15 @@
         return DictIteratorRepr(self, "items").newiter(hop)
 
     def rtype_method_iterkeys_with_hash(self, hop):
-        hop.exception_cannot_occur()
+        v_dic, = hop.inputargs(self)
+        hop.exception_is_here()
+        hop.gendirectcall(ll_ensure_indexes, v_dic)
         return DictIteratorRepr(self, "keys_with_hash").newiter(hop)
 
     def rtype_method_iteritems_with_hash(self, hop):
-        hop.exception_cannot_occur()
+        v_dic, = hop.inputargs(self)
+        hop.exception_is_here()
+        hop.gendirectcall(ll_ensure_indexes, v_dic)
         return DictIteratorRepr(self, "items_with_hash").newiter(hop)
 
     def rtype_method_clear(self, hop):
@@ -458,17 +464,30 @@
 
 IS_64BIT = sys.maxint != 2 ** 31 - 1
 
-FUNC_SHIFT = 2
-FUNC_MASK  = 0x03  # two bits
 if IS_64BIT:
-    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(4)
+    FUNC_SHIFT = 3
+    FUNC_MASK  = 0x07  # three bits
+    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG, FUNC_MUST_REINDEX = range(5)
 else:
-    FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(3)
+    FUNC_SHIFT = 2
+    FUNC_MASK  = 0x03  # two bits
+    FUNC_BYTE, FUNC_SHORT, FUNC_LONG, FUNC_MUST_REINDEX = range(4)
 TYPE_BYTE  = rffi.UCHAR
 TYPE_SHORT = rffi.USHORT
 TYPE_INT   = rffi.UINT
 TYPE_LONG  = lltype.Unsigned
 
+def ll_no_initial_index(d):
+    # Used when making new empty dicts, and when translating prebuilt dicts.
+    # Remove the index completely.  A dictionary must always have an
+    # index unless it is freshly created or freshly translated.  Most
+    # dict operations start with ll_call_lookup_function(), which will
+    # recompute the hashes and create the index.
+    ll_assert(d.num_live_items == d.num_ever_used_items, 
+         "ll_no_initial_index(): dict already in use")
+    d.lookup_function_no = FUNC_MUST_REINDEX
+    d.indexes = lltype.nullptr(llmemory.GCREF.TO)
+
 def ll_malloc_indexes_and_choose_lookup(d, n):
     # keep in sync with ll_clear_indexes() below
     if n <= 256:
@@ -508,6 +527,7 @@
 
 @jit.dont_look_inside
 def ll_call_insert_clean_function(d, hash, i):
+    assert i >= 0
     fun = d.lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
         ll_dict_store_clean(d, hash, i, TYPE_BYTE)
@@ -518,6 +538,8 @@
     elif fun == FUNC_LONG:
         ll_dict_store_clean(d, hash, i, TYPE_LONG)
     else:
+        # can't be still FUNC_MUST_REINDEX here
+        ll_assert(False, "ll_call_insert_clean_function(): invalid lookup_fun")
         assert False
 
 def ll_call_delete_by_entry_index(d, hash, i):
@@ -531,6 +553,8 @@
     elif fun == FUNC_LONG:
         ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
     else:
+        # can't be still FUNC_MUST_REINDEX here
+        ll_assert(False, "ll_call_delete_by_entry_index(): invalid lookup_fun")
         assert False
 
 def ll_valid_from_flag(entries, i):
@@ -648,15 +672,14 @@
     ll_dict_reindex(d, _ll_len_of_d_indexes(d))
 _ll_dict_rescue._dont_inline_ = True
 
-def _ll_dict_insertclean(d, key, value, hash):
+ at not_rpython
+def _ll_dict_insert_no_index(d, key, value):
     # never translated
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
     entry = d.entries[d.num_ever_used_items]
     entry.key = key
     entry.value = value
-    if hasattr(ENTRY, 'f_hash'):
-        entry.f_hash = hash
+    # note that f_hash is left uninitialized in prebuilt dicts
     if hasattr(ENTRY, 'f_valid'):
         entry.f_valid = True
     d.num_ever_used_items += 1
@@ -811,12 +834,13 @@
         # also possible that there are more dead items immediately behind the
         # last one, we reclaim all the dead items at the end of the ordereditem
         # at the same point.
-        i = d.num_ever_used_items - 2
-        while i >= 0 and not d.entries.valid(i):
+        i = index
+        while True:
             i -= 1
-        j = i + 1
-        assert j >= 0
-        d.num_ever_used_items = j
+            assert i >= 0
+            if d.entries.valid(i):    # must be at least one
+                break
+        d.num_ever_used_items = i + 1
 
     # If the dictionary is at least 87.5% dead items, then consider shrinking
     # it.
@@ -844,6 +868,50 @@
     else:
         ll_dict_reindex(d, new_size)
 
+def ll_ensure_indexes(d):
+    num = d.lookup_function_no
+    if num == FUNC_MUST_REINDEX:
+        ll_dict_create_initial_index(d)
+    else:
+        ll_assert((num & FUNC_MASK) != FUNC_MUST_REINDEX,
+                  "bad combination in lookup_function_no")
+
+def ll_dict_create_initial_index(d):
+    """Create the initial index for a dictionary.  The common case is
+    that 'd' is empty.  The uncommon case is that it is a prebuilt
+    dictionary frozen by translation, in which case we must rehash all
+    entries.  The common case must be seen by the JIT.
+    """
+    if d.num_live_items == 0:
+        ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+        d.resize_counter = DICT_INITSIZE * 2
+    else:
+        ll_dict_rehash_after_translation(d)
+
+ at jit.dont_look_inside
+def ll_dict_rehash_after_translation(d):
+    assert d.num_live_items == d.num_ever_used_items
+    assert not d.indexes
+    #
+    # recompute all hashes.  Needed if they are stored in d.entries,
+    # but do it anyway: otherwise, e.g. a string-keyed dictionary
+    # won't have a fasthash on its strings if their hash is still
+    # uncomputed.
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    for i in range(d.num_ever_used_items):
+        assert d.entries.valid(i)
+        d_entry = d.entries[i]
+        h = d.keyhash(d_entry.key)
+        if hasattr(ENTRY, 'f_hash'):
+            d_entry.f_hash = h
+        #else: purely for the side-effect it can have on d_entry.key
+    #
+    # Use the smallest acceptable size for ll_dict_reindex
+    new_size = DICT_INITSIZE
+    while new_size * 2 - d.num_live_items * 3 <= 0:
+        new_size *= 2
+    ll_dict_reindex(d, new_size)
+
 def ll_dict_reindex(d, new_size):
     if bool(d.indexes) and _ll_len_of_d_indexes(d) == new_size:
         ll_clear_indexes(d, new_size)   # hack: we can reuse the same array
@@ -857,12 +925,33 @@
     entries = d.entries
     i = 0
     ibound = d.num_ever_used_items
-    while i < ibound:
-        if entries.valid(i):
-            hash = entries.hash(i)
-            ll_call_insert_clean_function(d, hash, i)
-        i += 1
-    #old_entries.delete() XXXX!
+    #
+    # Write four loops, moving the check for the value of 'fun' out of
+    # the loops.  A small speed-up over ll_call_insert_clean_function().
+    fun = d.lookup_function_no     # == lookup_function_no & FUNC_MASK
+    if fun == FUNC_BYTE:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_BYTE)
+            i += 1
+    elif fun == FUNC_SHORT:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_SHORT)
+            i += 1
+    elif IS_64BIT and fun == FUNC_INT:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_INT)
+            i += 1
+    elif fun == FUNC_LONG:
+        while i < ibound:
+            if entries.valid(i):
+                ll_dict_store_clean(d, entries.hash(i), i, TYPE_LONG)
+            i += 1
+    else:
+        assert False
+
 
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
@@ -1013,10 +1102,11 @@
 def ll_newdict(DICT):
     d = DICT.allocate()
     d.entries = _ll_empty_array(DICT)
-    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    # Don't allocate an 'indexes' for empty dict.  It seems a typical
+    # program contains tons of empty dicts, so this might be a memory win.
     d.num_live_items = 0
     d.num_ever_used_items = 0
-    d.resize_counter = DICT_INITSIZE * 2
+    ll_no_initial_index(d)
     return d
 OrderedDictRepr.ll_newdict = staticmethod(ll_newdict)
 
@@ -1101,6 +1191,10 @@
                 # as soon as we do something like ll_dict_reindex().
                 if index == (dict.lookup_function_no >> FUNC_SHIFT):
                     dict.lookup_function_no += (1 << FUNC_SHIFT)
+                # note that we can't have modified a FUNC_MUST_REINDEX
+                # dict here because such dicts have no invalid entries
+                ll_assert((dict.lookup_function_no & FUNC_MASK) !=
+                      FUNC_MUST_REINDEX, "bad combination in _ll_dictnext")
             index = nextindex
         # clear the reference to the dict and prevent restarts
         iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
@@ -1146,6 +1240,8 @@
         return dict.entries[index].value
 
 def ll_dict_copy(dict):
+    ll_ensure_indexes(dict)
+
     DICT = lltype.typeOf(dict).TO
     newdict = DICT.allocate()
     newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
@@ -1180,6 +1276,10 @@
     DICT = lltype.typeOf(d).TO
     old_entries = d.entries
     d.entries = _ll_empty_array(DICT)
+    # note: we can't remove the index here, because it is possible that
+    # crazy Python code calls d.clear() from the method __eq__() called
+    # from ll_dict_lookup(d).  Instead, stick to the rule that once a
+    # dictionary has got an index, it will always have one.
     ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
     d.num_live_items = 0
     d.num_ever_used_items = 0
@@ -1190,6 +1290,7 @@
 def ll_dict_update(dic1, dic2):
     if dic1 == dic2:
         return
+    ll_ensure_indexes(dic2)    # needed for entries.hash() below
     ll_prepare_dict_update(dic1, dic2.num_live_items)
     i = 0
     while i < dic2.num_ever_used_items:
@@ -1216,6 +1317,7 @@
     # the case where dict.update() actually has a lot of collisions.
     # If num_extra is much greater than d.num_live_items the conditional_call
     # will trigger anyway, which is really the goal.
+    ll_ensure_indexes(d)
     x = num_extra - d.num_live_items
     jit.conditional_call(d.resize_counter <= x * 3,
                          _ll_dict_resize_to, d, num_extra)
@@ -1275,6 +1377,7 @@
     if dic.num_live_items == 0:
         raise KeyError
 
+    ll_ensure_indexes(dic)
     entries = dic.entries
 
     # find the last entry.  It's unclear if the loop below is still
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -1,9 +1,9 @@
 from weakref import WeakValueDictionary
 
 from rpython.annotator import model as annmodel
-from rpython.rlib import jit, types
+from rpython.rlib import jit, types, objectmodel
 from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated,
-    ll_hash_string, keepalive_until_here, specialize, enforceargs)
+    ll_hash_string, keepalive_until_here, specialize, enforceargs, dont_inline)
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.error import TyperError
@@ -383,6 +383,8 @@
             return 0
 
     @staticmethod
+    @dont_inline
+    @jit.dont_look_inside
     def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the
@@ -400,6 +402,7 @@
 
     @staticmethod
     def ll_strfasthash(s):
+        ll_assert(s.hash != 0, "ll_strfasthash: hash==0")
         return s.hash     # assumes that the hash is already computed
 
     @staticmethod
@@ -1258,7 +1261,8 @@
                               'gethash': LLHelpers.ll_strhash,
                               'length': LLHelpers.ll_length,
                               'find': LLHelpers.ll_find,
-                              'rfind': LLHelpers.ll_rfind}))
+                              'rfind': LLHelpers.ll_rfind},
+                    hints={'remove_hash': True}))
 UNICODE.become(GcStruct('rpy_unicode', ('hash', Signed),
                         ('chars', Array(UniChar, hints={'immutable': True})),
                         adtmeths={'malloc' : staticAdtMethod(mallocunicode),
@@ -1266,8 +1270,8 @@
                                   'copy_contents' : staticAdtMethod(copy_unicode_contents),
                                   'copy_contents_from_str' : staticAdtMethod(copy_unicode_contents),
                                   'gethash': LLHelpers.ll_strhash,
-                                  'length': LLHelpers.ll_length}
-                        ))
+                                  'length': LLHelpers.ll_length},
+                    hints={'remove_hash': True}))
 
 
 # TODO: make the public interface of the rstr module cleaner
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -749,22 +749,10 @@
     assert hash3 == identityhash(s3)
     assert hash3 == identityhash(s3.super)
     assert hash3 == identityhash(s3.super.super)
-    py.test.raises(ValueError, init_identity_hash, s3, hash3^1)
-    py.test.raises(ValueError, init_identity_hash, s3.super, hash3^4)
-    py.test.raises(ValueError, init_identity_hash, s3.super.super, hash3^9)
-
-    s3 = malloc(S3)
-    init_identity_hash(s3.super, -123)
-    assert -123 == identityhash(s3)
-    assert -123 == identityhash(s3.super)
-    assert -123 == identityhash(s3.super.super)
-    py.test.raises(ValueError, init_identity_hash, s3, 4313)

From pypy.commits at gmail.com  Tue Jan 31 11:52:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 08:52:32 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: Create milestone-2-progress,
 mark my first DONE task
Message-ID: <5890c0d0.0a081c0a.f5bd4.d9b6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5768:037575bb91da
Date: 2017-01-31 17:52 +0100
http://bitbucket.org/pypy/extradoc/changeset/037575bb91da/

Log:	Create milestone-2-progress, mark my first DONE task

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -9,47 +9,7 @@
 Misc stuff not formally in any milestone
 ----------------------------------------
 
-* At some point, review lib-python/conftest.py to remove the skips
-  due to deadlocks (search for "XXX:").
-  update 1: some have been reenabled already! (richard 30.sept)
-
-* collections.py: ``OrderedDict`` should again be a thin wrapper over
-  ``dict``.  The main pain point is ``move_to_end(last=False)``.  See
-  https://mail.python.org/pipermail/python-dev/2016-August/145837.html
-
-* compare ``dir(posix)`` on py3.5 and cpython 3.5.
-
-* ``KeyError('pip.exceptions',) in weakref callback <function
-  _get_module_lock.<locals>.cb at 0x00007f118e2c0020> ignored``
-  we're getting them now on start-up, investigate
-
-* Windows: issue 2310: kill WindowsError
-
-* bytearray: 'del x[:10]' is now amortized constant-time (DONE)
-
-* check that 'import array', say, finds and loads a file array.py,
-  whereas 'import gc' does not ('gc' is a built-in module in CPython but
-  'array' is typically an extension module; at least that's the case on
-  Linux with default compilation settings).
-
-* 'import stackless' fails
-
-* "except pyopcode.Return:" in pyframe can't be there, because that's
-  outside the JIT and it gives terrible performance
-  
-* PEP 475: Retry system calls failing with EINTR (DONE)
-
-* ast compiler: clean up POP_EXCEPT: either remove them, or use it to clean up
-  the "finally: name = None; del name" nonsense at the end of any except block
-
-* bonus: use all the features of _pypyjson from the json module again
-  (eg c_encode_basestring_ascii)
-
-* socket get lots of new methods (e.g. recvmsg, ...), all test stdlib tests
-  are now skipping them
-
-* _hashlib pbkdf2_hmac has a new 'fast' implemention in cpython,
-  unsure if we are eager to implement that right now
+moved to milestone-2-progress.rst
 
 
 Milestone 1 (Aug-Sep-Oct 2016)
diff --git a/planning/py3.5/milestone-2-progress.rst b/planning/py3.5/milestone-2-progress.rst
new file mode 100644
--- /dev/null
+++ b/planning/py3.5/milestone-2-progress.rst
@@ -0,0 +1,88 @@
+What would be cool to finish before the end of Milestone 2
+==========================================================
+
+
+In-progress ("Lock" section)
+----------------------------
+
+
+Misc stuff not formally in any milestone
+----------------------------------------
+
+* At some point, review lib-python/conftest.py to remove the skips
+  due to deadlocks (search for "XXX:").
+  update 1: some have been reenabled already! (richard 30.sept)
+
+* collections.py: ``OrderedDict`` should again be a thin wrapper over
+  ``dict``.  The main pain point is ``move_to_end(last=False)``.  See
+  https://mail.python.org/pipermail/python-dev/2016-August/145837.html
+
+* compare ``dir(posix)`` on py3.5 and cpython 3.5.
+
+* ``KeyError('pip.exceptions',) in weakref callback <function
+  _get_module_lock.<locals>.cb at 0x00007f118e2c0020> ignored``
+  we're getting them now on start-up, investigate
+
+* Windows: issue 2310: kill WindowsError
+
+* bytearray: 'del x[:10]' is now amortized constant-time (DONE)
+
+* check that 'import array', say, finds and loads a file array.py,
+  whereas 'import gc' does not ('gc' is a built-in module in CPython but
+  'array' is typically an extension module; at least that's the case on
+  Linux with default compilation settings).
+
+* 'import stackless' fails
+
+* "except pyopcode.Return:" in pyframe can't be there, because that's
+  outside the JIT and it gives terrible performance
+  
+* PEP 475: Retry system calls failing with EINTR (DONE)
+
+* ast compiler: clean up POP_EXCEPT: either remove them, or use it to clean up
+  the "finally: name = None; del name" nonsense at the end of any except block
+
+* bonus: use all the features of _pypyjson from the json module again
+  (eg c_encode_basestring_ascii)
+
+* socket get lots of new methods (e.g. recvmsg, ...), all test stdlib tests
+  are now skipping them
+
+* _hashlib pbkdf2_hmac has a new 'fast' implemention in cpython,
+  unsure if we are eager to implement that right now
+
+
+Milestone 2 (end 2016, beginning 2017)
+--------------------------------------
+
+[Text from the proposal, please add progress in parentheses or square brackets]
+
+Changes to the C API.
+
+We get an up-to-date ``cpyext`` module that supports CPython 3.5 C
+extension modules, including "argument clinic" and other parts of
+the C API that are new.  The goal is that ``cpyext`` works as well
+as it does on PyPy2.
+
+Additionaly we resolve several security related issues found in CPython 3.4/3.5:
+
+* Secure and interchangeable hash algorithm (PEP 456).
+  [DONE, rpython-hash branch]
+
+* New command line option for isolated mode.
+
+* Enhancements to multiprocessing modules.
+
+* HTTP cookie parsing is now stricter (issue 22796).
+
+The measure of when this milestone is reached is based on the
+following criteria: we can take a number of C extension modules that
+work on CPython 3.5 (without reaching into the internals, like a few
+modules do), and check that they work on PyPy 3.5 as well.  More
+specifically, for any C module with a 2.7 version that works on PyPy
+2.7, its 3.5 equivalent version must also work on PyPy 3.5.
+
+
+Done not formally in the Milestone
+----------------------------------
+

From pypy.commits at gmail.com  Tue Jan 31 13:56:17 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 31 Jan 2017 10:56:17 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: Implement _PyUnicode_Ready (POSIX-only,
 for now)
Message-ID: <5890ddd1.91abdf0a.f1422.48d9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89861:2c0828fb0311
Date: 2017-01-31 18:55 +0000
http://bitbucket.org/pypy/pypy/changeset/2c0828fb0311/

Log:	Implement _PyUnicode_Ready (POSIX-only, for now)

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -9,6 +9,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 import sys, py
 from pypy.module.cpyext.unicodeobject import *
+from pypy.module.cpyext.unicodeobject import _PyUnicode_Ready
 
 class AppTestUnicodeObject(AppTestCpythonExtensionBase):
     def test_unicodeobject(self):
@@ -759,3 +760,32 @@
                 PyUnicode_Splitlines(space, w_str, 0)))
         assert r"['a\n', 'b\n', 'c\n', 'd']" == space.unwrap(space.repr(
                 PyUnicode_Splitlines(space, w_str, 1)))
+
+    def test_Ready(self, space):
+        w_str = space.wrap(u'abc')  # ASCII
+        py_str = as_pyobj(space, w_str)
+        assert get_kind(py_str) == 0
+        _PyUnicode_Ready(space, w_str)
+        assert get_kind(py_str) == 1
+        assert get_ascii(py_str) == 1
+
+        w_str = space.wrap(u'caf&#233;')  # latin1
+        py_str = as_pyobj(space, w_str)
+        assert get_kind(py_str) == 0
+        _PyUnicode_Ready(space, w_str)
+        assert get_kind(py_str) == 1
+        assert get_ascii(py_str) == 0
+
+        w_str = space.wrap(u'&#1056;&#1086;&#1089;&#1089;&#1080;&#769;&#1103;')  # UCS2
+        py_str = as_pyobj(space, w_str)
+        assert get_kind(py_str) == 0
+        _PyUnicode_Ready(space, w_str)
+        assert get_kind(py_str) == 2
+        assert get_ascii(py_str) == 0
+
+        w_str = space.wrap(u'***\U0001f4a9***')  # UCS4
+        py_str = as_pyobj(space, w_str)
+        assert get_kind(py_str) == 0
+        _PyUnicode_Ready(space, w_str)
+        assert get_kind(py_str) == 4
+        assert get_ascii(py_str) == 0
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -1,5 +1,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.runicode import unicode_encode_latin_1, unicode_encode_utf_16
+
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
@@ -8,7 +10,7 @@
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
-    make_typedescr, get_typedescr)
+    make_typedescr, get_typedescr, as_pyobj)
 from pypy.module.cpyext.bytesobject import PyBytes_Check, PyBytes_FromObject
 from pypy.module._codecs.interp_codecs import CodecState
 from pypy.objspace.std import unicodeobject
@@ -37,6 +39,12 @@
 
 PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode")
 
+MAX_UNICODE = 1114111
+WCHAR_KIND = 0
+_1BYTE_KIND = 1
+_2BYTE_KIND = 2
+_4BYTE_KIND = 4
+
 
 def new_empty_unicode(space, length):
     """
@@ -83,13 +91,53 @@
     from pypy.module.cpyext.object import _dealloc
     _dealloc(space, py_obj)
 
+def get_len(py_obj):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    return py_obj.c_length
+
+def set_len(py_obj, n):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    py_obj.c_length = n
+
+def get_state(py_obj):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    return py_obj.c_state
+
+def get_kind(py_obj):
+    return get_state(py_obj).c_kind
+
+def set_kind(py_obj, value):
+    get_state(py_obj).c_kind = cts.cast('unsigned int', value)
+
+def get_ascii(py_obj):
+    return get_state(py_obj).c_ascii
+
+def set_ascii(py_obj, value):
+    get_state(py_obj).c_ascii = cts.cast('unsigned int', value)
+
+def get_wbuffer(py_obj):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    return py_obj.c_wstr
+
+def set_wbuffer(py_obj, wbuf):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    py_obj.c_wstr = wbuf
+
+def get_utf8_len(py_obj):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    return py_obj.c_utf8_length
+
+def set_utf8_len(py_obj, n):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    py_obj.c_utf8_length = n
+
 def get_utf8(py_obj):
     py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
     return py_obj.c_utf8
 
 def set_utf8(py_obj, buf):
     py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
-    py_obj.c_utf8 = buf
+    py_obj.c_utf8 = cts.cast('char *', buf)
 
 def get_wsize(py_obj):
     py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
@@ -99,13 +147,14 @@
     py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
     py_obj.c_wstr_length = value
 
-def get_wbuffer(py_obj):
-    py_obj = cts.cast('PyASCIIObject*', py_obj)
-    return py_obj.c_wstr
+def get_data(py_obj):
+    py_obj = cts.cast('PyUnicodeObject*', py_obj)
+    return py_obj.c_data
 
-def set_wbuffer(py_obj, wbuf):
-    py_obj = cts.cast('PyASCIIObject*', py_obj)
-    py_obj.c_wstr = wbuf
+def set_data(py_obj, p_data):
+    py_obj = cts.cast('PyUnicodeObject*', py_obj)
+    py_obj.c_data = cts.cast('void *', p_data)
+
 
 @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
 def Py_UNICODE_ISSPACE(space, ch):
@@ -235,6 +284,50 @@
     # PyPy is always ready.
     return space.w_True
 
+ at cts.decl("int _PyUnicode_Ready(PyObject *unicode)", error=-1)
+def _PyUnicode_Ready(space, w_obj):
+    assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
+    py_obj = as_pyobj(space, w_obj)
+    assert get_kind(py_obj) == WCHAR_KIND
+    maxchar = 0
+    for c in w_obj._value:
+        if ord(c) > maxchar:
+            maxchar = ord(c)
+            if maxchar > MAX_UNICODE:
+                raise oefmt(space.w_ValueError,
+                    "Character U+%d is not in range [U+0000; U+10ffff]",
+                    maxchar)
+    if maxchar < 256:
+        ucs1_data = rffi.str2charp(unicode_encode_latin_1(
+            w_obj._value, len(w_obj._value), errors='strict'))
+        set_data(py_obj, ucs1_data)
+        set_kind(py_obj, _1BYTE_KIND)
+        if maxchar < 128:
+            set_ascii(py_obj, 1)
+            set_utf8(py_obj, get_data(py_obj))
+            set_utf8_len(py_obj, get_wsize(py_obj))
+        else:
+            set_ascii(py_obj, 0)
+            set_utf8(py_obj, 0)
+            set_utf8_len(py_obj, 0)
+    elif maxchar < 65536:
+        ucs2_str = unicode_encode_utf_16(
+            w_obj._value, len(w_obj._value), errors='strict')
+        ucs2_data = cts.cast('Py_UCS2 *', rffi.str2charp(ucs2_str))
+        set_data(py_obj, ucs2_data)
+        set_len(py_obj, get_wsize(py_obj))
+        set_kind(py_obj, _2BYTE_KIND)
+        set_utf8(py_obj, 0)
+        set_utf8_len(py_obj, 0)
+    else:
+        ucs4_data = get_wbuffer(py_obj)
+        set_data(py_obj, ucs4_data)
+        set_len(py_obj, get_wsize(py_obj))
+        set_kind(py_obj, _4BYTE_KIND)
+        set_utf8(py_obj, 0)
+        set_utf8_len(py_obj, 0)
+
+
 @cpython_api([rffi.VOIDP], rffi.CWCHARP, error=CANNOT_FAIL)
 def PyUnicode_AS_UNICODE(space, ref):
     """Return a pointer to the internal Py_UNICODE buffer of the object.  ref

From pypy.commits at gmail.com  Tue Jan 31 14:50:03 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 31 Jan 2017 11:50:03 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: add comments
Message-ID: <5890ea6b.6f98df0a.bc1d7.6130@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89862:84f3095253fa
Date: 2017-01-31 19:49 +0000
http://bitbucket.org/pypy/pypy/changeset/84f3095253fa/

Log:	add comments

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -311,6 +311,7 @@
             set_utf8(py_obj, 0)
             set_utf8_len(py_obj, 0)
     elif maxchar < 65536:
+        # XXX: assumes that sizeof(wchar_t) == 4
         ucs2_str = unicode_encode_utf_16(
             w_obj._value, len(w_obj._value), errors='strict')
         ucs2_data = cts.cast('Py_UCS2 *', rffi.str2charp(ucs2_str))
@@ -320,6 +321,7 @@
         set_utf8(py_obj, 0)
         set_utf8_len(py_obj, 0)
     else:
+        # XXX: assumes that sizeof(wchar_t) == 4
         ucs4_data = get_wbuffer(py_obj)
         set_data(py_obj, ucs4_data)
         set_len(py_obj, get_wsize(py_obj))
@@ -331,7 +333,15 @@
 @cpython_api([rffi.VOIDP], rffi.CWCHARP, error=CANNOT_FAIL)
 def PyUnicode_AS_UNICODE(space, ref):
     """Return a pointer to the internal Py_UNICODE buffer of the object.  ref
-    has to be a PyUnicodeObject (not checked)."""
+    has to be a PyUnicodeObject (not checked).
+
+    CPython description:
+
+    Alias for PyUnicode_AsUnicode().  This will create a wchar_t/Py_UNICODE
+    representation on demand.  Using this macro is very inefficient now,
+    try to port your code to use the new PyUnicode_*BYTE_DATA() macros or
+    use PyUnicode_WRITE() and PyUnicode_READ().
+    """
     if not get_wbuffer(ref):
         # Copy unicode buffer
         w_unicode = from_ref(space, rffi.cast(PyObject, ref))

From pypy.commits at gmail.com  Tue Jan 31 15:33:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 31 Jan 2017 12:33:26 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: Reimplement PyUnicode_AS_UNICODE and
 PyUnicode_AS_DATA as C macros
Message-ID: <5890f496.08061c0a.c3931.6558@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89863:47a51ca1c26f
Date: 2017-01-31 20:32 +0000
http://bitbucket.org/pypy/pypy/changeset/47a51ca1c26f/

Log:	Reimplement PyUnicode_AS_UNICODE and PyUnicode_AS_DATA as C macros

diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -7,6 +7,20 @@
 
 #include <cpyext_unicodeobject.h>
 
+/* Alias for PyUnicode_AsUnicode().  This will create a wchar_t/Py_UNICODE
+   representation on demand.  Using this macro is very inefficient now,
+   try to port your code to use the new PyUnicode_*BYTE_DATA() macros or
+   use PyUnicode_WRITE() and PyUnicode_READ(). */
+
+#define PyUnicode_AS_UNICODE(op) \
+    (assert(PyUnicode_Check(op)), \
+     (((PyASCIIObject *)(op))->wstr) ? (((PyASCIIObject *)(op))->wstr) : \
+      PyUnicode_AsUnicode((PyObject *)(op)))
+
+#define PyUnicode_AS_DATA(op) \
+    ((const char *)(PyUnicode_AS_UNICODE(op)))
+
+
 PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char *format, va_list vargs);
 PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char *format, ...);
 
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -226,13 +226,11 @@
 
     def test_AS(self, space):
         word = space.wrap(u'spam')
-        array = rffi.cast(rffi.CWCHARP, PyUnicode_AS_DATA(space, word))
-        array2 = PyUnicode_AS_UNICODE(space, word)
-        array3 = PyUnicode_AsUnicode(space, word)
+        array = rffi.cast(rffi.CWCHARP, PyUnicode_AsUnicode(space, word))
+        array2 = PyUnicode_AsUnicode(space, word)
         for (i, char) in enumerate(space.unwrap(word)):
             assert array[i] == char
             assert array2[i] == char
-            assert array3[i] == char
         with raises_w(space, TypeError):
             PyUnicode_AsUnicode(space, space.newbytes('spam'))
 
@@ -625,7 +623,7 @@
         count1 = space.int_w(space.len(w_x))
         target_chunk = lltype.malloc(rffi.CWCHARP.TO, count1, flavor='raw')
 
-        x_chunk = PyUnicode_AS_UNICODE(space, w_x)
+        x_chunk = PyUnicode_AsUnicode(space, w_x)
         Py_UNICODE_COPY(space, target_chunk, x_chunk, 4)
         w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, 4))
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -253,12 +253,6 @@
     """Get the maximum ordinal for a Unicode character."""
     return runicode.UNICHR(runicode.MAXUNICODE)
 
- at cpython_api([rffi.VOIDP], rffi.CCHARP, error=CANNOT_FAIL)
-def PyUnicode_AS_DATA(space, ref):
-    """Return a pointer to the internal buffer of the object. o has to be a
-    PyUnicodeObject (not checked)."""
-    return rffi.cast(rffi.CCHARP, PyUnicode_AS_UNICODE(space, ref))
-
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PyUnicode_GET_DATA_SIZE(space, w_obj):
     """Return the size of the object's internal buffer in bytes.  o has to be a
@@ -330,25 +324,6 @@
         set_utf8_len(py_obj, 0)
 
 
- at cpython_api([rffi.VOIDP], rffi.CWCHARP, error=CANNOT_FAIL)
-def PyUnicode_AS_UNICODE(space, ref):
-    """Return a pointer to the internal Py_UNICODE buffer of the object.  ref
-    has to be a PyUnicodeObject (not checked).
-
-    CPython description:
-
-    Alias for PyUnicode_AsUnicode().  This will create a wchar_t/Py_UNICODE
-    representation on demand.  Using this macro is very inefficient now,
-    try to port your code to use the new PyUnicode_*BYTE_DATA() macros or
-    use PyUnicode_WRITE() and PyUnicode_READ().
-    """
-    if not get_wbuffer(ref):
-        # Copy unicode buffer
-        w_unicode = from_ref(space, rffi.cast(PyObject, ref))
-        u = space.unicode_w(w_unicode)
-        set_wbuffer(ref, rffi.unicode2wcharp(u))
-    return get_wbuffer(ref)
-
 @cpython_api([PyObject], rffi.CWCHARP)
 def PyUnicode_AsUnicode(space, ref):
     """Return a read-only pointer to the Unicode object's internal Py_UNICODE
@@ -357,7 +332,12 @@
     w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
     if not space.issubtype_w(w_type, space.w_unicode):
         raise oefmt(space.w_TypeError, "expected unicode object")
-    return PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
+    if not get_wbuffer(ref):
+        # Copy unicode buffer
+        w_unicode = from_ref(space, rffi.cast(PyObject, ref))
+        u = space.unicode_w(w_unicode)
+        set_wbuffer(ref, rffi.unicode2wcharp(u))
+    return get_wbuffer(ref)
 
 @cts.decl("char * PyUnicode_AsUTF8(PyObject *unicode)")
 def PyUnicode_AsUTF8(space, ref):
@@ -402,8 +382,7 @@
     string may or may not be 0-terminated.  It is the responsibility of the caller
     to make sure that the wchar_t string is 0-terminated in case this is
     required by the application."""
-    ref = rffi.cast(PyUnicodeObject, ref)
-    c_buffer = PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
+    c_buffer = PyUnicode_AsUnicode(space, ref)
     c_length = get_wsize(ref)
 
     # If possible, try to copy the 0-termination as well