From noreply at buildbot.pypy.org  Wed Jul  1 10:13:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 10:13:57 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof-review: hg merge default
Message-ID: <20150701081357.070241C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: vmprof-review
Changeset: r78368:81c9b346a396
Date: 2015-06-30 17:23 +0200
http://bitbucket.org/pypy/pypy/changeset/81c9b346a396/

Log:	hg merge default

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -465,9 +465,9 @@
             mk.rule('%.o %.gcmap', '%.vmprof.s', [
                 '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py '
                     '-t $*.vmprof.s > $*.gctmp',
-                '$(CC) -o $*.o -c $*.asmgcc.lbl.s',
+                '$(CC) -o $*.o -c $*.vmprof.lbl.s',
                 'mv $*.gctmp $*.gcmap',
-                'rm $*.asmgcc.lbl.s'])
+                'rm $*.vmprof.lbl.s'])
             
             # the rule to compute gcmaptable.s
             mk.rule('gcmaptable.s', '$(GCMAPFILES)',

From noreply at buildbot.pypy.org  Wed Jul  1 10:13:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 10:13:58 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof-review: starting rvmprof
Message-ID: <20150701081358.391F21C146D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: vmprof-review
Changeset: r78369:af777b42cf86
Date: 2015-06-30 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/af777b42cf86/

Log:	starting rvmprof

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -0,0 +1,92 @@
+import sys
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import rgc
+
+MAX_CODES = 8000
+
+# ____________________________________________________________
+
+
+class VMProf(object):
+
+    def __init__(self):
+        self._code_classes = set()
+        self._gather_all_code_objs = lambda: None
+        self._cleanup_()
+
+    def _cleanup_(self):
+        self._current_codes = StringBuilder()
+        if sys.maxint == 2147483647:
+            self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
+        else:
+            self._code_unique_id = 0x7000000000000000
+
+    @specialize.argtype(1)
+    def register_code(self, code, name):
+        """Register the code object.  Call when a new code object is made.
+        """
+        uid = self._code_unique_id
+        self._code_unique_id = uid + 4    # so we have two bits to mark stuff
+        code._vmprof_unique_id = uid
+        #
+        b = self._current_codes
+        b.append('\x02')
+        write_long_to_string_builder(uid, b)
+        write_long_to_string_builder(len(name), b)
+        b.append(name)
+        if b.getlength() >= MAX_CODES:
+            self._flush_codes()
+
+    def register_code_object_class(self, CodeClass, full_name_func):
+        """NOT_RPYTHON
+        Register statically the class 'CodeClass' as containing user
+        code objects.
+
+        full_name_func() is a function called at runtime with an
+        instance of CodeClass and it should return a string.  This
+        is the string stored in the vmprof file identifying the code
+        object.  It can be directly an unbound method of CodeClass.
+
+        Instances of the CodeClass will have a new attribute called
+        '_vmprof_unique_id', but that's managed internally.
+        """
+        if CodeClass in self._code_classes:
+            return
+        CodeClass._vmprof_unique_id = 0     # default value: "unknown"
+        self._code_classes.add(CodeClass)
+        #
+        def try_cast_to_pycode(gcref):
+            return rgc.try_cast_gcref_to_instance(CodeClass, gcref)
+        #
+        def gather_all_code_objs():
+            all_code_objs = rgc.do_get_objects(try_cast_to_pycode)
+            for code in all_code_objs:
+                self.register_code(code, full_name_func(code))
+            prev()
+        # make a chained list of the gather() functions for all
+        # the types of code objects
+        prev = self._gather_all_code_objs
+        self._gather_all_code_objs = gather_all_code_objs
+
+
+def write_long_to_string_builder(l, b):
+    if sys.maxint == 2147483647:
+        b.append(chr(l & 0xff))
+        b.append(chr((l >> 8) & 0xff))
+        b.append(chr((l >> 16) & 0xff))
+        b.append(chr((l >> 24) & 0xff))
+    else:
+        b.append(chr(l & 0xff))
+        b.append(chr((l >> 8) & 0xff))
+        b.append(chr((l >> 16) & 0xff))
+        b.append(chr((l >> 24) & 0xff))
+        b.append(chr((l >> 32) & 0xff))
+        b.append(chr((l >> 40) & 0xff))
+        b.append(chr((l >> 48) & 0xff))
+        b.append(chr((l >> 56) & 0xff))
+
+
+ at specialize.memo()
+def get_vmprof():
+    return VMProf()

From noreply at buildbot.pypy.org  Wed Jul  1 10:13:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 10:13:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #867: backport CPython's
	92656b5df2f2
Message-ID: <20150701081359.5C8C31C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78370:0dd6dbaecd11
Date: 2015-07-01 10:14 +0200
http://bitbucket.org/pypy/pypy/changeset/0dd6dbaecd11/

Log:	Issue #867: backport CPython's 92656b5df2f2

diff --git a/lib-python/2.7/test/test_urllib2.py b/lib-python/2.7/test/test_urllib2.py
--- a/lib-python/2.7/test/test_urllib2.py
+++ b/lib-python/2.7/test/test_urllib2.py
@@ -291,6 +291,7 @@
         self.req_headers = []
         self.data = None
         self.raise_on_endheaders = False
+        self.sock = None
         self._tunnel_headers = {}
 
     def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
diff --git a/lib-python/2.7/urllib2.py b/lib-python/2.7/urllib2.py
--- a/lib-python/2.7/urllib2.py
+++ b/lib-python/2.7/urllib2.py
@@ -1200,6 +1200,12 @@
                 r = h.getresponse(buffering=True)
             except TypeError: # buffering kw not supported
                 r = h.getresponse()
+            # If the server does not send us a 'Connection: close' header,
+            # HTTPConnection assumes the socket should be left open. Manually
+            # mark the socket to be closed when this response object goes away.
+            if h.sock:
+                h.sock.close()
+                h.sock = None
 
         # Pick apart the HTTPResponse object to get the addinfourl
         # object initialized properly.

From noreply at buildbot.pypy.org  Wed Jul  1 11:13:42 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  1 Jul 2015 11:13:42 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix handling of inputargs
Message-ID: <20150701091342.C0E581C0822@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78371:854053774fc4
Date: 2015-07-01 10:21 +0200
http://bitbucket.org/pypy/pypy/changeset/854053774fc4/

Log:	fix handling of inputargs

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -300,8 +300,11 @@
         self.short_boxes = exported_state.short_boxes
         self.initial_virtual_state = target_token.virtual_state
 
-        for i, arg in enumerate(exported_state.orig_inputargs):
-            arg.set_forwarded(self.inputargs[i])
+        inpargs = self.initial_virtual_state.make_inputargs(
+            exported_state.orig_inputargs, self.optimizer)
+        for i, arg in enumerate(inpargs):
+            if arg is not self.inputargs[i]:
+                arg.set_forwarded(self.inputargs[i])
         for box in self.inputargs:
             preamble_info = exported_state.exported_values[box]
             self.optimizer.setinfo_from_preamble(box, preamble_info)

From noreply at buildbot.pypy.org  Wed Jul  1 11:13:44 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  1 Jul 2015 11:13:44 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start writing more direct
	unrolling tests
Message-ID: <20150701091344.01D561C0822@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78372:d655c193c36a
Date: 2015-07-01 11:13 +0200
http://bitbucket.org/pypy/pypy/changeset/d655c193c36a/

Log:	start writing more direct unrolling tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -0,0 +1,53 @@
+
+""" More direct tests for unrolling
+"""
+
+from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest,\
+     LLtypeMixin, FakeMetaInterpStaticData
+from rpython.jit.metainterp.history import (TreeLoop, AbstractDescr,
+                                            JitCellToken, TargetToken)
+from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.optimizeopt.virtualstate import \
+     NotVirtualStateInfo, LEVEL_CONSTANT
+
+class FakeOptimizer(object):
+    optearlyforce = None
+     
+class TestUnroll(BaseTest, LLtypeMixin):
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
+    
+    def optimize(self, ops):
+        loop = self.parse(ops, postprocess=self.postprocess)
+        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+        self.add_guard_future_condition(loop)
+        operations =  loop.operations
+        jumpop = operations[-1]
+        assert jumpop.getopnum() == rop.JUMP
+        inputargs = loop.inputargs
+
+        jump_args = jumpop.getarglist()[:]
+        operations = operations[:-1]
+
+        preamble = TreeLoop('preamble')
+        preamble.inputargs = inputargs
+
+        token = JitCellToken()
+        start_label = ResOperation(rop.LABEL, inputargs, descr=TargetToken(token))
+        stop_label = ResOperation(rop.LABEL, jump_args, descr=token)
+        preamble.operations = [start_label] + operations + [stop_label]
+        start_state = self._do_optimize_loop(preamble, None,
+                                             export_state=True)
+        vs = preamble.operations[-1].getdescr().virtual_state
+        return start_state, vs
+
+    def test_simple(self):
+        loop = """
+        [i0]
+        i1 = int_add(i0, 1)
+        guard_value(i1, 1) []
+        jump(i1)
+        """
+        es, vs = self.optimize(loop)
+        assert isinstance(vs.state[0], NotVirtualStateInfo)
+        assert vs.make_inputargs([1], FakeOptimizer()) == []
+        assert vs.state[0].level == LEVEL_CONSTANT

From noreply at buildbot.pypy.org  Wed Jul  1 11:24:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 11:24:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #1708: two tests that fail
 (probably showing two different bugs)
Message-ID: <20150701092436.BAE971C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78373:66c5ff3cad3c
Date: 2015-07-01 10:58 +0200
http://bitbucket.org/pypy/pypy/changeset/66c5ff3cad3c/

Log:	Issue #1708: two tests that fail (probably showing two different
	bugs)

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1217,6 +1217,43 @@
         finally:
             sys.path_hooks.pop()
 
+    def test_meta_path_import_star_1(self):
+        class ImportHook(object):
+            def find_module(self, fullname, path=None):
+                assert not fullname.endswith('*')
+                if fullname == 'meta_path_pseudo_module':
+                    return self
+            def load_module(self, fullname):
+                assert fullname == 'meta_path_pseudo_module'
+                return new.module('meta_path_pseudo_module')
+
+        import sys, new
+        sys.meta_path.append(ImportHook())
+        try:
+            exec "from meta_path_pseudo_module import *" in {}
+        finally:
+            sys.meta_path.pop()
+
+    def test_meta_path_import_star_2(self):
+        class ImportHook(object):
+            def find_module(self, fullname, path=None):
+                if fullname.startswith('meta_path_2_pseudo_module'):
+                    return self
+            def load_module(self, fullname):
+                assert fullname == 'meta_path_2_pseudo_module'
+                m = new.module('meta_path_2_pseudo_module')
+                m.__path__ = ['/some/random/dir']
+                sys.modules['meta_path_2_pseudo_module'] = m
+                return m
+
+        import sys, new
+        sys.meta_path.append(ImportHook())
+        try:
+            exec "from meta_path_2_pseudo_module import *" in {}
+        finally:
+            sys.meta_path.pop()
+
+
 class AppTestPyPyExtension(object):
     spaceconfig = dict(usemodules=['imp', 'zipimport', '__pypy__'])
 

From noreply at buildbot.pypy.org  Wed Jul  1 11:24:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 11:24:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for test_meta_path_import_star_2
Message-ID: <20150701092437.EE0F61C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78374:e83ef97376fe
Date: 2015-07-01 11:08 +0200
http://bitbucket.org/pypy/pypy/changeset/e83ef97376fe/

Log:	Fix for test_meta_path_import_star_2

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -349,9 +349,11 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    return None
+            else:
+                # this only runs if fromlist_w != ['*']
+                for w_name in fromlist_w:
+                    if try_getattr(space, w_mod, w_name) is None:
+                        return None
         return w_mod
     return first
 
@@ -389,10 +391,12 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    load_part(space, w_path, prefix, space.str0_w(w_name),
-                              w_mod, tentative=1)
+            else:
+                # this only runs if fromlist_w != ['*']
+                for w_name in fromlist_w:
+                    if try_getattr(space, w_mod, w_name) is None:
+                        load_part(space, w_path, prefix, space.str0_w(w_name),
+                                  w_mod, tentative=1)
         return w_mod
     else:
         return first

From noreply at buildbot.pypy.org  Wed Jul  1 11:24:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 11:24:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: Skip the other test with a comment.
Message-ID: <20150701092439.113481C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78375:811532eaa04d
Date: 2015-07-01 11:24 +0200
http://bitbucket.org/pypy/pypy/changeset/811532eaa04d/

Log:	Skip the other test with a comment.

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1217,7 +1217,16 @@
         finally:
             sys.path_hooks.pop()
 
-    def test_meta_path_import_star_1(self):
+    def test_meta_path_import_error_1(self):
+        # as far as I can tell, the problem is that in CPython, if you
+        # use an import hook that doesn't update sys.modules, then the
+        # import succeeds; but at the same time, you can have the same
+        # result without an import hook (see test_del_from_sys_modules)
+        # and then the import fails.  This looks like even more mess
+        # to replicate, so we ignore it until someone really hits this
+        # case...
+        skip("looks like an inconsistency in CPython")
+
         class ImportHook(object):
             def find_module(self, fullname, path=None):
                 assert not fullname.endswith('*')
@@ -1225,12 +1234,13 @@
                     return self
             def load_module(self, fullname):
                 assert fullname == 'meta_path_pseudo_module'
+                # we "forget" to update sys.modules
                 return new.module('meta_path_pseudo_module')
 
         import sys, new
         sys.meta_path.append(ImportHook())
         try:
-            exec "from meta_path_pseudo_module import *" in {}
+            import meta_path_pseudo_module
         finally:
             sys.meta_path.pop()
 

From noreply at buildbot.pypy.org  Wed Jul  1 12:35:04 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed,  1 Jul 2015 12:35:04 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: correctly emit reduction operation in a
 guard exit that compiles a bridge (was missing before)
Message-ID: <20150701103504.E37631C1C8E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78376:ac77811327eb
Date: 2015-07-01 12:35 +0200
http://bitbucket.org/pypy/pypy/changeset/ac77811327eb/

Log:	correctly emit reduction operation in a guard exit that compiles a
	bridge (was missing before) added prod(...) as accumulator

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -37,19 +37,10 @@
 class BadToken(Exception):
     pass
 
-class FakeArguments(W_Root):
-    def __init__(self, args_w, kw_w):
-        self.args_w = args_w
-        self.kw_w = kw_w
-
-    def unpack(self):
-        return self.args_w, self.kw_w
-
-
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring", "count_nonzero",
                         "argsort", "cumsum", "logical_xor_reduce"]
-TWO_ARG_FUNCTIONS = ["dot", 'multiply', 'take', 'searchsorted']
+TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted', 'multiply']
 TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype', 'reshape']
 THREE_ARG_FUNCTIONS = ['where']
 
@@ -787,7 +778,7 @@
                 raise ArgumentNotAnArray
             if self.name == "dot":
                 w_res = arr.descr_dot(interp.space, arg)
-            if self.name == "multiply":
+            elif self.name == 'multiply':
                 w_res = arr.descr_mul(interp.space, arg)
             elif self.name == 'take':
                 w_res = arr.descr_take(interp.space, arg)
@@ -808,7 +799,7 @@
             if self.name == "where":
                 w_res = where(interp.space, arr, arg1, arg2)
             else:
-                assert False
+                assert False # unreachable code
         elif self.name in TWO_ARG_FUNCTIONS_OR_NONE:
             if len(self.args) != 2:
                 raise ArgumentMismatch
@@ -822,7 +813,7 @@
                 assert isinstance(w_arg, ArrayConstant)
                 w_res = arr.reshape(interp.space, w_arg.wrap(interp.space))
             else:
-                assert False, "missing two arg impl for: %s" % (self.name,)
+                assert False
         else:
             raise WrongFunctionName
         if isinstance(w_res, W_NDimArray):
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -421,8 +421,8 @@
                 lval = left_impl.getitem(i1).convert_to(space, dtype)
                 rval = right_impl.getitem(i2).convert_to(space, dtype)
                 oval = dtype.itemtype.add(oval, dtype.itemtype.mul(lval, rval))
-                i1 += s1
-                i2 += s2
+                i1 += jit.promote(s1)
+                i2 += jit.promote(s2)
             outi.setitem(outs, oval)
             outs = outi.next(outs)
             rights = righti.next(rights)
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -118,15 +118,18 @@
         retval = self.interp.eval_graph(self.graph, [i])
         return retval
 
-    def define_matrix_dot():
+    def define_dot_matrix():
         return """
         mat = |16|
         m = reshape(mat, [4,4])
+        vec = [0,1,2,3]
+        a = dot(m, vec)
+        a -> 3
         """
 
-    def test_matrix_dot(self):
-        result = self.run("matrix_dot")
-        assert int(result) == 45
+    def test_dot_matrix(self):
+        result = self.run("dot_matrix")
+        assert int(result) == 86
         self.check_vectorized(1, 1)
 
     def define_float32_copy():
@@ -523,6 +526,7 @@
             expected *= i * 2
         assert result == expected
         self.check_trace_count(1)
+        self.check_vectorized(1, 1)
 
     def define_max():
         return """
@@ -534,7 +538,7 @@
     def test_max(self):
         result = self.run("max")
         assert result == 128
-        self.check_vectorized(1, 0) # TODO reduce
+        self.check_vectorized(1, 0)
 
     def define_min():
         return """
@@ -546,7 +550,7 @@
     def test_min(self):
         result = self.run("min")
         assert result == -128
-        self.check_vectorized(1, 0) # TODO reduce
+        self.check_vectorized(1, 0)
 
     def define_any():
         return """
@@ -820,8 +824,8 @@
     def test_dot(self):
         result = self.run("dot")
         assert result == 184
-        self.check_trace_count(3)
-        self.check_vectorized(3,0)
+        self.check_trace_count(5)
+        self.check_vectorized(3,1)
 
     def define_argsort():
         return """
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -880,9 +880,6 @@
             if isinstance(box, BoxVectorAccum):
                 if box.operator == '+':
                     value = sum(value)
-                elif box.operator == '-':
-                    def sub(acc, x): return acc - x
-                    value = reduce(sub, value, 0)
                 elif box.operator == '*':
                     def prod(acc, x): return acc * x
                     value = reduce(prod, value, 1)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -559,6 +559,8 @@
                                              self.current_clt.allgcrefs,
                                              self.current_clt.frame_info)
         self._check_frame_depth(self.mc, regalloc.get_gcmap())
+        #import pdb; pdb.set_trace()
+        self._accum_update_at_exit(arglocs, inputargs, faildescr, regalloc)
         frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, operations)
         codeendpos = self.mc.get_relative_pos()
         self.write_pending_failure_recoveries(regalloc)
@@ -1865,7 +1867,7 @@
         startpos = self.mc.get_relative_pos()
         #
         self._accum_update_at_exit(guardtok.fail_locs, guardtok.failargs,
-                                   regalloc)
+                                   guardtok.faildescr, regalloc)
         #
         fail_descr, target = self.store_info_on_descr(startpos, guardtok)
         self.mc.PUSH(imm(fail_descr))
@@ -2529,67 +2531,60 @@
     # vector operations
     # ________________________________________
 
-    def _accum_update_at_exit(self, fail_locs, fail_args, regalloc):
+    def _accum_update_at_exit(self, fail_locs, fail_args, faildescr, regalloc):
         """ If accumulation is done in this loop, at the guard exit
         some vector registers must be adjusted to yield the correct value"""
         assert regalloc is not None
-        for i,arg in enumerate(fail_args):
-            if arg is None:
-                continue
+        accum_info = faildescr.rd_accum_list
+        while accum_info:
+            pos = accum_info.position
+            loc = fail_locs[pos]
+            assert isinstance(loc, RegLoc)
+            arg = fail_args[pos]
             if isinstance(arg, BoxVectorAccum):
-                assert arg.scalar_var is not None
-                loc = fail_locs[i]
-                assert isinstance(loc, RegLoc)
-                assert loc.is_xmm
-                tgtloc = regalloc.force_allocate_reg(arg.scalar_var, fail_args)
-                assert tgtloc is not None
-                if arg.operator == '+':
-                    # reduction using plus
-                    self._accum_reduce_sum(arg, loc, tgtloc)
-                    fail_locs[i] = tgtloc
-                    regalloc.possibly_free_var(arg)
-                    fail_args[i] = arg.scalar_var
-                else:
-                    raise NotImplementedError("accum operator %s not implemented" %
-                                                (arg.operator)) 
+                arg = arg.scalar_var
+            assert arg is not None
+            tgtloc = regalloc.force_allocate_reg(arg, fail_args)
+            if accum_info.operation == '+':
+                # reduction using plus
+                self._accum_reduce_sum(arg, loc, tgtloc)
+            elif accum_info.operation == '*':
+                self._accum_reduce_mul(arg, loc, tgtloc)
+            else:
+                import pdb; pdb.set_trace()
+                not_implemented("accum operator %s not implemented" %
+                                            (accum_info.operation)) 
+            fail_locs[pos] = tgtloc
+            regalloc.possibly_free_var(arg)
+            accum_info = accum_info.prev
 
-    def _accum_reduce_sum(self, vector_var, accumloc, targetloc):
-        assert isinstance(vector_var, BoxVectorAccum)
-        #
-        type = vector_var.gettype()
-        size = vector_var.getsize()
-        if type == FLOAT:
-            if size == 8:
-                # r = (r[0]+r[1],r[0]+r[1])
-                self.mc.HADDPD(accumloc, accumloc)
-                # upper bits (> 64) are dirty (but does not matter)
-                if accumloc is not targetloc:
-                    self.mov(targetloc, accumloc)
-                return
-            if size == 4:
-                # r = (r[0]+r[1],r[2]+r[3],r[0]+r[1],r[2]+r[3])
-                self.mc.HADDPS(accumloc, accumloc)
-                self.mc.HADDPS(accumloc, accumloc)
-                # invoking it a second time will gather the whole sum
-                # at the first element position
-                # the upper bits (>32) are dirty (but does not matter)
-                if accumloc is not targetloc:
-                    self.mov(targetloc, accumloc)
-                return
-        elif type == INT:
+    def _accum_reduce_mul(self, arg, accumloc, targetloc):
+        scratchloc = X86_64_SCRATCH_REG
+        self.mc.mov(scratchloc, accumloc)
+        # swap the two elements
+        self.mc.SHUFPS_xxi(scratchloc.value, scratchloc.value, 0x01)
+        self.mc.MULPD(accumloc, scratchloc)
+        if accumloc is not targetloc:
+            self.mc.mov(targetloc, accumloc)
+
+    def _accum_reduce_sum(self, arg, accumloc, targetloc):
+        # Currently the accumulator can ONLY be the biggest
+        # size for X86 -> 64 bit float/int
+        if arg.type == FLOAT:
+            # r = (r[0]+r[1],r[0]+r[1])
+            self.mc.HADDPD(accumloc, accumloc)
+            # upper bits (> 64) are dirty (but does not matter)
+            if accumloc is not targetloc:
+                self.mov(targetloc, accumloc)
+            return
+        elif arg.type == INT:
             scratchloc = X86_64_SCRATCH_REG
-            if size == 8:
-                self.mc.PEXTRQ_rxi(targetloc.value, accumloc.value, 0)
-                self.mc.PEXTRQ_rxi(scratchloc.value, accumloc.value, 1)
-                self.mc.ADD(targetloc, scratchloc)
-                return
-            if size == 4:
-                self.mc.PHADDD(accumloc, accumloc)
-                self.mc.PHADDD(accumloc, accumloc)
-                self.mc.PEXTRD_rxi(targetloc.value, accumloc.value, 0)
-                return
+            self.mc.PEXTRQ_rxi(targetloc.value, accumloc.value, 0)
+            self.mc.PEXTRQ_rxi(scratchloc.value, accumloc.value, 1)
+            self.mc.ADD(targetloc, scratchloc)
+            return
 
-        raise NotImplementedError("reduce sum for %s not impl." % vector_var)
+        not_implemented("reduce sum for %s not impl." % arg)
 
     def genop_vec_getarrayitem_raw(self, op, arglocs, resloc):
         # considers item scale (raw_load does not)
@@ -2655,7 +2650,7 @@
             # There is no 64x64 bit packed mul and I did not find one
             # for 8 bit either. It is questionable if it gives any benefit
             # for 8 bit.
-            raise NotImplementedError("")
+            not_implemented("int8/64 mul")
 
     def genop_vec_int_add(self, op, arglocs, resloc):
         loc0, loc1, size_loc = arglocs
@@ -2757,7 +2752,7 @@
             # the speedup might only be modest...
             # the optimization does not emit such code!
             msg = "vec int signext (%d->%d)" % (size, tosize)
-            raise NotImplementedError(msg)
+            not_implemented(msg)
 
     def genop_vec_float_expand(self, op, arglocs, resloc):
         srcloc, sizeloc = arglocs
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -488,7 +488,8 @@
 
 class ResumeGuardDescr(ResumeDescr):
     _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
-               'rd_frame_info_list', 'rd_pendingfields', 'status')
+               'rd_frame_info_list', 'rd_pendingfields', 'rd_accum_list',
+               'status')
     
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
@@ -496,6 +497,7 @@
     rd_virtuals = None
     rd_frame_info_list = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
+    rd_accum_list = None
 
     status = r_uint(0)
 
@@ -507,6 +509,7 @@
         self.rd_pendingfields = other.rd_pendingfields
         self.rd_virtuals = other.rd_virtuals
         self.rd_numb = other.rd_numb
+        self.rd_accum_list = other.rd_accum_list
         # we don't copy status
 
     ST_BUSY_FLAG    = 0x01     # if set, busy tracing from the guard
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -416,6 +416,7 @@
         if vbox.gettype() == INT:
             return self.extend_int(vbox, newtype)
         else:
+            import pdb; pdb.set_trace()
             raise NotImplementedError("cannot yet extend float")
 
     def extend_int(self, vbox, newtype):
@@ -856,8 +857,9 @@
 
 class Accum(object):
     PLUS = '+'
+    MULTIPLY = '*'
 
-    def __init__(self, var=None, pos=-1, operator=PLUS):
+    def __init__(self, var, pos, operator):
         self.var = var
         self.pos = pos
         self.operator = operator
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1369,6 +1369,22 @@
         i32 = int_ge(i30, i25)
         guard_false(i32, descr=<Guard0x7f9f03ab17d0>) [p0, i29, i30, i31, p19, None, None, None]
         jump(p0, p19, i30, i31, i29, i8, i25, descr=TargetToken(140320937897104))
+
+        """
+        trace ="""
+        [i0, i1, i16, i17, i18, i5, p6, p7, p8, f19, p10, p11, p12, p13, p14, p15, i20, i21]
+        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f249eb2e510>) [i5, i18, i17, i16, i1, i0, p15, p14, p13, p12, p11, p10, p8, p7, p6, f19]
+        f22 = raw_load(i20, i18, descr=floatarraydescr)
+        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7f249eb2ec90>) [i5, i18, i17, i16, i1, i0, p15, p14, p13, p12, p11, p10, p8, p7, p6, f22, f19]
+        f23 = raw_load(i21, i17, descr=floatarraydescr)
+        f24 = float_mul(f22, f23)
+        f25 = float_add(f19, f24)
+        i26 = int_add(i18, 8)
+        i27 = int_add(i17, 8)
+        i28 = int_lt(i16, i5)
+        guard_true(i28, descr=<rpython.jit.metainterp.compile.ResumeGuardTrueDescr object at 0x7f249eb99290>) [i5, i26, i27, i16, i1, i0, p15, p14, p13, p12, p11, p10, p8, p7, p6, f25, None]
+        i31 = int_add(i16, 1)
+        jump(i0, i1, i31, i27, i26, i5, p6, p7, p8, f25, p10, p11, p12, p13, p14, p15, i20, i21)
         """
         # schedule 885 -> ptype is non for raw_load?
         opt = self.vectorize(self.parse_loop(trace))
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -8,7 +8,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp import resoperation
 from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.metainterp.resume import Snapshot
+from rpython.jit.metainterp.resume import Snapshot, AccumInfo
 
 # ____________________________________________________________
 # Misc. utilities
@@ -213,6 +213,8 @@
         return True
 
     def rename_failargs(self, guard, clone=False):
+        from rpython.jit.metainterp.history import BoxVectorAccum
+        from rpython.jit.metainterp.compile import ResumeGuardDescr
         if guard.getfailargs() is not None:
             if clone:
                 args = guard.getfailargs()[:]
@@ -220,6 +222,11 @@
                 args = guard.getfailargs()
             for i,arg in enumerate(args):
                 value = self.rename_map.get(arg,arg)
+                if value is not arg and isinstance(value, BoxVectorAccum):
+                    descr = guard.getdescr()
+                    assert isinstance(descr,ResumeGuardDescr)
+                    ai = AccumInfo(descr.rd_accum_list, i, value.operator)
+                    descr.rd_accum_list = ai
                 args[i] = value
             return args
         return None
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -767,7 +767,7 @@
         lop = lnode.getoperation()
         opnum = lop.getopnum()
 
-        if opnum in (rop.FLOAT_ADD, rop.INT_ADD):
+        if opnum in (rop.FLOAT_ADD, rop.INT_ADD, rop.FLOAT_MUL):
             roper = rnode.getoperation()
             assert lop.numargs() == 2 and lop.result is not None
             accum_var, accum_pos = self.getaccumulator_variable(lop, roper, origin_pack)
@@ -802,7 +802,10 @@
                 # of leading/preceding signext/floatcast instructions needs to be
                 # considered. => tree pattern matching problem.
                 return None
-            accum = Accum(accum_var, accum_pos, Accum.PLUS)
+            operator = Accum.PLUS
+            if opnum == rop.FLOAT_ADD:
+                operator = Accum.MULTIPLY
+            accum = Accum(accum_var, accum_pos, operator)
             return AccumPair(lnode, rnode, ptype, ptype, accum)
 
         return None
@@ -824,14 +827,22 @@
             # create a new vector box for the parameters
             box = pack.input_type.new_vector_box()
             size = vec_reg_size // pack.input_type.getsize()
-            op = ResOperation(rop.VEC_BOX, [ConstInt(size)], box)
-            sched_data.invariant_oplist.append(op)
-            result = box.clonebox()
-            # clear the box to zero TODO might not be zero for every reduction?
-            op = ResOperation(rop.VEC_INT_XOR, [box, box], result)
-            sched_data.invariant_oplist.append(op)
-            box = result
-            result = BoxVectorAccum(box, accum.var, '+')
+            # reset the box to zeros or ones
+            if accum.operator == Accum.PLUS:
+                op = ResOperation(rop.VEC_BOX, [ConstInt(size)], box)
+                sched_data.invariant_oplist.append(op)
+                result = box.clonebox()
+                op = ResOperation(rop.VEC_INT_XOR, [box, box], result)
+                sched_data.invariant_oplist.append(op)
+                box = result
+            elif accum.operator == Accum.MULTIPLY:
+                # multiply is only supported by floats
+                op = ResOperation(rop.VEC_FLOAT_EXPAND, [ConstInt(1)], box)
+                sched_data.invariant_oplist.append(op)
+            else:
+                import pdb; pdb.set_trace()
+                raise NotImplementedError
+            result = BoxVectorAccum(box, accum.var, accum.operator)
             # pack the scalar value
             op = ResOperation(getpackopnum(box.gettype()),
                               [box, accum.var, ConstInt(0), ConstInt(1)], result)
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -34,6 +34,13 @@
         self.jitcode = jitcode
         self.pc = pc
 
+class AccumInfo(object):
+    __slots__ = ('prev', 'position', 'operation')
+    def __init__(self, prev, position, operation):
+        self.prev = prev
+        self.operation = operation
+        self.position = position
+
 def _ensure_parent_resumedata(framestack, n):
     target = framestack[n]
     if n == 0:

From noreply at buildbot.pypy.org  Wed Jul  1 13:08:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 13:08:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Untested: try to fix an issue where the
 ctypes callback is invoked
Message-ID: <20150701110821.ECBFE1C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78377:03f531cb70fe
Date: 2015-07-01 13:08 +0200
http://bitbucket.org/pypy/pypy/changeset/03f531cb70fe/

Log:	Untested: try to fix an issue where the ctypes callback is invoked
	in some unexpected thread

diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -27,8 +27,10 @@
     callback_ptr = global_counter.get(userdata.addarg)
     w_callable = callback_ptr.w_callable
     argtypes = callback_ptr.argtypes
+    must_leave = False
     space = callback_ptr.space
     try:
+        must_leave = space.threadlocals.try_enter_thread(space)
         args_w = [None] * len(argtypes)
         for i in range(len(argtypes)):
             argtype = argtypes[i]
@@ -50,6 +52,8 @@
             resshape = letter2tp(space, callback_ptr.result)
             for i in range(resshape.size):
                 ll_res[i] = '\x00'
+    if must_leave:
+        space.threadlocals.leave_thread(space)
 
 class W_CallbackPtr(W_DataInstance):
 

From noreply at buildbot.pypy.org  Wed Jul  1 13:48:43 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed,  1 Jul 2015 13:48:43 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: resolving issues with * accumulation,
 there where some assembler routines i did not implement correctly
 (but where not invoked beforehand)
Message-ID: <20150701114843.23F151C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78378:88fff9cde657
Date: 2015-07-01 13:48 +0200
http://bitbucket.org/pypy/pypy/changeset/88fff9cde657/

Log:	resolving issues with * accumulation, there where some assembler
	routines i did not implement correctly (but where not invoked
	beforehand)

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -559,7 +559,6 @@
                                              self.current_clt.allgcrefs,
                                              self.current_clt.frame_info)
         self._check_frame_depth(self.mc, regalloc.get_gcmap())
-        #import pdb; pdb.set_trace()
         self._accum_update_at_exit(arglocs, inputargs, faildescr, regalloc)
         frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, operations)
         codeendpos = self.mc.get_relative_pos()
@@ -2551,7 +2550,6 @@
             elif accum_info.operation == '*':
                 self._accum_reduce_mul(arg, loc, tgtloc)
             else:
-                import pdb; pdb.set_trace()
                 not_implemented("accum operator %s not implemented" %
                                             (accum_info.operation)) 
             fail_locs[pos] = tgtloc
@@ -2559,13 +2557,13 @@
             accum_info = accum_info.prev
 
     def _accum_reduce_mul(self, arg, accumloc, targetloc):
-        scratchloc = X86_64_SCRATCH_REG
-        self.mc.mov(scratchloc, accumloc)
+        scratchloc = X86_64_XMM_SCRATCH_REG
+        self.mov(accumloc, scratchloc)
         # swap the two elements
         self.mc.SHUFPS_xxi(scratchloc.value, scratchloc.value, 0x01)
         self.mc.MULPD(accumloc, scratchloc)
         if accumloc is not targetloc:
-            self.mc.mov(targetloc, accumloc)
+            self.mov(accumloc, targetloc)
 
     def _accum_reduce_sum(self, arg, accumloc, targetloc):
         # Currently the accumulator can ONLY be the biggest
@@ -2575,7 +2573,7 @@
             self.mc.HADDPD(accumloc, accumloc)
             # upper bits (> 64) are dirty (but does not matter)
             if accumloc is not targetloc:
-                self.mov(targetloc, accumloc)
+                self.mov(accumloc, targetloc)
             return
         elif arg.type == INT:
             scratchloc = X86_64_SCRATCH_REG
@@ -2757,7 +2755,9 @@
     def genop_vec_float_expand(self, op, arglocs, resloc):
         srcloc, sizeloc = arglocs
         size = sizeloc.value
-        if size == 4:
+        if isinstance(srcloc, ConstFloatLoc):
+            self.mov(srcloc, resloc)
+        elif size == 4:
             # the register allocator forces src to be the same as resloc
             # r = (s[0], s[0], r[0], r[0])
             # since resloc == srcloc: r = (r[0], r[0], r[0], r[0])
@@ -2864,7 +2864,7 @@
                         # if source is a normal register (unpack)
                         assert count == 1
                         assert si == 0
-                        self.mov(X86_64_XMM_SCRATCH_REG, srcloc)
+                        self.mov(srcloc, X86_64_XMM_SCRATCH_REG)
                         src = X86_64_XMM_SCRATCH_REG.value
                     select = ((si & 0x3) << 6)|((ri & 0x3) << 4)
                     self.mc.INSERTPS_xxi(resloc.value, src, select)
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -81,13 +81,11 @@
         rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[1] = y
         return ConstFloatLoc(adr)
 
-    def expand_float(self, var, const):
-        assert isinstance(var, BoxVector)
-        if var.getsize() == 4:
+    def expand_float(self, size, const):
+        if size == 4:
             loc = self.expand_single_float(const)
         else:
             loc = self.expand_double_float(const)
-        self.reg_bindings[var] = loc
         return loc
 
     def expand_double_float(self, f):
@@ -1632,16 +1630,19 @@
     consider_vec_float_unpack = consider_vec_int_unpack
 
     def consider_vec_float_expand(self, op):
+        result = op.result
+        assert isinstance(result, BoxVector)
         arg = op.getarg(0)
+        args = op.getarglist()
         if isinstance(arg, Const):
-            resloc = self.xrm.expand_float(op.result, arg)
-            # TODO consider this
-            return
-        args = op.getarglist()
-        resloc = self.xrm.force_result_in_reg(op.result, arg, args)
-        assert isinstance(op.result, BoxVector)
+            resloc = self.xrm.force_allocate_reg(result)
+            srcloc = self.xrm.expand_float(result.getsize(), arg)
+        else:
+            resloc = self.xrm.force_result_in_reg(op.result, arg, args)
+            srcloc = resloc
+
         size = op.result.getsize()
-        self.perform(op, [resloc, imm(size)], resloc)
+        self.perform(op, [srcloc, imm(size)], resloc)
 
     def consider_vec_int_expand(self, op):
         arg = op.getarg(0)
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -488,8 +488,7 @@
 
 class ResumeGuardDescr(ResumeDescr):
     _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
-               'rd_frame_info_list', 'rd_pendingfields', 'rd_accum_list',
-               'status')
+               'rd_frame_info_list', 'rd_pendingfields', 'status')
     
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -156,7 +156,7 @@
     index = -1
     final_descr = False
 
-    _attrs_ = ('adr_jump_offset', 'rd_locs', 'rd_loop_token')
+    _attrs_ = ('adr_jump_offset', 'rd_locs', 'rd_loop_token', 'rd_accum_list')
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         raise NotImplementedError
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -416,7 +416,6 @@
         if vbox.gettype() == INT:
             return self.extend_int(vbox, newtype)
         else:
-            import pdb; pdb.set_trace()
             raise NotImplementedError("cannot yet extend float")
 
     def extend_int(self, vbox, newtype):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -803,7 +803,7 @@
                 # considered. => tree pattern matching problem.
                 return None
             operator = Accum.PLUS
-            if opnum == rop.FLOAT_ADD:
+            if opnum == rop.FLOAT_MUL:
                 operator = Accum.MULTIPLY
             accum = Accum(accum_var, accum_pos, operator)
             return AccumPair(lnode, rnode, ptype, ptype, accum)
@@ -837,11 +837,10 @@
                 box = result
             elif accum.operator == Accum.MULTIPLY:
                 # multiply is only supported by floats
-                op = ResOperation(rop.VEC_FLOAT_EXPAND, [ConstInt(1)], box)
+                op = ResOperation(rop.VEC_FLOAT_EXPAND, [ConstFloat(1.0)], box)
                 sched_data.invariant_oplist.append(op)
             else:
-                import pdb; pdb.set_trace()
-                raise NotImplementedError
+                raise NotImplementedError("can only handle + and *")
             result = BoxVectorAccum(box, accum.var, accum.operator)
             # pack the scalar value
             op = ResOperation(getpackopnum(box.gettype()),
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -512,8 +512,8 @@
     'VEC_FLOAT_PACK/4',          # VEC_FLOAT_PACK(vX, var/const, index, item_count)
     'VEC_INT_UNPACK/3',          # iX|fX = VEC_INT_UNPACK(vX, index, item_count)
     'VEC_INT_PACK/4',            # VEC_INT_PACK(vX, var/const, index, item_count)
-    'VEC_FLOAT_EXPAND/1',        # vX = VEC_FLOAT_EXPAND(var/const, item_count)
-    'VEC_INT_EXPAND/1',          # vX = VEC_INT_EXPAND(var/const, item_count)
+    'VEC_FLOAT_EXPAND/1',        # vX = VEC_FLOAT_EXPAND(var/const)
+    'VEC_INT_EXPAND/1',          # vX = VEC_INT_EXPAND(var/const)
     'VEC_BOX/1',
     '_VEC_PURE_LAST',
     #

From noreply at buildbot.pypy.org  Wed Jul  1 14:22:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 14:22:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: Return 1, not '(int)&W_IntObject1'
Message-ID: <20150701122202.3CF971C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78379:c9045acfde4a
Date: 2015-07-01 14:22 +0200
http://bitbucket.org/pypy/pypy/changeset/c9045acfde4a/

Log:	Return 1, not '(int)&W_IntObject1'

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -38,4 +38,4 @@
     'C') or Fortran-style (fortran is 'F') contiguous or either one
     (fortran is 'A').  Return 0 otherwise."""
     # PyPy only supports contiguous Py_buffers for now.
-    return space.wrap(1)
+    return 1

From noreply at buildbot.pypy.org  Wed Jul  1 14:38:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 14:38:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: Don't use 'rffi.cast(rffi.UINT,
 pointer)'!  This casts to a 32-bit
Message-ID: <20150701123856.C3DF41C1216@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78380:90e5ecacb270
Date: 2015-07-01 14:38 +0200
http://bitbucket.org/pypy/pypy/changeset/90e5ecacb270/

Log:	Don't use 'rffi.cast(rffi.UINT, pointer)'! This casts to a 32-bit
	integer, not a full machine-word integer.

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -136,7 +136,7 @@
     def __init__(self, ctx, protos):
         self.protos = protos
         self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
-        NPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+        NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
         # can be used to create both types of sockets
@@ -151,7 +151,7 @@
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
-        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if npn and npn.protos:
             data_ptr[0] = npn.buf
             len_ptr[0] = rffi.cast(rffi.UINT, len(npn.protos))
@@ -163,7 +163,7 @@
 
     @staticmethod
     def selectNPN_cb(s, out_ptr, outlen_ptr, server, server_len, args):
-        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if npn and npn.protos:
             client = npn.buf
             client_len = len(npn.protos)
@@ -182,7 +182,7 @@
     def __init__(self, ctx, protos):
         self.protos = protos
         self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
-        ALPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+        ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
             if libssl_SSL_CTX_set_alpn_protos(
@@ -197,7 +197,7 @@
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
-        alpn = ALPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        alpn = ALPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if alpn and alpn.protos:
             server = alpn.buf
             server_len = len(alpn.protos)

From noreply at buildbot.pypy.org  Wed Jul  1 15:22:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 15:22:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Copy this thread initialization logic
 from module/_cffi_backend/ccallback.py
Message-ID: <20150701132224.08CC31C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78381:9f9765fe1fa0
Date: 2015-07-01 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/9f9765fe1fa0/

Log:	Copy this thread initialization logic from
	module/_cffi_backend/ccallback.py

diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -79,6 +79,14 @@
         if tracker.DO_TRACING:
             addr = rffi.cast(lltype.Signed, self.ll_callback.ll_closure)
             tracker.trace_allocation(addr, self)
+        #
+        # We must setup the GIL here, in case the callback is invoked in
+        # some other non-Pythonic thread.  This is the same as ctypes on
+        # CPython (but only when creating a callback; on CPython it occurs
+        # as soon as we import _ctypes)
+        if space.config.translation.thread:
+            from pypy.module.thread.os_thread import setup_threads
+            setup_threads(space)
 
     def free(self):
         if tracker.DO_TRACING:

From noreply at buildbot.pypy.org  Wed Jul  1 18:05:18 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed,  1 Jul 2015 18:05:18 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: updated docs
Message-ID: <20150701160518.49C2F1C11D3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78382:c878b10c3713
Date: 2015-07-01 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/c878b10c3713/

Log:	updated docs

diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
--- a/rpython/doc/jit/vectorization.rst
+++ b/rpython/doc/jit/vectorization.rst
@@ -21,11 +21,7 @@
 
 Reduction is implemented:
 
-* sum
-
-Planned reductions:
-
-* all, any, prod, min, max
+* sum, prod, any, all
 
 Constant & Variable Expansion
 -----------------------------
@@ -35,7 +31,7 @@
 Guard Strengthening
 -------------------
 
-Unrolled guards are strengthend on a arithmetical level (See GuardStrengthenOpt).
+Unrolled guards are strengthend on an arithmetical level (See GuardStrengthenOpt).
 The resulting vector trace will only have one guard that checks the index.
 
 Calculations on the index variable that are redundant (because of the merged
@@ -57,5 +53,6 @@
 * For a guard that checks true/false on a vector integer regsiter, it would be handy
   to have 2 xmm registers (one filled with zero bits and the other with one every bit).
   This cuts down 2 instructions for guard checking, trading for higher register pressure.
+* prod, sum are only supported by 64 bit data types
 
 .. _PMUL: http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025

From noreply at buildbot.pypy.org  Wed Jul  1 18:05:19 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed,  1 Jul 2015 18:05:19 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: restricted input size for vectorization
 to 75 operations max (dont see any chance for it to succeed with more than
 20-30 operations)
Message-ID: <20150701160519.6FB311C146D@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78383:a01f8e4831c4
Date: 2015-07-01 18:05 +0200
http://bitbucket.org/pypy/pypy/changeset/a01f8e4831c4/

Log:	restricted input size for vectorization to 75 operations max (dont
	see any chance for it to succeed with more than 20-30 operations)
	reduce resetting rval at the end of the loop

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -220,6 +220,7 @@
             return rval
         cur_value = func(calc_dtype, cur_value, rval)
         obj_state = obj_iter.next(obj_state)
+        rval = None
     return cur_value
 
 reduce_cum_driver = jit.JitDriver(
@@ -346,6 +347,7 @@
             temp_state = temp_iter.next(temp_state)
         else:
             temp_state = out_state
+        w_val = None
     return out
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -50,6 +50,11 @@
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, False)
     orig_ops = loop.operations
+    if len(orig_ops) >= 75:
+        # if more than 75 operations are present in this loop,
+        # it won't be possible to vectorize. There are too many
+        # guards that prevent parallel execution of instructions
+        return
     start = -1
     end = -1
     try:

From noreply at buildbot.pypy.org  Wed Jul  1 19:24:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 19:24:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Minor optimization
Message-ID: <20150701172431.746141C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78384:e99ce6af254c
Date: 2015-07-01 19:08 +0200
http://bitbucket.org/pypy/pypy/changeset/e99ce6af254c/

Log:	Minor optimization

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -194,9 +194,9 @@
 
     def switch_to_object_strategy(self):
         list_w = self.getitems()
-        self.strategy = self.space.fromcache(ObjectListStrategy)
-        # XXX this is quite indirect
-        self.init_from_list_w(list_w)
+        object_strategy = self.space.fromcache(ObjectListStrategy)
+        self.strategy = object_strategy
+        object_strategy.init_from_list_w(self, list_w)
 
     def _temporarily_as_objects(self):
         if self.strategy is self.space.fromcache(ObjectListStrategy):

From noreply at buildbot.pypy.org  Wed Jul  1 19:24:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 19:24:32 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: A branch to try out the
 list-of-ints-or-floats strategy
Message-ID: <20150701172432.9A9701C11D3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78385:bc456eaaec25
Date: 2015-07-01 19:08 +0200
http://bitbucket.org/pypy/pypy/changeset/bc456eaaec25/

Log:	A branch to try out the list-of-ints-or-floats strategy


From noreply at buildbot.pypy.org  Wed Jul  1 19:24:33 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 19:24:33 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: in-progress
Message-ID: <20150701172433.BA4151C1277@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78386:d38f6591e679
Date: 2015-07-01 19:18 +0200
http://bitbucket.org/pypy/pypy/changeset/d38f6591e679/

Log:	in-progress

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -14,6 +14,7 @@
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
     import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize)
+from rpython.rlib import longlong2float
 from rpython.tool.sourcetools import func_with_new_name
 
 from pypy.interpreter.baseobjspace import W_Root
@@ -73,33 +74,56 @@
             return SizeListStrategy(space, sizehint)
         return space.fromcache(EmptyListStrategy)
 
-    # check for ints
-    for w_obj in list_w:
-        if not type(w_obj) is W_IntObject:
+    w_firstobj = list_w[0]
+    check_int_or_float = False
+
+    if type(w_firstobj) is W_IntObject:
+        # check for all-ints
+        for i in range(1, len(list_w)):
+            w_obj = list_w[i]
+            if type(w_obj) is not W_IntObject:
+                check_int_or_float = (type(w_obj) is W_FloatObject)
+                break
+        else:
+            return space.fromcache(IntegerListStrategy)
+
+    elif type(w_firstobj) is W_BytesObject:
+        # check for all-strings
+        for i in range(1, len(list_w)):
+            if type(list_w[i]) is not W_BytesObject:
+                break
+        else:
+            return space.fromcache(BytesListStrategy)
+
+    elif type(w_firstobj) is W_UnicodeObject:
+        # check for all-unicodes
+        for i in range(1, len(list_w)):
+            if type(list_w[i]) is not W_UnicodeObject:
+                break
+        else:
+            return space.fromcache(UnicodeListStrategy)
+
+    elif type(w_firstobj) is W_FloatObject:
+        # check for all-floats
+        for i in range(1, len(list_w)):
+            w_obj = list_w[i]
+            if type(w_obj) is not W_FloatObject:
+                check_int_or_float = (type(w_obj) is W_IntObject)
+                break
+        else:
+            return space.fromcache(FloatListStrategy)
+
+    if check_int_or_float:
+        for w_obj in list_w:
+            if type(w_obj) is W_IntObject:
+                if longlong2float.can_encode_int32(space.int_w(w_obj)):
+                    continue    # ok
+            elif type(w_obj) is W_FloatObject:
+                if longlong2float.can_encode_float(space.float_w(w_obj)):
+                    continue    # ok
             break
-    else:
-        return space.fromcache(IntegerListStrategy)
-
-    # check for strings
-    for w_obj in list_w:
-        if not type(w_obj) is W_BytesObject:
-            break
-    else:
-        return space.fromcache(BytesListStrategy)
-
-    # check for unicode
-    for w_obj in list_w:
-        if not type(w_obj) is W_UnicodeObject:
-            break
-    else:
-        return space.fromcache(UnicodeListStrategy)
-
-    # check for floats
-    for w_obj in list_w:
-        if not type(w_obj) is W_FloatObject:
-            break
-    else:
-        return space.fromcache(FloatListStrategy)
+        else:
+            return space.fromcache(IntOrFloatListStrategy)
 
     return space.fromcache(ObjectListStrategy)
 
@@ -1382,12 +1406,15 @@
             return W_ListObject.from_storage_and_strategy(
                     self.space, storage, self)
 
+    def switch_to_next_strategy(self, w_list, w_sample_item):
+        w_list.switch_to_object_strategy()
+
     def append(self, w_list, w_item):
         if self.is_correct_type(w_item):
             self.unerase(w_list.lstorage).append(self.unwrap(w_item))
             return
 
-        w_list.switch_to_object_strategy()
+        self.switch_to_next_strategy(w_list, w_item)
         w_list.append(w_item)
 
     def insert(self, w_list, index, w_item):
@@ -1397,7 +1424,7 @@
             l.insert(index, self.unwrap(w_item))
             return
 
-        w_list.switch_to_object_strategy()
+        self.switch_to_next_strategy(w_list, w_item)
         w_list.insert(index, w_item)
 
     def _extend_from_list(self, w_list, w_other):
@@ -1673,7 +1700,6 @@
 
     def _safe_find(self, w_list, obj, start, stop):
         from rpython.rlib.rfloat import isnan
-        from rpython.rlib.longlong2float import float2longlong
         #
         l = self.unerase(w_list.lstorage)
         stop = min(stop, len(l))
@@ -1683,10 +1709,10 @@
                 if val == obj:
                     return i
         else:
-            search = float2longlong(obj)
+            search = longlong2float.float2longlong(obj)
             for i in range(start, stop):
                 val = l[i]
-                if float2longlong(val) == search:
+                if longlong2float.float2longlong(val) == search:
                     return i
         raise ValueError
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -733,6 +733,11 @@
         list_copy[0] = 42
         assert list_orig == [1, 2, 3]
 
+    def test_int_or_float(self):
+        space = self.space
+        w_l = W_ListObject(space, [space.wrap(1), space.wrap(2.3)])
+        xxxxx
+
 
 class TestW_ListStrategiesDisabled:
     spaceconfig = {"objspace.std.withliststrategies": False}
diff --git a/rpython/rlib/longlong2float.py b/rpython/rlib/longlong2float.py
--- a/rpython/rlib/longlong2float.py
+++ b/rpython/rlib/longlong2float.py
@@ -7,6 +7,7 @@
 """
 
 from __future__ import with_statement
+import sys
 from rpython.annotator import model as annmodel
 from rpython.rlib.rarithmetic import r_int64
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -99,3 +100,29 @@
         [v_longlong] = hop.inputargs(lltype.SignedLongLong)
         hop.exception_cannot_occur()
         return hop.genop("convert_longlong_bytes_to_float", [v_longlong], resulttype=lltype.Float)
+
+# ____________________________________________________________
+
+
+# For encoding integers inside nonstandard NaN bit patterns.
+#   ff ff ff fe xx xx xx xx    (signed 32-bit int)
+nan_high_word_int32 = -2       # -2 == (int)0xfffffffe
+nan_encoded_zero = rffi.cast(rffi.LONGLONG, nan_high_word_int32 << 32)
+
+def encode_int32_into_longlong_nan(value):
+    return (nan_encoded_zero +
+            rffi.cast(rffi.LONGLONG, rffi.cast(rffi.UINT, value)))
+
+def decode_int32_from_longlong_nan(value):
+    return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+
+def is_int32_from_longlong_nan(value):
+    return (value >> 32) == nan_high_word_int32
+
+def can_encode_int32(value):
+    if sys.maxint == 2147483647:
+        return True
+    return value == rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+
+def can_encode_float(value):
+    return (float2longlong(value) >> 32) != nan_high_word_int32

From noreply at buildbot.pypy.org  Wed Jul  1 19:24:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 19:24:34 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: more tests
Message-ID: <20150701172434.D394E1C146D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78387:75683789a8bd
Date: 2015-07-01 19:24 +0200
http://bitbucket.org/pypy/pypy/changeset/75683789a8bd/

Log:	more tests

diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -1,4 +1,5 @@
 import sys
+import py
 from pypy.objspace.std.listobject import (
     W_ListObject, EmptyListStrategy, ObjectListStrategy, IntegerListStrategy,
     FloatListStrategy, BytesListStrategy, RangeListStrategy,
@@ -166,7 +167,6 @@
         assert isinstance(l.strategy, IntegerListStrategy)
 
     def test_list_empty_after_delete(self):
-        import py
         py.test.skip("return to emptyliststrategy is not supported anymore")
         l = W_ListObject(self.space, [self.space.wrap(3)])
         assert isinstance(l.strategy, IntegerListStrategy)
@@ -733,6 +733,31 @@
         list_copy[0] = 42
         assert list_orig == [1, 2, 3]
 
+    def test_int_or_float_special_nan(self):
+        from rpython.rlib import longlong2float, rarithmetic
+        space = self.space
+        ll = rarithmetic.r_longlong(0xfffffffe12345678 - 2**64)
+        specialnan = longlong2float.longlong2float(ll)
+        w_l = W_ListObject(space, [space.wrap(1), space.wrap(specialnan)])
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+
+    def test_int_or_float_int_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        ok1 = 2**31 - 1
+        ok2 = -2**31
+        ovf1 = ok1 + 1
+        ovf2 = ok2 - 1
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(ovf1)])
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(ovf2)])
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(ok1)])
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(ok2)])
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+
     def test_int_or_float(self):
         space = self.space
         w_l = W_ListObject(space, [space.wrap(1), space.wrap(2.3)])

From noreply at buildbot.pypy.org  Wed Jul  1 23:17:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 23:17:04 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: Start writing the
	strategy
Message-ID: <20150701211704.7E5DE1C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78388:2e22e8a3d2d4
Date: 2015-07-01 22:15 +0200
http://bitbucket.org/pypy/pypy/changeset/2e22e8a3d2d4/

Log:	Start writing the strategy

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1716,6 +1716,41 @@
                     return i
         raise ValueError
 
+class IntOrFloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
+    _none_value = longlong2float.float2longlong(0.0)
+
+    def wrap(self, llval):
+        if longlong2float.is_int32_from_longlong_nan(llval):
+            intval = longlong2float.decode_int32_from_longlong_nan(llval)
+            return self.space.wrap(intval)
+        else:
+            floatval = longlong2float.longlong2float(llval)
+            return self.space.wrap(floatval)
+
+    def unwrap(self, w_int_or_float):
+        if type(w_int_or_float) is W_IntObject:
+            intval = self.space.int_w(w_int_or_float)
+            return longlong2float.encode_int32_into_longlong_nan(intval)
+        else:
+            floatval = self.space.float_w(w_int_or_float)
+            return longlong2float.float2longlong(floatval)
+
+    erase, unerase = rerased.new_erasing_pair("longlong")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def is_correct_type(self, w_obj):
+        if type(w_obj) is W_IntObject:
+            intval = self.space.int_w(w_obj)
+            return longlong2float.can_encode_int32(intval)
+        elif type(w_obj) is W_FloatObject:
+            floatval = self.space.float_w(w_obj)
+            return longlong2float.can_encode_float(floatval)
+        else:
+            return False
+
 
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -3,7 +3,8 @@
 from pypy.objspace.std.listobject import (
     W_ListObject, EmptyListStrategy, ObjectListStrategy, IntegerListStrategy,
     FloatListStrategy, BytesListStrategy, RangeListStrategy,
-    SimpleRangeListStrategy, make_range_list, UnicodeListStrategy)
+    SimpleRangeListStrategy, make_range_list, UnicodeListStrategy,
+    IntOrFloatListStrategy)
 from pypy.objspace.std import listobject
 from pypy.objspace.std.test.test_listobject import TestW_ListObject
 
@@ -759,9 +760,23 @@
         assert isinstance(w_l.strategy, IntOrFloatListStrategy)
 
     def test_int_or_float(self):
+        from rpython.rlib.rfloat import INFINITY, NAN
         space = self.space
+        w = space.wrap
         w_l = W_ListObject(space, [space.wrap(1), space.wrap(2.3)])
-        xxxxx
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(w(int(2**31-1)))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(w(-5.1))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(w(INFINITY))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(w(NAN))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(w(-NAN))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(space.newlist([]))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
 
 
 class TestW_ListStrategiesDisabled:
diff --git a/rpython/rlib/test/test_longlong2float.py b/rpython/rlib/test/test_longlong2float.py
--- a/rpython/rlib/test/test_longlong2float.py
+++ b/rpython/rlib/test/test_longlong2float.py
@@ -1,7 +1,7 @@
 from rpython.translator.c.test.test_genc import compile
 from rpython.rlib.longlong2float import longlong2float, float2longlong
 from rpython.rlib.longlong2float import uint2singlefloat, singlefloat2uint
-from rpython.rlib.rarithmetic import r_singlefloat
+from rpython.rlib.rarithmetic import r_singlefloat, r_longlong
 from rpython.rtyper.test.test_llinterp import interpret
 
 
@@ -21,6 +21,9 @@
     yield -inf
     yield inf / inf     # nan
 
+def test_float2longlong():
+    assert float2longlong(0.0) == r_longlong(0)
+
 def test_longlong_as_float():
     for x in enum_floats():
         res = fn(x)

From noreply at buildbot.pypy.org  Wed Jul  1 23:17:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 23:17:05 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: int -> int-or-float
Message-ID: <20150701211705.9D32C1C1545@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78389:4ed327cd6401
Date: 2015-07-01 23:03 +0200
http://bitbucket.org/pypy/pypy/changeset/4ed327cd6401/

Log:	int -> int-or-float

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1667,6 +1667,30 @@
                     self.space, storage, self)
         return self._base_setslice(w_list, start, step, slicelength, w_other)
 
+
+    def switch_to_next_strategy(self, w_list, w_sample_item):
+        if type(w_sample_item) is W_FloatObject:
+            l = self.unerase(w_list.lstorage)
+            for intval in l:
+                if not longlong2float.can_encode_int32(intval):
+                    break
+            else:
+                floatval = self.space.float_w(w_sample_item)
+                if longlong2float.can_encode_float(floatval):
+                    # yes, we can switch to IntOrFloatListStrategy
+                    generalized_list = [
+                        longlong2float.encode_int32_into_longlong_nan(intval)
+                        for intval in l]
+                    generalized_list.append(
+                        longlong2float.float2longlong(floatval))
+                    strategy = self.space.fromcache(IntOrFloatListStrategy)
+                    w_list.strategy = strategy
+                    w_list.lstorage = strategy.erase(generalized_list)
+                    return
+        # no, fall back to ObjectListStrategy
+        w_list.switch_to_object_strategy()
+
+
 class FloatListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -759,7 +759,7 @@
         w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(ok2)])
         assert isinstance(w_l.strategy, IntOrFloatListStrategy)
 
-    def test_int_or_float(self):
+    def test_int_or_float_base(self):
         from rpython.rlib.rfloat import INFINITY, NAN
         space = self.space
         w = space.wrap
@@ -769,6 +769,8 @@
         assert isinstance(w_l.strategy, IntOrFloatListStrategy)
         w_l.append(w(-5.1))
         assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        assert space.int_w(w_l.getitem(2)) == 2**31-1
+        assert space.float_w(w_l.getitem(3)) == -5.1
         w_l.append(w(INFINITY))
         assert isinstance(w_l.strategy, IntOrFloatListStrategy)
         w_l.append(w(NAN))
@@ -778,6 +780,16 @@
         w_l.append(space.newlist([]))
         assert isinstance(w_l.strategy, ObjectListStrategy)
 
+    def test_int_or_float_from_integer(self):
+        space = self.space
+        w = space.wrap
+        w_l = W_ListObject(space, [space.wrap(int(-2**31))])
+        assert isinstance(w_l.strategy, IntegerListStrategy)
+        w_l.append(w(-5.1))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        assert space.int_w(w_l.getitem(0)) == -2**31
+        assert space.float_w(w_l.getitem(1)) == -5.1
+
 
 class TestW_ListStrategiesDisabled:
     spaceconfig = {"objspace.std.withliststrategies": False}

From noreply at buildbot.pypy.org  Wed Jul  1 23:17:06 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 23:17:06 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: More tests,
	and float -> int-or-float
Message-ID: <20150701211706.BCB1F1C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78390:12404829e22a
Date: 2015-07-01 23:17 +0200
http://bitbucket.org/pypy/pypy/changeset/12404829e22a/

Log:	More tests, and float -> int-or-float

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1675,18 +1675,17 @@
                 if not longlong2float.can_encode_int32(intval):
                     break
             else:
-                floatval = self.space.float_w(w_sample_item)
-                if longlong2float.can_encode_float(floatval):
-                    # yes, we can switch to IntOrFloatListStrategy
-                    generalized_list = [
-                        longlong2float.encode_int32_into_longlong_nan(intval)
-                        for intval in l]
-                    generalized_list.append(
-                        longlong2float.float2longlong(floatval))
-                    strategy = self.space.fromcache(IntOrFloatListStrategy)
-                    w_list.strategy = strategy
-                    w_list.lstorage = strategy.erase(generalized_list)
-                    return
+                # yes, we can switch to IntOrFloatListStrategy
+                # (ignore here the extremely unlikely case where
+                # w_sample_item is just the wrong nonstandard NaN float;
+                # it will caught later and yet another switch will occur)
+                generalized_list = [
+                    longlong2float.encode_int32_into_longlong_nan(intval)
+                    for intval in l]
+                strategy = self.space.fromcache(IntOrFloatListStrategy)
+                w_list.strategy = strategy
+                w_list.lstorage = strategy.erase(generalized_list)
+                return
         # no, fall back to ObjectListStrategy
         w_list.switch_to_object_strategy()
 
@@ -1740,6 +1739,29 @@
                     return i
         raise ValueError
 
+    def switch_to_next_strategy(self, w_list, w_sample_item):
+        if type(w_sample_item) is W_IntObject:
+            intval = self.space.int_w(w_sample_item)
+            if longlong2float.can_encode_int32(intval):
+                # xxx we should be able to use the same lstorage, but
+                # there is a typing issue (float vs longlong)...
+                l = self.unerase(w_list.lstorage)
+                generalized_list = []
+                for floatval in l:
+                    if not longlong2float.can_encode_float(floatval):
+                        break
+                    generalized_list.append(
+                        longlong2float.float2longlong(floatval))
+                else:
+                    # yes, we can switch to IntOrFloatListStrategy
+                    strategy = self.space.fromcache(IntOrFloatListStrategy)
+                    w_list.strategy = strategy
+                    w_list.lstorage = strategy.erase(generalized_list)
+                    return
+        # no, fall back to ObjectListStrategy
+        w_list.switch_to_object_strategy()
+
+
 class IntOrFloatListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -789,6 +789,72 @@
         assert isinstance(w_l.strategy, IntOrFloatListStrategy)
         assert space.int_w(w_l.getitem(0)) == -2**31
         assert space.float_w(w_l.getitem(1)) == -5.1
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_integer_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        w = space.wrap
+        ovf1 = -2**31 - 1
+        w_l = W_ListObject(space, [space.wrap(ovf1)])
+        assert isinstance(w_l.strategy, IntegerListStrategy)
+        w_l.append(w(-5.1))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert space.int_w(w_l.getitem(0)) == ovf1
+        assert space.float_w(w_l.getitem(1)) == -5.1
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_integer_special_nan(self):
+        from rpython.rlib import longlong2float, rarithmetic
+        space = self.space
+        w = space.wrap
+        w_l = W_ListObject(space, [space.wrap(int(-2**31))])
+        assert isinstance(w_l.strategy, IntegerListStrategy)
+        ll = rarithmetic.r_longlong(0xfffffffe12345678 - 2**64)
+        specialnan = longlong2float.longlong2float(ll)
+        w_l.append(w(specialnan))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert space.int_w(w_l.getitem(0)) == -2**31
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_float(self):
+        space = self.space
+        w = space.wrap
+        w_l = W_ListObject(space, [space.wrap(-42.5)])
+        assert isinstance(w_l.strategy, FloatListStrategy)
+        w_l.append(w(-15))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        assert space.float_w(w_l.getitem(0)) == -42.5
+        assert space.int_w(w_l.getitem(1)) == -15
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_float_int_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        w = space.wrap
+        ovf1 = 2 ** 31
+        w_l = W_ListObject(space, [space.wrap(1.2)])
+        assert isinstance(w_l.strategy, FloatListStrategy)
+        w_l.append(w(ovf1))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert space.float_w(w_l.getitem(0)) == 1.2
+        assert space.int_w(w_l.getitem(1)) == ovf1
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_float_special_nan(self):
+        from rpython.rlib import longlong2float, rarithmetic
+        space = self.space
+        w = space.wrap
+        ll = rarithmetic.r_longlong(0xfffffffe12345678 - 2**64)
+        specialnan = longlong2float.longlong2float(ll)
+        w_l = W_ListObject(space, [space.wrap(specialnan)])
+        assert isinstance(w_l.strategy, FloatListStrategy)
+        w_l.append(w(42))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert space.int_w(w_l.getitem(1)) == 42
+        assert space.len_w(w_l) == 2
 
 
 class TestW_ListStrategiesDisabled:

From noreply at buildbot.pypy.org  Wed Jul  1 23:53:51 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 23:53:51 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: Some skipped tests (for
	later maybe)
Message-ID: <20150701215351.9D33C1C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78391:4cfaef813f82
Date: 2015-07-01 23:26 +0200
http://bitbucket.org/pypy/pypy/changeset/4cfaef813f82/

Log:	Some skipped tests (for later maybe)

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1448,7 +1448,7 @@
             except IndexError:
                 raise
         else:
-            w_list.switch_to_object_strategy()
+            self.switch_to_next_strategy(w_list, w_item)
             w_list.setitem(index, w_item)
 
     def setslice(self, w_list, start, step, slicelength, w_other):
@@ -1741,8 +1741,8 @@
 
     def switch_to_next_strategy(self, w_list, w_sample_item):
         if type(w_sample_item) is W_IntObject:
-            intval = self.space.int_w(w_sample_item)
-            if longlong2float.can_encode_int32(intval):
+            sample_intval = self.space.int_w(w_sample_item)
+            if longlong2float.can_encode_int32(sample_intval):
                 # xxx we should be able to use the same lstorage, but
                 # there is a typing issue (float vs longlong)...
                 l = self.unerase(w_list.lstorage)
@@ -1797,6 +1797,9 @@
         else:
             return False
 
+    def list_is_correct_type(self, w_list):
+        return w_list.strategy is self.space.fromcache(IntOrFloatListStrategy)
+
 
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -856,6 +856,44 @@
         assert space.int_w(w_l.getitem(1)) == 42
         assert space.len_w(w_l) == 2
 
+    def test_int_or_float_extend(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3), space.wrap(4.5)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [0, 1.2, 3, 4.5]
+
+    def test_int_or_float_extend_mixed(self):
+        py.test.skip("XXX not implemented")
+        # lst = [0]; lst += [1.2]
+        # lst = [0]; lst += [1.2, 3]
+        # lst = [1.2]; lst += [0]
+        # lst = [1.2]; lst += [0, 3.4]
+        # lst = [0, 1.2]; lst += [3]
+        # lst = [0, 1.2]; lst += [3.4]
+
+    def test_int_or_float_setslice(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3), space.wrap(4.5)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3, 4.5, 1.2]
+
+    def test_int_or_float_setslice_mixed(self):
+        py.test.skip("XXX not implemented")
+        # lst = [0]; lst[:] = [1.2]
+        # lst = [0]; lst[:] = [1.2, 3]
+        # lst = [1.2]; lst[:] = [0]
+        # lst = [1.2]; lst[:] = [0, 3.4]
+        # lst = [0, 1.2]; lst[:] = [3]
+        # lst = [0, 1.2]; lst[:] = [3.4]
+
 
 class TestW_ListStrategiesDisabled:
     spaceconfig = {"objspace.std.withliststrategies": False}

From noreply at buildbot.pypy.org  Wed Jul  1 23:53:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 23:53:52 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: int-or-float sort()
Message-ID: <20150701215352.D1CB61C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78392:a46ca0349945
Date: 2015-07-01 23:37 +0200
http://bitbucket.org/pypy/pypy/changeset/a46ca0349945/

Log:	int-or-float sort()

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1613,6 +1613,9 @@
     def getitems(self, w_list):
         return self.unerase(w_list.lstorage)
 
+    # no sort() method here: W_ListObject.descr_sort() handles this
+    # case explicitly
+
 
 class IntegerListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
@@ -1800,6 +1803,17 @@
     def list_is_correct_type(self, w_list):
         return w_list.strategy is self.space.fromcache(IntOrFloatListStrategy)
 
+    def sort(self, w_list, reverse):
+        l = self.unerase(w_list.lstorage)
+        sorter = IntOrFloatSort(l, len(l))
+        # Reverse sort stability achieved by initially reversing the list,
+        # applying a stable forward sort, then reversing the final result.
+        if reverse:
+            l.reverse()
+        sorter.sort()
+        if reverse:
+            l.reverse()
+
 
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
@@ -1896,6 +1910,7 @@
 TimSort = make_timsort_class()
 IntBaseTimSort = make_timsort_class()
 FloatBaseTimSort = make_timsort_class()
+IntOrFloatBaseTimSort = make_timsort_class()
 StringBaseTimSort = make_timsort_class()
 UnicodeBaseTimSort = make_timsort_class()
 
@@ -1926,6 +1941,19 @@
         return a < b
 
 
+class IntOrFloatSort(IntOrFloatBaseTimSort):
+    def lt(self, a, b):
+        if longlong2float.is_int32_from_longlong_nan(a):
+            fa = float(longlong2float.decode_int32_from_longlong_nan(a))
+        else:
+            fa = longlong2float.longlong2float(a)
+        if longlong2float.is_int32_from_longlong_nan(b):
+            fb = float(longlong2float.decode_int32_from_longlong_nan(b))
+        else:
+            fb = longlong2float.longlong2float(b)
+        return fa < fb
+
+
 class StringSort(StringBaseTimSort):
     def lt(self, a, b):
         return a < b
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -894,6 +894,17 @@
         # lst = [0, 1.2]; lst[:] = [3]
         # lst = [0, 1.2]; lst[:] = [3.4]
 
+    def test_int_or_float_sort(self):
+        space = self.space
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(1),
+                                   space.wrap(1.0), space.wrap(5)])
+        w_l.sort(False)
+        assert [(type(x), x) for x in space.unwrap(w_l)] == [
+            (int, 1), (float, 1.0), (float, 1.2), (int, 5)]
+        w_l.sort(True)
+        assert [(type(x), x) for x in space.unwrap(w_l)] == [
+            (int, 5), (float, 1.2), (int, 1), (float, 1.0)]
+
 
 class TestW_ListStrategiesDisabled:
     spaceconfig = {"objspace.std.withliststrategies": False}

From noreply at buildbot.pypy.org  Wed Jul  1 23:53:54 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 23:53:54 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: Support the (likely)
 most common mixed cases:
Message-ID: <20150701215354.0059E1C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78393:8e7ea72a85bf
Date: 2015-07-01 23:53 +0200
http://bitbucket.org/pypy/pypy/changeset/8e7ea72a85bf/

Log:	Support the (likely) most common mixed cases:

	 [int-or-float].extend([int])

	 [int-or-float].extend([float])

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1671,20 +1671,26 @@
         return self._base_setslice(w_list, start, step, slicelength, w_other)
 
 
+    def int_2_float_or_int(self, w_list):
+        l = self.unerase(w_list.lstorage)
+        if not longlong2float.CAN_ALWAYS_ENCODE_INT32:
+            for intval in l:
+                if not longlong2float.can_encode_int32(intval):
+                    raise ValueError
+        return [longlong2float.encode_int32_into_longlong_nan(intval)
+                for intval in l]
+
     def switch_to_next_strategy(self, w_list, w_sample_item):
         if type(w_sample_item) is W_FloatObject:
-            l = self.unerase(w_list.lstorage)
-            for intval in l:
-                if not longlong2float.can_encode_int32(intval):
-                    break
+            try:
+                generalized_list = self.int_2_float_or_int(w_list)
+            except ValueError:
+                pass
             else:
                 # yes, we can switch to IntOrFloatListStrategy
                 # (ignore here the extremely unlikely case where
                 # w_sample_item is just the wrong nonstandard NaN float;
                 # it will caught later and yet another switch will occur)
-                generalized_list = [
-                    longlong2float.encode_int32_into_longlong_nan(intval)
-                    for intval in l]
                 strategy = self.space.fromcache(IntOrFloatListStrategy)
                 w_list.strategy = strategy
                 w_list.lstorage = strategy.erase(generalized_list)
@@ -1742,19 +1748,26 @@
                     return i
         raise ValueError
 
+    def float_2_float_or_int(self, w_list):
+        l = self.unerase(w_list.lstorage)
+        generalized_list = []
+        for floatval in l:
+            if not longlong2float.can_encode_float(floatval):
+                raise ValueError
+            generalized_list.append(
+                longlong2float.float2longlong(floatval))
+        return generalized_list
+
     def switch_to_next_strategy(self, w_list, w_sample_item):
         if type(w_sample_item) is W_IntObject:
             sample_intval = self.space.int_w(w_sample_item)
             if longlong2float.can_encode_int32(sample_intval):
                 # xxx we should be able to use the same lstorage, but
                 # there is a typing issue (float vs longlong)...
-                l = self.unerase(w_list.lstorage)
-                generalized_list = []
-                for floatval in l:
-                    if not longlong2float.can_encode_float(floatval):
-                        break
-                    generalized_list.append(
-                        longlong2float.float2longlong(floatval))
+                try:
+                    generalized_list = self.float_2_float_or_int(w_list)
+                except ValueError:
+                    pass
                 else:
                     # yes, we can switch to IntOrFloatListStrategy
                     strategy = self.space.fromcache(IntOrFloatListStrategy)
@@ -1814,6 +1827,29 @@
         if reverse:
             l.reverse()
 
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_longlong(self, w_list, longlong_list):
+        l = self.unerase(w_list.lstorage)
+        l += longlong_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(IntegerListStrategy):
+            try:
+                longlong_list = w_other.strategy.int_2_float_or_int(w_other)
+            except ValueError:
+                pass
+            else:
+                return self._extend_longlong(w_list, longlong_list)
+        if w_other.strategy is self.space.fromcache(FloatListStrategy):
+            try:
+                longlong_list = w_other.strategy.float_2_float_or_int(w_other)
+            except ValueError:
+                pass
+            else:
+                return self._extend_longlong(w_list, longlong_list)
+        return self._base_extend_from_list(w_list, w_other)
+
 
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -866,7 +866,28 @@
         assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
         assert space.unwrap(w_l1) == [0, 1.2, 3, 4.5]
 
-    def test_int_or_float_extend_mixed(self):
+    def test_int_or_float_extend_mixed_1(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert [(type(x), x) for x in space.unwrap(w_l1)] == [
+            (int, 0), (float, 1.2), (int, 3)]
+
+    def test_int_or_float_extend_mixed_2(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3.4)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, FloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [0, 1.2, 3.4]
+
+    def test_int_or_float_extend_mixed_3(self):
         py.test.skip("XXX not implemented")
         # lst = [0]; lst += [1.2]
         # lst = [0]; lst += [1.2, 3]
diff --git a/rpython/rlib/longlong2float.py b/rpython/rlib/longlong2float.py
--- a/rpython/rlib/longlong2float.py
+++ b/rpython/rlib/longlong2float.py
@@ -119,8 +119,10 @@
 def is_int32_from_longlong_nan(value):
     return (value >> 32) == nan_high_word_int32
 
+CAN_ALWAYS_ENCODE_INT32 = (sys.maxint == 2147483647)
+
 def can_encode_int32(value):
-    if sys.maxint == 2147483647:
+    if CAN_ALWAYS_ENCODE_INT32:
         return True
     return value == rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
 

From noreply at buildbot.pypy.org  Wed Jul  1 23:54:19 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  1 Jul 2015 23:54:19 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: These two cases are now
	implemented
Message-ID: <20150701215419.BB3E41C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78394:be46d1ff397d
Date: 2015-07-01 23:54 +0200
http://bitbucket.org/pypy/pypy/changeset/be46d1ff397d/

Log:	These two cases are now implemented

diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -893,8 +893,6 @@
         # lst = [0]; lst += [1.2, 3]
         # lst = [1.2]; lst += [0]
         # lst = [1.2]; lst += [0, 3.4]
-        # lst = [0, 1.2]; lst += [3]
-        # lst = [0, 1.2]; lst += [3.4]
 
     def test_int_or_float_setslice(self):
         space = self.space

From noreply at buildbot.pypy.org  Thu Jul  2 00:10:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 00:10:11 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: Add comment
Message-ID: <20150701221011.7414D1C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78395:e00563965dbd
Date: 2015-07-02 00:10 +0200
http://bitbucket.org/pypy/pypy/changeset/e00563965dbd/

Log:	Add comment

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1979,6 +1979,13 @@
 
 class IntOrFloatSort(IntOrFloatBaseTimSort):
     def lt(self, a, b):
+        # This relies on the fact that casting from a decoded int to a
+        # float is an exact operation.  If we had full 64-bit
+        # integers, this cast would loose precision.  But this works
+        # because the integers are only 32-bit.  This would also work
+        # even if we encoded larger integers: as long as they are
+        # encoded inside a subset of the mantissa of a float, then the
+        # cast-to-float will be exact.
         if longlong2float.is_int32_from_longlong_nan(a):
             fa = float(longlong2float.decode_int32_from_longlong_nan(a))
         else:

From noreply at buildbot.pypy.org  Thu Jul  2 01:46:34 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu,  2 Jul 2015 01:46:34 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix syntax error.
Message-ID: <20150701234634.4F8C21C0A5B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78396:378aaa60e293
Date: 2015-07-02 01:46 +0200
http://bitbucket.org/pypy/pypy/changeset/378aaa60e293/

Log:	Fix syntax error.

diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -457,7 +457,6 @@
                 {"mode": "w+", "buffering": 2},
                 {"mode": "w+b", "buffering": 0},
             ]:
-            print kwargs
             f = _io.open(self.tmpfile, **kwargs)
             f.close()
             raises(ValueError, f.flush)

From noreply at buildbot.pypy.org  Thu Jul  2 01:54:49 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu,  2 Jul 2015 01:54:49 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20150701235449.C32761C146D@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78397:2ac10a3a0884
Date: 2015-07-02 01:54 +0200
http://bitbucket.org/pypy/pypy/changeset/2ac10a3a0884/

Log:	hg merge default

diff --git a/lib-python/2.7/test/test_urllib2.py b/lib-python/2.7/test/test_urllib2.py
--- a/lib-python/2.7/test/test_urllib2.py
+++ b/lib-python/2.7/test/test_urllib2.py
@@ -291,6 +291,7 @@
         self.req_headers = []
         self.data = None
         self.raise_on_endheaders = False
+        self.sock = None
         self._tunnel_headers = {}
 
     def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
diff --git a/lib-python/2.7/urllib2.py b/lib-python/2.7/urllib2.py
--- a/lib-python/2.7/urllib2.py
+++ b/lib-python/2.7/urllib2.py
@@ -1200,6 +1200,12 @@
                 r = h.getresponse(buffering=True)
             except TypeError: # buffering kw not supported
                 r = h.getresponse()
+            # If the server does not send us a 'Connection: close' header,
+            # HTTPConnection assumes the socket should be left open. Manually
+            # mark the socket to be closed when this response object goes away.
+            if h.sock:
+                h.sock.close()
+                h.sock = None
 
         # Pick apart the HTTPResponse object to get the addinfourl
         # object initialized properly.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -135,7 +135,7 @@
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
 is not reliable in PyPy (nor Jython nor IronPython).  It also means that
-weak references may stay alive for a bit longer than expected.  This
+**weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
 suddenly they will really be dead, raising a ``ReferenceError`` on the
@@ -143,6 +143,24 @@
 ``ReferenceError`` at any place that uses them.  (Or, better yet, don't use
 ``weakref.proxy()`` at all; use ``weakref.ref()``.)
 
+Note a detail in the `documentation for weakref callbacks`__:
+
+    If callback is provided and not None, *and the returned weakref
+    object is still alive,* the callback will be called when the object
+    is about to be finalized.
+
+There are cases where, due to CPython's refcount semantics, a weakref
+dies immediately before or after the objects it points to (typically
+with some circular reference).  If it happens to die just after, then
+the callback will be invoked.  In a similar case in PyPy, both the
+object and the weakref will be considered as dead at the same time,
+and the callback will not be invoked.  (Issue `#2030`__)
+
+.. __: https://docs.python.org/2/library/weakref.html
+.. __: https://bitbucket.org/pypy/pypy/issue/2030/
+
+---------------------------------
+
 There are a few extra implications from the difference in the GC.  Most
 notably, if an object has a ``__del__``, the ``__del__`` is never called more
 than once in PyPy; but CPython will call the same ``__del__`` several times
@@ -321,9 +339,8 @@
 Miscellaneous
 -------------
 
-* Hash randomization (``-R``) is ignored in PyPy.  As documented in
-  http://bugs.python.org/issue14621, some of us believe it has no
-  purpose in CPython either.
+* Hash randomization (``-R``) `is ignored in PyPy`_.  In CPython
+  before 3.4 it has `little point`_.
 
 * You can't store non-string keys in type objects.  For example::
 
@@ -338,7 +355,8 @@
   for about 1400 calls.
 
 * since the implementation of dictionary is different, the exact number
-  which ``__hash__`` and ``__eq__`` are called is different. Since CPython
+  of times that ``__hash__`` and ``__eq__`` are called is different. 
+  Since CPython
   does not give any specific guarantees either, don't rely on it.
 
 * assignment to ``__class__`` is limited to the cases where it
@@ -395,3 +413,12 @@
   interactive mode. In a released version, this behaviour is suppressed, but
   setting the environment variable PYPY_IRC_TOPIC will bring it back. Note that
   downstream package providers have been known to totally disable this feature.
+
+* PyPy's readline module was rewritten from scratch: it is not GNU's
+  readline.  It should be mostly compatible, and it adds multiline
+  support (see ``multiline_input()``).  On the other hand,
+  ``parse_and_bind()`` calls are ignored (issue `#2072`_).
+
+.. _`is ignored in PyPy`: http://bugs.python.org/issue14621
+.. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
+.. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -225,9 +225,13 @@
         if (isinstance(w_cdata, cdataobj.W_CDataNewOwning) or
             isinstance(w_cdata, cdataobj.W_CDataPtrToStructOrUnion)):
             if i != 0:
-                space = self.space
-                raise oefmt(space.w_IndexError,
+                raise oefmt(self.space.w_IndexError,
                             "cdata '%s' can only be indexed by 0", self.name)
+        else:
+            if not w_cdata.unsafe_escaping_ptr():
+                raise oefmt(self.space.w_RuntimeError,
+                            "cannot dereference null pointer from cdata '%s'",
+                            self.name)
         return self
 
     def _check_slice_index(self, w_cdata, start, stop):
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -175,6 +175,8 @@
                     return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
                 if is_getattr and attr == '__dict__':
                     return self.full_dict_copy()
+                if is_getattr and attr == '__name__':
+                    return self.descr_repr()
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2099,8 +2099,7 @@
     p = cast(BVoidP, 123456)
     py.test.raises(TypeError, "p[0]")
     p = cast(BVoidP, 0)
-    if 'PY_DOT_PY' in globals(): py.test.skip("NULL crashes early on py.py")
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises((TypeError, RuntimeError), "p[0]")
 
 def test_iter():
     BInt = new_primitive_type("int")
@@ -3333,6 +3332,15 @@
     check(4 | 8,  "CHB", "GTB")
     check(4 | 16, "CHB", "ROB")
 
+def test_dereference_null_ptr():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    p = cast(BIntPtr, 0)
+    py.test.raises(RuntimeError, "p[0]")
+    py.test.raises(RuntimeError, "p[0] = 42")
+    py.test.raises(RuntimeError, "p[42]")
+    py.test.raises(RuntimeError, "p[42] = -1")
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "1.1.2"
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1011,3 +1011,4 @@
         assert MYFOO == 42
         assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
+        assert lib.__name__ == repr(lib)
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -457,6 +457,8 @@
                 {"mode": "w+", "buffering": 2},
                 {"mode": "w+b", "buffering": 0},
             ]:
+            if "b" not in kwargs["mode"]:
+                kwargs["encoding"] = "ascii"
             f = _io.open(self.tmpfile, **kwargs)
             f.close()
             raises(ValueError, f.flush)
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -27,8 +27,10 @@
     callback_ptr = global_counter.get(userdata.addarg)
     w_callable = callback_ptr.w_callable
     argtypes = callback_ptr.argtypes
+    must_leave = False
     space = callback_ptr.space
     try:
+        must_leave = space.threadlocals.try_enter_thread(space)
         args_w = [None] * len(argtypes)
         for i in range(len(argtypes)):
             argtype = argtypes[i]
@@ -50,6 +52,8 @@
             resshape = letter2tp(space, callback_ptr.result)
             for i in range(resshape.size):
                 ll_res[i] = '\x00'
+    if must_leave:
+        space.threadlocals.leave_thread(space)
 
 class W_CallbackPtr(W_DataInstance):
 
@@ -75,6 +79,14 @@
         if tracker.DO_TRACING:
             addr = rffi.cast(lltype.Signed, self.ll_callback.ll_closure)
             tracker.trace_allocation(addr, self)
+        #
+        # We must setup the GIL here, in case the callback is invoked in
+        # some other non-Pythonic thread.  This is the same as ctypes on
+        # CPython (but only when creating a callback; on CPython it occurs
+        # as soon as we import _ctypes)
+        if space.config.translation.thread:
+            from pypy.module.thread.os_thread import setup_threads
+            setup_threads(space)
 
     def free(self):
         if tracker.DO_TRACING:
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -141,7 +141,7 @@
     def __init__(self, ctx, protos):
         self.protos = protos
         self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
-        NPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+        NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
         # can be used to create both types of sockets
@@ -156,7 +156,7 @@
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
-        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if npn and npn.protos:
             data_ptr[0] = npn.buf
             len_ptr[0] = rffi.cast(rffi.UINT, len(npn.protos))
@@ -168,7 +168,7 @@
 
     @staticmethod
     def selectNPN_cb(s, out_ptr, outlen_ptr, server, server_len, args):
-        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if npn and npn.protos:
             client = npn.buf
             client_len = len(npn.protos)
@@ -187,7 +187,7 @@
     def __init__(self, ctx, protos):
         self.protos = protos
         self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
-        ALPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+        ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
             if libssl_SSL_CTX_set_alpn_protos(
@@ -202,7 +202,7 @@
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
-        alpn = ALPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        alpn = ALPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if alpn and alpn.protos:
             server = alpn.buf
             server_len = len(alpn.protos)
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -26,7 +26,7 @@
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['vmprof.h', 'trampoline.h'],
-    separate_module_files = [SRC.join('trampoline.asmgcc.s')],
+    separate_module_files = [SRC.join('trampoline.vmprof.s')],
     libraries = ['dl'],
     
     post_include_bits=["""
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
deleted file mode 100644
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ /dev/null
@@ -1,16 +0,0 @@
-// NOTE: you need to use TABs, not spaces!
-        
-	.text
-	.p2align 4,,-1
-	.globl	pypy_execute_frame_trampoline
-	.type	pypy_execute_frame_trampoline, @function
-pypy_execute_frame_trampoline:
-	.cfi_startproc
-	pushq	%rcx
-	.cfi_def_cfa_offset 16
-	call pypy_pyframe_execute_frame at PLT
-	popq	%rcx
-	.cfi_def_cfa_offset 8
-	ret
-	.cfi_endproc
-	.size	pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline
diff --git a/pypy/module/_vmprof/src/trampoline.vmprof.s b/pypy/module/_vmprof/src/trampoline.vmprof.s
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/trampoline.vmprof.s
@@ -0,0 +1,15 @@
+// NOTE: you need to use TABs, not spaces!
+        
+	.text
+	.globl	pypy_execute_frame_trampoline
+	.type	pypy_execute_frame_trampoline, @function
+pypy_execute_frame_trampoline:
+	.cfi_startproc
+	pushq	%rcx
+	.cfi_def_cfa_offset 16
+	call pypy_pyframe_execute_frame at PLT
+	popq	%rcx
+	.cfi_def_cfa_offset 8
+	ret
+	.cfi_endproc
+	.size	pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -305,7 +305,6 @@
 
 static int remove_sigprof_timer(void) {
     static struct itimerval timer;
-    last_period_usec = 0;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = 0;
     timer.it_value.tv_sec = 0;
@@ -317,11 +316,15 @@
 }
 
 static void atfork_disable_timer(void) {
-    remove_sigprof_timer();
+    if (last_period_usec) {
+        remove_sigprof_timer();
+    }
 }
 
 static void atfork_enable_timer(void) {
-    install_sigprof_timer(last_period_usec);
+    if (last_period_usec) {
+        install_sigprof_timer(last_period_usec);
+    }
 }
 
 static int install_pthread_atfork_hooks(void) {
@@ -412,6 +415,7 @@
     if (remove_sigprof_timer() == -1) {
 		return -1;
 	}
+    last_period_usec = 0;
     if (remove_sigprof_handler() == -1) {
 		return -1;
 	}
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -338,9 +338,11 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    return None
+            else:
+                # this only runs if fromlist_w != ['*']
+                for w_name in fromlist_w:
+                    if try_getattr(space, w_mod, w_name) is None:
+                        return None
         return w_mod
     return first
 
@@ -378,10 +380,12 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    load_part(space, w_path, prefix, space.str0_w(w_name),
-                              w_mod, tentative=1)
+            else:
+                # this only runs if fromlist_w != ['*']
+                for w_name in fromlist_w:
+                    if try_getattr(space, w_mod, w_name) is None:
+                        load_part(space, w_path, prefix, space.str0_w(w_name),
+                                  w_mod, tentative=1)
         return w_mod
     else:
         return first
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1373,6 +1373,53 @@
         finally:
             sys.path_hooks.pop()
 
+    def test_meta_path_import_error_1(self):
+        # as far as I can tell, the problem is that in CPython, if you
+        # use an import hook that doesn't update sys.modules, then the
+        # import succeeds; but at the same time, you can have the same
+        # result without an import hook (see test_del_from_sys_modules)
+        # and then the import fails.  This looks like even more mess
+        # to replicate, so we ignore it until someone really hits this
+        # case...
+        skip("looks like an inconsistency in CPython")
+
+        class ImportHook(object):
+            def find_module(self, fullname, path=None):
+                assert not fullname.endswith('*')
+                if fullname == 'meta_path_pseudo_module':
+                    return self
+            def load_module(self, fullname):
+                assert fullname == 'meta_path_pseudo_module'
+                # we "forget" to update sys.modules
+                return new.module('meta_path_pseudo_module')
+
+        import sys, new
+        sys.meta_path.append(ImportHook())
+        try:
+            import meta_path_pseudo_module
+        finally:
+            sys.meta_path.pop()
+
+    def test_meta_path_import_star_2(self):
+        class ImportHook(object):
+            def find_module(self, fullname, path=None):
+                if fullname.startswith('meta_path_2_pseudo_module'):
+                    return self
+            def load_module(self, fullname):
+                assert fullname == 'meta_path_2_pseudo_module'
+                m = new.module('meta_path_2_pseudo_module')
+                m.__path__ = ['/some/random/dir']
+                sys.modules['meta_path_2_pseudo_module'] = m
+                return m
+
+        import sys, new
+        sys.meta_path.append(ImportHook())
+        try:
+            exec "from meta_path_2_pseudo_module import *" in {}
+        finally:
+            sys.meta_path.pop()
+
+
 class AppTestPyPyExtension(object):
     spaceconfig = dict(usemodules=['imp', 'zipimport', '__pypy__'])
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -645,7 +645,7 @@
     next_item = _new_next('item')
 
 
-def create_iterator_classes(dictimpl, override_next_item=None):
+def create_iterator_classes(dictimpl):
     if not hasattr(dictimpl, 'wrapkey'):
         wrapkey = lambda space, key: key
     else:
@@ -688,15 +688,12 @@
             self.iterator = strategy.getiteritems(impl)
             BaseIteratorImplementation.__init__(self, space, strategy, impl)
 
-        if override_next_item is not None:
-            next_item_entry = override_next_item
-        else:
-            def next_item_entry(self):
-                for key, value in self.iterator:
-                    return (wrapkey(self.space, key),
-                            wrapvalue(self.space, value))
-                else:
-                    return None, None
+        def next_item_entry(self):
+            for key, value in self.iterator:
+                return (wrapkey(self.space, key),
+                        wrapvalue(self.space, value))
+            else:
+                return None, None
 
     class IterClassReversed(BaseKeyIterator):
         def __init__(self, space, strategy, impl):
@@ -729,22 +726,7 @@
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
         # the logic is to call prepare_dict_update() after the first setitem():
         # it gives the w_updatedict a chance to switch its strategy.
-        if override_next_item is not None:
-            # this is very similar to the general version, but the difference
-            # is that it is specialized to call a specific next_item()
-            iteritems = IterClassItems(self.space, self, w_dict)
-            w_key, w_value = iteritems.next_item()
-            if w_key is None:
-                return
-            w_updatedict.setitem(w_key, w_value)
-            w_updatedict.strategy.prepare_update(w_updatedict,
-                                                 w_dict.length() - 1)
-            while True:
-                w_key, w_value = iteritems.next_item()
-                if w_key is None:
-                    return
-                w_updatedict.setitem(w_key, w_value)
-        else:
+        if 1:     # (preserve indentation)
             iteritems = self.getiteritems(w_dict)
             if not same_strategy(self, w_updatedict):
                 # Different strategy.  Try to copy one item of w_dict
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -167,19 +167,26 @@
         return iter(self.unerase(w_dict.dstorage)[1])
 
     def getiteritems(self, w_dict):
-        keys = self.unerase(w_dict.dstorage)[0]
-        return iter(range(len(keys)))
+        return Zip(*self.unerase(w_dict.dstorage))
 
     wrapkey = _wrapkey
 
 
-def next_item(self):
-    strategy = self.strategy
-    assert isinstance(strategy, KwargsDictStrategy)
-    for i in self.iterator:
-        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
-        return _wrapkey(self.space, keys[i]), values_w[i]
-    else:
-        return None, None
+class Zip(object):
+    def __init__(self, list1, list2):
+        assert len(list1) == len(list2)
+        self.list1 = list1
+        self.list2 = list2
+        self.i = 0
 
-create_iterator_classes(KwargsDictStrategy, override_next_item=next_item)
+    def __iter__(self):
+        return self
+
+    def next(self):
+        i = self.i
+        if i >= len(self.list1):
+            raise StopIteration
+        self.i = i + 1
+        return (self.list1[i], self.list2[i])
+
+create_iterator_classes(KwargsDictStrategy)
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -193,9 +193,9 @@
 
     def switch_to_object_strategy(self):
         list_w = self.getitems()
-        self.strategy = self.space.fromcache(ObjectListStrategy)
-        # XXX this is quite indirect
-        self.init_from_list_w(list_w)
+        object_strategy = self.space.fromcache(ObjectListStrategy)
+        self.strategy = object_strategy
+        object_strategy.init_from_list_w(self, list_w)
 
     def _temporarily_as_objects(self):
         if self.strategy is self.space.fromcache(ObjectListStrategy):
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -160,6 +160,14 @@
         assert a == 3
         assert "KwargsDictStrategy" in self.get_strategy(d)
 
+    def test_iteritems_bug(self):
+        def f(**args):
+            return args
+
+        d = f(a=2, b=3, c=4)
+        for key, value in d.items():
+            None in d
+
     def test_unicode(self):
         """
         def f(**kwargs):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -187,7 +187,12 @@
         [i0]
         jump(i0)
         """
-        self.optimize_loop(ops, expected)
+        short = """
+        [i2]
+        p3 = cast_int_to_ptr(i2)
+        jump(i2)
+        """
+        self.optimize_loop(ops, expected, expected_short=short)
 
     def test_reverse_of_cast_2(self):
         ops = """
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -439,8 +439,8 @@
 
             mk.definition('PYTHON', get_recent_cpython_executable())
 
-            mk.definition('GCMAPFILES', '$(subst .asmgcc.s,.gcmap,$(subst .c,.gcmap,$(SOURCES)))')
-            mk.definition('OBJECTS1', '$(subst .asmgcc.s,.o,$(subst .c,.o,$(SOURCES)))')
+            mk.definition('GCMAPFILES', '$(subst .vmprof.s,.gcmap,$(subst .c,.gcmap,$(SOURCES)))')
+            mk.definition('OBJECTS1', '$(subst .vmprof.s,.o,$(subst .c,.o,$(SOURCES)))')
             mk.definition('OBJECTS', '$(OBJECTS1) gcmaptable.s')
 
             # the CFLAGS passed to gcc when invoked to assembler the .s file
@@ -462,12 +462,12 @@
                 'rm $*.s $*.lbl.s'])
 
             # this is for manually written assembly files which needs to be parsed by asmgcc
-            mk.rule('%.o %.gcmap', '%.asmgcc.s', [
+            mk.rule('%.o %.gcmap', '%.vmprof.s', [
                 '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py '
-                    '-t $*.asmgcc.s > $*.gctmp',
-                '$(CC) -o $*.o -c $*.asmgcc.lbl.s',
+                    '-t $*.vmprof.s > $*.gctmp',
+                '$(CC) -o $*.o -c $*.vmprof.lbl.s',
                 'mv $*.gctmp $*.gcmap',
-                'rm $*.asmgcc.lbl.s'])
+                'rm $*.vmprof.lbl.s'])
             
             # the rule to compute gcmaptable.s
             mk.rule('gcmaptable.s', '$(GCMAPFILES)',

From noreply at buildbot.pypy.org  Thu Jul  2 09:07:49 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  2 Jul 2015 09:07:49 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: preventing signext from int16,
 int8 to greater or vice versa, the first direction was ignored up to now
Message-ID: <20150702070749.CF0811C11D3@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78398:8022a49a8008
Date: 2015-07-02 09:07 +0200
http://bitbucket.org/pypy/pypy/changeset/8022a49a8008/

Log:	preventing signext from int16,int8 to greater or vice versa, the
	first direction was ignored up to now

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -7,7 +7,7 @@
 class TestMicroNumPy(BaseTestPyPyC):
 
     arith_comb = [('+','float','float', 4*3427,   3427, 1.0,3.0),
-             ('+','float','int',   9*7834,   7843, 4.0,5.0),
+             ('+','float','int',   9*7844,   7843, 4.0,5.0),
              ('+','int','float',   8*2571,   2571, 9.0,-1.0),
              ('+','float','int',   -18*2653,   2653, 4.0,-22.0),
              ('+','int','int',     -1*1499,   1499, 24.0,-25.0),
@@ -21,7 +21,7 @@
              ('|','int','int',     0,        1500, 0,0),
             ]
     type_permuated = []
-    types = { 'int': ['int32','int64'],
+    types = { 'int': ['int32','int64','int8','int16'],
               'float': ['float32', 'float64']
             }
     for arith in arith_comb:
@@ -63,8 +63,8 @@
         ('all','int', 1, 6757, 1),
     ]
     type_permuated = []
-    types = { 'int': ['int64'],
-              'float': ['float64']
+    types = { 'int': ['int8','int16','int32','int64'],
+              'float': ['float32','float64']
             }
     for arith in arith_comb:
         t1 = arith[1]
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -804,8 +804,9 @@
         self.costmodel = costmodel
 
     def _prevent_signext(self, outsize, insize):
-        if outsize < 4 and insize != outsize:
-            raise NotAProfitableLoop
+        if insize != outsize:
+            if outsize < 4 or insize < 4:
+                raise NotAProfitableLoop
 
     def as_vector_operation(self, pack, preproc_renamer):
         assert pack.opcount() > 1

From noreply at buildbot.pypy.org  Thu Jul  2 10:00:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 10:00:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: hg backout e83ef97376fe: wrong fix
Message-ID: <20150702080011.3F49E1C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78399:3bb6c8ee92a4
Date: 2015-07-02 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/3bb6c8ee92a4/

Log:	hg backout e83ef97376fe: wrong fix

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -349,11 +349,9 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            else:
-                # this only runs if fromlist_w != ['*']
-                for w_name in fromlist_w:
-                    if try_getattr(space, w_mod, w_name) is None:
-                        return None
+            for w_name in fromlist_w:
+                if try_getattr(space, w_mod, w_name) is None:
+                    return None
         return w_mod
     return first
 
@@ -391,12 +389,10 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            else:
-                # this only runs if fromlist_w != ['*']
-                for w_name in fromlist_w:
-                    if try_getattr(space, w_mod, w_name) is None:
-                        load_part(space, w_path, prefix, space.str0_w(w_name),
-                                  w_mod, tentative=1)
+            for w_name in fromlist_w:
+                if try_getattr(space, w_mod, w_name) is None:
+                    load_part(space, w_path, prefix, space.str0_w(w_name),
+                              w_mod, tentative=1)
         return w_mod
     else:
         return first

From noreply at buildbot.pypy.org  Thu Jul  2 10:00:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 10:00:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: A simpler fix for "from x import *",
 with more tests showing the problem
Message-ID: <20150702080012.720201C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78400:79617073c0fe
Date: 2015-07-02 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/79617073c0fe/

Log:	A simpler fix for "from x import *", with more tests showing the
	problem of the previous one.

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -349,6 +349,11 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
+                else:
+                    fromlist_w = []
+                    # "from x import *" with x already imported and no x.__all__
+                    # always succeeds without doing more imports.  It will
+                    # just copy everything from x.__dict__ as it is now.
             for w_name in fromlist_w:
                 if try_getattr(space, w_mod, w_name) is None:
                     return None
@@ -389,6 +394,8 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
+                else:
+                    fromlist_w = []
             for w_name in fromlist_w:
                 if try_getattr(space, w_mod, w_name) is None:
                     load_part(space, w_path, prefix, space.str0_w(w_name),
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -66,6 +66,14 @@
              b          = "insubpackage = 1",
              )
     setuppkg("pkg.pkg2", a='', b='')
+    setuppkg("pkg.withall",
+             __init__  = "__all__ = ['foobar']",
+             foobar    = "found = 123")
+    setuppkg("pkg.withoutall",
+             __init__  = "",
+             foobar    = "found = 123")
+    setuppkg("pkg.bogusall",
+             __init__  = "__all__ = 42")
     setuppkg("pkg_r", inpkg = "import x.y")
     setuppkg("pkg_r.x")
     setuppkg("x", y='')
@@ -677,6 +685,32 @@
         import imp
         raises(ValueError, imp.load_module, "", "", "", [1, 2, 3, 4])
 
+    def test_import_star_finds_submodules_with___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withall import *" in d
+            assert d["foobar"].found == 123
+
+    def test_import_star_does_not_find_submodules_without___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withoutall import *" in d
+            assert "foobar" not in d
+        import pkg.withoutall.foobar     # <- import it here only
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withoutall import *" in d
+            assert d["foobar"].found == 123
+
+    def test_import_star_with_bogus___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            try:
+                exec "from pkg.bogusall import *" in {}
+            except TypeError:
+                pass    # 'int' object does not support indexing
+            else:
+                raise AssertionError("should have failed")
+
 
 class TestAbi:
     def test_abi_tag(self):

From noreply at buildbot.pypy.org  Thu Jul  2 10:35:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 10:35:42 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: hg merge default
Message-ID: <20150702083542.0BE1A1C1067@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78401:cfc5cf6de7d8
Date: 2015-07-02 10:01 +0200
http://bitbucket.org/pypy/pypy/changeset/cfc5cf6de7d8/

Log:	hg merge default

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -349,11 +349,14 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            else:
-                # this only runs if fromlist_w != ['*']
-                for w_name in fromlist_w:
-                    if try_getattr(space, w_mod, w_name) is None:
-                        return None
+                else:
+                    fromlist_w = []
+                    # "from x import *" with x already imported and no x.__all__
+                    # always succeeds without doing more imports.  It will
+                    # just copy everything from x.__dict__ as it is now.
+            for w_name in fromlist_w:
+                if try_getattr(space, w_mod, w_name) is None:
+                    return None
         return w_mod
     return first
 
@@ -391,12 +394,12 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            else:
-                # this only runs if fromlist_w != ['*']
-                for w_name in fromlist_w:
-                    if try_getattr(space, w_mod, w_name) is None:
-                        load_part(space, w_path, prefix, space.str0_w(w_name),
-                                  w_mod, tentative=1)
+                else:
+                    fromlist_w = []
+            for w_name in fromlist_w:
+                if try_getattr(space, w_mod, w_name) is None:
+                    load_part(space, w_path, prefix, space.str0_w(w_name),
+                              w_mod, tentative=1)
         return w_mod
     else:
         return first
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -66,6 +66,14 @@
              b          = "insubpackage = 1",
              )
     setuppkg("pkg.pkg2", a='', b='')
+    setuppkg("pkg.withall",
+             __init__  = "__all__ = ['foobar']",
+             foobar    = "found = 123")
+    setuppkg("pkg.withoutall",
+             __init__  = "",
+             foobar    = "found = 123")
+    setuppkg("pkg.bogusall",
+             __init__  = "__all__ = 42")
     setuppkg("pkg_r", inpkg = "import x.y")
     setuppkg("pkg_r.x")
     setuppkg("x", y='')
@@ -677,6 +685,32 @@
         import imp
         raises(ValueError, imp.load_module, "", "", "", [1, 2, 3, 4])
 
+    def test_import_star_finds_submodules_with___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withall import *" in d
+            assert d["foobar"].found == 123
+
+    def test_import_star_does_not_find_submodules_without___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withoutall import *" in d
+            assert "foobar" not in d
+        import pkg.withoutall.foobar     # <- import it here only
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withoutall import *" in d
+            assert d["foobar"].found == 123
+
+    def test_import_star_with_bogus___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            try:
+                exec "from pkg.bogusall import *" in {}
+            except TypeError:
+                pass    # 'int' object does not support indexing
+            else:
+                raise AssertionError("should have failed")
+
 
 class TestAbi:
     def test_abi_tag(self):

From noreply at buildbot.pypy.org  Thu Jul  2 10:35:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 10:35:43 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: lst = [0]; lst += [1.2]
Message-ID: <20150702083543.41B6E1C1067@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78402:da0023834a68
Date: 2015-07-02 10:06 +0200
http://bitbucket.org/pypy/pypy/changeset/da0023834a68/

Log:	lst = [0]; lst += [1.2]

	lst = [0]; lst += [1.2, 3]

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1658,6 +1658,11 @@
             assert other is not None
             l += other
             return
+        if (w_other.strategy is self.space.fromcache(FloatListStrategy) or
+            w_other.strategy is self.space.fromcache(IntOrFloatListStrategy)):
+            if self.switch_to_int_or_float_strategy(w_list):
+                w_list.extend(w_other)
+                return
         return self._base_extend_from_list(w_list, w_other)
 
 
@@ -1680,20 +1685,23 @@
         return [longlong2float.encode_int32_into_longlong_nan(intval)
                 for intval in l]
 
+    def switch_to_int_or_float_strategy(self, w_list):
+        try:
+            generalized_list = self.int_2_float_or_int(w_list)
+        except ValueError:
+            return False
+        strategy = self.space.fromcache(IntOrFloatListStrategy)
+        w_list.strategy = strategy
+        w_list.lstorage = strategy.erase(generalized_list)
+        return True
+
     def switch_to_next_strategy(self, w_list, w_sample_item):
         if type(w_sample_item) is W_FloatObject:
-            try:
-                generalized_list = self.int_2_float_or_int(w_list)
-            except ValueError:
-                pass
-            else:
+            if self.switch_to_int_or_float_strategy(w_list):
                 # yes, we can switch to IntOrFloatListStrategy
                 # (ignore here the extremely unlikely case where
                 # w_sample_item is just the wrong nonstandard NaN float;
                 # it will caught later and yet another switch will occur)
-                strategy = self.space.fromcache(IntOrFloatListStrategy)
-                w_list.strategy = strategy
-                w_list.lstorage = strategy.erase(generalized_list)
                 return
         # no, fall back to ObjectListStrategy
         w_list.switch_to_object_strategy()
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -888,9 +888,27 @@
         assert space.unwrap(w_l1) == [0, 1.2, 3.4]
 
     def test_int_or_float_extend_mixed_3(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0)])
+        w_l2 = W_ListObject(space, [space.wrap(3.4)])
+        assert isinstance(w_l1.strategy, IntegerListStrategy)
+        assert isinstance(w_l2.strategy, FloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [0, 3.4]
+
+    def test_int_or_float_extend_mixed_4(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0)])
+        w_l2 = W_ListObject(space, [space.wrap(3.4), space.wrap(-2)])
+        assert isinstance(w_l1.strategy, IntegerListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [0, 3.4, -2]
+
+    def test_int_or_float_extend_mixed_5(self):
         py.test.skip("XXX not implemented")
-        # lst = [0]; lst += [1.2]
-        # lst = [0]; lst += [1.2, 3]
         # lst = [1.2]; lst += [0]
         # lst = [1.2]; lst += [0, 3.4]
 

From noreply at buildbot.pypy.org  Thu Jul  2 10:35:44 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 10:35:44 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: lst = [1.2]; lst += [0]
Message-ID: <20150702083544.73A2C1C1067@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78403:cadc17f46535
Date: 2015-07-02 10:12 +0200
http://bitbucket.org/pypy/pypy/changeset/cadc17f46535/

Log:	lst = [1.2]; lst += [0]

	lst = [1.2]; lst += [0, 3.4]

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1738,6 +1738,21 @@
     def getitems_float(self, w_list):
         return self.unerase(w_list.lstorage)
 
+
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if (w_other.strategy is self.space.fromcache(IntegerListStrategy) or
+            w_other.strategy is self.space.fromcache(IntOrFloatListStrategy)):
+            # xxx a case that we don't optimize: [3.4].extend([9999999999999])
+            # will cause a switch to int-or-float, followed by another
+            # switch to object
+            if self.switch_to_int_or_float_strategy(w_list):
+                w_list.extend(w_other)
+                return
+        return self._base_extend_from_list(w_list, w_other)
+
+
     def _safe_find(self, w_list, obj, start, stop):
         from rpython.rlib.rfloat import isnan
         #
@@ -1766,21 +1781,24 @@
                 longlong2float.float2longlong(floatval))
         return generalized_list
 
+    def switch_to_int_or_float_strategy(self, w_list):
+        # xxx we should be able to use the same lstorage, but
+        # there is a typing issue (float vs longlong)...
+        try:
+            generalized_list = self.float_2_float_or_int(w_list)
+        except ValueError:
+            return False
+        strategy = self.space.fromcache(IntOrFloatListStrategy)
+        w_list.strategy = strategy
+        w_list.lstorage = strategy.erase(generalized_list)
+        return True
+
     def switch_to_next_strategy(self, w_list, w_sample_item):
         if type(w_sample_item) is W_IntObject:
             sample_intval = self.space.int_w(w_sample_item)
             if longlong2float.can_encode_int32(sample_intval):
-                # xxx we should be able to use the same lstorage, but
-                # there is a typing issue (float vs longlong)...
-                try:
-                    generalized_list = self.float_2_float_or_int(w_list)
-                except ValueError:
-                    pass
-                else:
+                if self.switch_to_int_or_float_strategy(w_list):
                     # yes, we can switch to IntOrFloatListStrategy
-                    strategy = self.space.fromcache(IntOrFloatListStrategy)
-                    w_list.strategy = strategy
-                    w_list.lstorage = strategy.erase(generalized_list)
                     return
         # no, fall back to ObjectListStrategy
         w_list.switch_to_object_strategy()
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -908,9 +908,37 @@
         assert space.unwrap(w_l1) == [0, 3.4, -2]
 
     def test_int_or_float_extend_mixed_5(self):
-        py.test.skip("XXX not implemented")
-        # lst = [1.2]; lst += [0]
-        # lst = [1.2]; lst += [0, 3.4]
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(-2.5)])
+        w_l2 = W_ListObject(space, [space.wrap(42)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [-2.5, 42]
+
+    def test_int_or_float_extend_mixed_5_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        ovf1 = 2 ** 35
+        w_l1 = W_ListObject(space, [space.wrap(-2.5)])
+        w_l2 = W_ListObject(space, [space.wrap(ovf1)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, ObjectListStrategy)
+        assert space.unwrap(w_l1) == [-2.5, ovf1]
+
+    def test_int_or_float_extend_mixed_6(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(-2.5)])
+        w_l2 = W_ListObject(space, [space.wrap(3.4), space.wrap(-2)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [-2.5, 3.4, -2]
 
     def test_int_or_float_setslice(self):
         space = self.space

From noreply at buildbot.pypy.org  Thu Jul  2 10:35:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 10:35:45 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: setslice() support and
	clean-ups
Message-ID: <20150702083545.8CE691C1067@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78404:ecfee265b225
Date: 2015-07-02 10:35 +0200
http://bitbucket.org/pypy/pypy/changeset/ecfee265b225/

Log:	setslice() support and clean-ups

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1673,11 +1673,17 @@
             storage = self.erase(w_other.getitems_int())
             w_other = W_ListObject.from_storage_and_strategy(
                     self.space, storage, self)
+        if (w_other.strategy is self.space.fromcache(FloatListStrategy) or
+            w_other.strategy is self.space.fromcache(IntOrFloatListStrategy)):
+            if self.switch_to_int_or_float_strategy(w_list):
+                w_list.setslice(start, step, slicelength, w_other)
+                return
         return self._base_setslice(w_list, start, step, slicelength, w_other)
 
 
-    def int_2_float_or_int(self, w_list):
-        l = self.unerase(w_list.lstorage)
+    @staticmethod
+    def int_2_float_or_int(w_list):
+        l = IntegerListStrategy.unerase(w_list.lstorage)
         if not longlong2float.CAN_ALWAYS_ENCODE_INT32:
             for intval in l:
                 if not longlong2float.can_encode_int32(intval):
@@ -1753,6 +1759,17 @@
         return self._base_extend_from_list(w_list, w_other)
 
 
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if (w_other.strategy is self.space.fromcache(IntegerListStrategy) or
+            w_other.strategy is self.space.fromcache(IntOrFloatListStrategy)):
+            if self.switch_to_int_or_float_strategy(w_list):
+                w_list.setslice(start, step, slicelength, w_other)
+                return
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+
     def _safe_find(self, w_list, obj, start, stop):
         from rpython.rlib.rfloat import isnan
         #
@@ -1771,8 +1788,9 @@
                     return i
         raise ValueError
 
-    def float_2_float_or_int(self, w_list):
-        l = self.unerase(w_list.lstorage)
+    @staticmethod
+    def float_2_float_or_int(w_list):
+        l = FloatListStrategy.unerase(w_list.lstorage)
         generalized_list = []
         for floatval in l:
             if not longlong2float.can_encode_float(floatval):
@@ -1862,20 +1880,43 @@
     def _extend_from_list(self, w_list, w_other):
         if w_other.strategy is self.space.fromcache(IntegerListStrategy):
             try:
-                longlong_list = w_other.strategy.int_2_float_or_int(w_other)
+                longlong_list = IntegerListStrategy.int_2_float_or_int(w_other)
             except ValueError:
                 pass
             else:
                 return self._extend_longlong(w_list, longlong_list)
         if w_other.strategy is self.space.fromcache(FloatListStrategy):
             try:
-                longlong_list = w_other.strategy.float_2_float_or_int(w_other)
+                longlong_list = FloatListStrategy.float_2_float_or_int(w_other)
             except ValueError:
                 pass
             else:
                 return self._extend_longlong(w_list, longlong_list)
         return self._base_extend_from_list(w_list, w_other)
 
+    _base_setslice = setslice
+
+    def _temporary_longlong_list(self, longlong_list):
+        storage = self.erase(longlong_list)
+        return W_ListObject.from_storage_and_strategy(self.space, storage, self)
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(IntegerListStrategy):
+            try:
+                longlong_list = IntegerListStrategy.int_2_float_or_int(w_other)
+            except ValueError:
+                pass
+            else:
+                w_other = self._temporary_longlong_list(longlong_list)
+        elif w_other.strategy is self.space.fromcache(FloatListStrategy):
+            try:
+                longlong_list = FloatListStrategy.float_2_float_or_int(w_other)
+            except ValueError:
+                pass
+            else:
+                w_other = self._temporary_longlong_list(longlong_list)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
 
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -318,7 +318,7 @@
 
         l = W_ListObject(space, [w(1.1), w(2.2), w(3.3)])
         assert isinstance(l.strategy, FloatListStrategy)
-        l.extend(W_ListObject(space, [w(4), w(5), w(6)]))
+        l.extend(W_ListObject(space, [w("abc"), w("def"), w("ghi")]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
     def test_empty_extend_with_any(self):
@@ -950,14 +950,78 @@
         assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
         assert space.unwrap(w_l1) == [3, 4.5, 1.2]
 
-    def test_int_or_float_setslice_mixed(self):
-        py.test.skip("XXX not implemented")
-        # lst = [0]; lst[:] = [1.2]
-        # lst = [0]; lst[:] = [1.2, 3]
-        # lst = [1.2]; lst[:] = [0]
-        # lst = [1.2]; lst[:] = [0, 3.4]
-        # lst = [0, 1.2]; lst[:] = [3]
-        # lst = [0, 1.2]; lst[:] = [3.4]
+    def test_int_or_float_setslice_mixed_1(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(12)])
+        w_l2 = W_ListObject(space, [space.wrap(3.2), space.wrap(4.5)])
+        assert isinstance(w_l1.strategy, IntegerListStrategy)
+        assert isinstance(w_l2.strategy, FloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3.2, 4.5, 12]
+
+    def test_int_or_float_setslice_mixed_2(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(12)])
+        w_l2 = W_ListObject(space, [space.wrap(3.2), space.wrap(45)])
+        assert isinstance(w_l1.strategy, IntegerListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3.2, 45, 12]
+
+    def test_int_or_float_setslice_mixed_3(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0.1), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(32), space.wrap(45)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [32, 45, 1.2]
+
+    def test_int_or_float_setslice_mixed_4(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0.1), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3.2), space.wrap(45)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3.2, 45, 1.2]
+
+    def test_int_or_float_setslice_mixed_5(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(32), space.wrap(45)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [32, 45, 1.2]
+
+    def test_int_or_float_setslice_mixed_5_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        ovf1 = 2 ** 35
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(32), space.wrap(ovf1)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, ObjectListStrategy)
+        assert space.unwrap(w_l1) == [32, ovf1, 1.2]
+
+    def test_int_or_float_setslice_mixed_6(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3.2), space.wrap(4.5)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, FloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3.2, 4.5, 1.2]
 
     def test_int_or_float_sort(self):
         space = self.space

From noreply at buildbot.pypy.org  Thu Jul  2 10:47:54 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  2 Jul 2015 10:47:54 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: product assembler now uses SHUFPD
 instead of SHUFPS and moves two floats instead of one to the accumulator
Message-ID: <20150702084754.AFA371C026F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78405:03f3796197b8
Date: 2015-07-02 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/03f3796197b8/

Log:	product assembler now uses SHUFPD instead of SHUFPS and moves two
	floats instead of one to the accumulator resolving test issues with
	the recent changes

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -515,19 +515,29 @@
 
     def define_prod():
         return """
-        a = |30|
+        a = [1,2,3,4,1,2,3,4]
+        prod(a)
+        """
+
+    def define_prod_zero():
+        return """
+        a = [1,2,3,4,1,2,3,0]
         prod(a)
         """
 
     def test_prod(self):
         result = self.run("prod")
-        expected = 1
-        for i in range(30):
-            expected *= i * 2
-        assert result == expected
+        assert int(result) == 576
         self.check_trace_count(1)
         self.check_vectorized(1, 1)
 
+    def test_prod_zero(self):
+        result = self.run("prod_zero")
+        assert int(result) == 0
+        self.check_trace_count(1)
+        self.check_vectorized(1, 1)
+
+
     def define_max():
         return """
         a = |30|
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -42,7 +42,8 @@
             import _numpypy.multiarray as np
             a = np.array([{a}]*{count}, dtype='{adtype}')
             b = np.array([{b}]*{count}, dtype='{bdtype}')
-            c = a {op} b
+            for i in range(20):
+                c = a {op} b
             return c.sum()
         """.format(op=op, adtype=adtype, bdtype=bdtype, count=count, a=a, b=b)
         exec py.code.Source(source).compile()
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2560,8 +2560,8 @@
         scratchloc = X86_64_XMM_SCRATCH_REG
         self.mov(accumloc, scratchloc)
         # swap the two elements
-        self.mc.SHUFPS_xxi(scratchloc.value, scratchloc.value, 0x01)
-        self.mc.MULPD(accumloc, scratchloc)
+        self.mc.SHUFPD_xxi(scratchloc.value, scratchloc.value, 0x01)
+        self.mc.MULSD(accumloc, scratchloc)
         if accumloc is not targetloc:
             self.mov(accumloc, targetloc)
 
@@ -2756,7 +2756,8 @@
         srcloc, sizeloc = arglocs
         size = sizeloc.value
         if isinstance(srcloc, ConstFloatLoc):
-            self.mov(srcloc, resloc)
+            # they are aligned!
+            self.mc.MOVAPD(resloc, srcloc)
         elif size == 4:
             # the register allocator forces src to be the same as resloc
             # r = (s[0], s[0], r[0], r[0])
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -162,10 +162,12 @@
     @staticmethod
     def by_descr(descr, vec_reg_size):
         _t = INT
+        signed = descr.is_item_signed()
         if descr.is_array_of_floats() or descr.concrete_type == FLOAT:
             _t = FLOAT
+            signed = False
         size = descr.get_item_size_in_bytes()
-        pt = PackType(_t, size, descr.is_item_signed(), vec_reg_size // size)
+        pt = PackType(_t, size, signed, vec_reg_size // size)
         return pt
 
     def __init__(self, type, size, signed, count=-1):
@@ -214,13 +216,13 @@
         return self.count
 
 
+PT_GENERIC = PackType(PackType.UNKNOWN_TYPE, -1, False)
 PT_FLOAT_2 = PackType(FLOAT, 4, False, 2)
 PT_DOUBLE_2 = PackType(FLOAT, 8, False, 2)
-PT_FLOAT_GENERIC = PackType(INT, -1, True)
+PT_FLOAT_GENERIC = PackType(INT, -1, False)
 PT_INT64 = PackType(INT, 8, True)
 PT_INT32_2 = PackType(INT, 4, True, 2)
 PT_INT_GENERIC = PackType(INT, -1, True)
-PT_GENERIC = PackType(PackType.UNKNOWN_TYPE, -1, False)
 
 INT_RES = PT_INT_GENERIC
 FLOAT_RES = PT_FLOAT_GENERIC
@@ -239,8 +241,7 @@
     def check_if_pack_supported(self, pack):
         op0 = pack.operations[0].getoperation()
         if self.input_type is None:
-            # must be a load operation
-            assert op0.is_raw_load()
+            # must be a load/guard op
             return
         insize = self.input_type.getsize()
         if op0.casts_box():
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -136,7 +136,7 @@
         savings = self.savings(loop1)
         assert savings == 2
 
-    @py.test.mark.parametrize("bytes,s", [(1,None),(2,None),(4,0),(8,-1)])
+    @py.test.mark.parametrize("bytes,s", [(1,None),(2,None),(4,0),(8,0)])
     def test_sum_float_to_int(self, bytes, s):
         loop1 = self.parse("""
         f10 = raw_load(p0, i0, descr=double)
@@ -189,8 +189,11 @@
         f106 = cast_int_to_float(i110)
         f107 = cast_int_to_float(i111)
         """)
-        savings = self.savings(loop1)
-        assert savings <= -2
+        try:
+            self.savings(loop1)
+            py.test.fail("must not profitable!")
+        except NotAProfitableLoop:
+            pass
 
 class Test(CostModelBaseTest, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -15,9 +15,9 @@
 from rpython.jit.tool.oparser import parse as opparse
 from rpython.jit.tool.oparser_model import get_model
 
-F64 = PackType('f',8,True,2)
-F32 = PackType('f',4,True,4)
-F32_2 =  PackType('f',4,True,2)
+F64 = PackType('f',8,False,2)
+F32 = PackType('f',4,False,4)
+F32_2 =  PackType('f',4,False,2)
 I64 = PackType('i',8,True,2)
 I32 = PackType('i',4,True,4)
 I32_2 =  PackType('i',4,True,2)
@@ -180,7 +180,7 @@
         f10 = float_add(f0, 73.0)
         f11 = float_add(f1, 73.0)
         """)
-        pack1 = self.pack(loop1, 0, 2, I64, I64)
+        pack1 = self.pack(loop1, 0, 2, F64, F64)
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
         v10[f64|2] = vec_box(2)
@@ -275,10 +275,10 @@
         raw_store(p1, i7, i24, descr=short)
         raw_store(p1, i8, i25, descr=short)
         """)
-        pack1 = self.pack(loop1, 0, 8, None, I64)
-        pack2 = self.pack(loop1, 8, 16, I64, I32_2)
+        pack1 = self.pack(loop1, 0, 8, None, F64)
+        pack2 = self.pack(loop1, 8, 16, F64, I32_2)
         I16_2 = PackType('i',2,True,2)
-        pack3 = self.pack(loop1, 16, 24, I32, I16_2)
+        pack3 = self.pack(loop1, 16, 24, I32_2, I16_2)
         pack4 = self.pack(loop1, 24, 32, I16, None)
         def void(b,c):
             pass
@@ -323,17 +323,17 @@
         raw_store(p1, i3, i12, descr=float)
         raw_store(p1, i4, i13, descr=float)
         """)
-        pack1 = self.pack(loop1, 0, 4, None, I64)
-        pack2 = self.pack(loop1, 4, 8, I64, I32_2)
+        pack1 = self.pack(loop1, 0, 4, None, F64)
+        pack2 = self.pack(loop1, 4, 8, F64, I32_2)
         pack3 = self.pack(loop1, 8, 12, I32, None)
         loop2 = self.schedule(loop1, [pack1,pack2,pack3])
         loop3 = self.parse("""
         v44[f64|2] = vec_raw_load(p0, i1, 2, descr=double) 
         v45[f64|2] = vec_raw_load(p0, i3, 2, descr=double) 
-        v46[f32|2] = vec_cast_float_to_singlefloat(v44[f64|2]) 
-        v47[f32|2] = vec_cast_float_to_singlefloat(v45[f64|2]) 
-        v41[f32|4] = vec_float_pack(v46[f32|2], v47[f32|2], 2, 2) 
-        vec_raw_store(p1, i1, v41[f32|4], descr=float)
+        v46[i32|2] = vec_cast_float_to_singlefloat(v44[f64|2]) 
+        v47[i32|2] = vec_cast_float_to_singlefloat(v45[f64|2]) 
+        v41[i32|4] = vec_int_pack(v46[i32|2], v47[i32|2], 2, 2) 
+        vec_raw_store(p1, i1, v41[i32|4], descr=float)
         """, False)
         self.assert_equal(loop2, loop3)
 
@@ -350,7 +350,7 @@
         """)
         pack1 = self.pack(loop1, 0, 2, None, I64)
         pack2 = self.pack(loop1, 2, 4, I64, I64)
-        pack3 = self.pack(loop1, 4, 6, None, I64)
+        pack3 = self.pack(loop1, 4, 6, I64, None)
         loop2 = self.schedule(loop1, [pack1,pack2,pack3], prepend_invariant=True)
         loop3 = self.parse("""
         v9[i64|2] = vec_int_expand(255)
@@ -372,10 +372,10 @@
         pack2 = self.pack(loop1, 2, 4, I32_2, None)
         loop2 = self.schedule(loop1, [pack1,pack2], prepend_invariant=True)
         loop3 = self.parse("""
-        v1[ui32|2] = vec_raw_load(p0, i1, 2, descr=float)
-        i10 = vec_int_unpack(v1[ui32|2], 0, 1)
+        v1[i32|2] = vec_raw_load(p0, i1, 2, descr=float)
+        i10 = vec_int_unpack(v1[i32|2], 0, 1)
         raw_store(p0, i3, i10, descr=float)
-        i11 = vec_int_unpack(v1[ui32|2], 1, 1)
+        i11 = vec_int_unpack(v1[i32|2], 1, 1)
         raw_store(p0, i4, i11, descr=float)
         """, False)
         # unfortunate ui32 is the type for float32... the unsigned u is for
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -44,6 +44,9 @@
         loop.operations = pre + loop.operations
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
+        for op in loop.operations:
+            if op.getopnum() == rop.GUARD_EARLY_EXIT and op.getdescr() is None:
+                op.setdescr(compile.ResumeAtLoopHeaderDescr())
         return loop
 
     def assert_vectorize(self, loop, expected_loop, call_pure_results=None):
@@ -1356,6 +1359,7 @@
 
 
     def test_abc(self):
+        py.test.skip()
         trace="""
         # int32 sum
         label(p0, p19, i18, i24, i14, i8, i25, descr=TargetToken(140320937897104))
@@ -1369,22 +1373,6 @@
         i32 = int_ge(i30, i25)
         guard_false(i32, descr=<Guard0x7f9f03ab17d0>) [p0, i29, i30, i31, p19, None, None, None]
         jump(p0, p19, i30, i31, i29, i8, i25, descr=TargetToken(140320937897104))
-
-        """
-        trace ="""
-        [i0, i1, i16, i17, i18, i5, p6, p7, p8, f19, p10, p11, p12, p13, p14, p15, i20, i21]
-        guard_early_exit(descr=<rpython.jit.metainterp.compile.ResumeAtLoopHeaderDescr object at 0x7f249eb2e510>) [i5, i18, i17, i16, i1, i0, p15, p14, p13, p12, p11, p10, p8, p7, p6, f19]
-        f22 = raw_load(i20, i18, descr=floatarraydescr)
-        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7f249eb2ec90>) [i5, i18, i17, i16, i1, i0, p15, p14, p13, p12, p11, p10, p8, p7, p6, f22, f19]
-        f23 = raw_load(i21, i17, descr=floatarraydescr)
-        f24 = float_mul(f22, f23)
-        f25 = float_add(f19, f24)
-        i26 = int_add(i18, 8)
-        i27 = int_add(i17, 8)
-        i28 = int_lt(i16, i5)
-        guard_true(i28, descr=<rpython.jit.metainterp.compile.ResumeGuardTrueDescr object at 0x7f249eb99290>) [i5, i26, i27, i16, i1, i0, p15, p14, p13, p12, p11, p10, p8, p7, p6, f25, None]
-        i31 = int_add(i16, 1)
-        jump(i0, i1, i31, i27, i26, i5, p6, p7, p8, f25, p10, p11, p12, p13, p14, p15, i20, i21)
         """
         # schedule 885 -> ptype is non for raw_load?
         opt = self.vectorize(self.parse_loop(trace))

From noreply at buildbot.pypy.org  Thu Jul  2 10:56:06 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 10:56:06 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: Test,
	and fixes for 32-bit
Message-ID: <20150702085606.651C01C1067@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78406:5e7568a7f96d
Date: 2015-07-02 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/5e7568a7f96d/

Log:	Test, and fixes for 32-bit

diff --git a/rpython/rlib/longlong2float.py b/rpython/rlib/longlong2float.py
--- a/rpython/rlib/longlong2float.py
+++ b/rpython/rlib/longlong2float.py
@@ -9,7 +9,7 @@
 from __future__ import with_statement
 import sys
 from rpython.annotator import model as annmodel
-from rpython.rlib.rarithmetic import r_int64
+from rpython.rlib.rarithmetic import r_int64, intmask
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -107,7 +107,7 @@
 # For encoding integers inside nonstandard NaN bit patterns.
 #   ff ff ff fe xx xx xx xx    (signed 32-bit int)
 nan_high_word_int32 = -2       # -2 == (int)0xfffffffe
-nan_encoded_zero = rffi.cast(rffi.LONGLONG, nan_high_word_int32 << 32)
+nan_encoded_zero = r_int64(nan_high_word_int32 << 32)
 
 def encode_int32_into_longlong_nan(value):
     return (nan_encoded_zero +
@@ -117,7 +117,7 @@
     return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
 
 def is_int32_from_longlong_nan(value):
-    return (value >> 32) == nan_high_word_int32
+    return intmask(value >> 32) == nan_high_word_int32
 
 CAN_ALWAYS_ENCODE_INT32 = (sys.maxint == 2147483647)
 
@@ -127,4 +127,4 @@
     return value == rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
 
 def can_encode_float(value):
-    return (float2longlong(value) >> 32) != nan_high_word_int32
+    return intmask(float2longlong(value) >> 32) != nan_high_word_int32
diff --git a/rpython/rlib/test/test_longlong2float.py b/rpython/rlib/test/test_longlong2float.py
--- a/rpython/rlib/test/test_longlong2float.py
+++ b/rpython/rlib/test/test_longlong2float.py
@@ -66,3 +66,32 @@
     for x in enum_floats():
         res = fn2(x)
         assert repr(res) == repr(float(r_singlefloat(x)))
+
+# ____________________________________________________________
+
+def fn_encode_nan(f1, i2):
+    from rpython.rlib.longlong2float import can_encode_float, can_encode_int32
+    from rpython.rlib.longlong2float import encode_int32_into_longlong_nan
+    from rpython.rlib.longlong2float import decode_int32_from_longlong_nan
+    from rpython.rlib.longlong2float import is_int32_from_longlong_nan
+    from rpython.rlib.rfloat import isnan
+    assert can_encode_float(f1)
+    assert can_encode_int32(i2)
+    l1 = float2longlong(f1)
+    l2 = encode_int32_into_longlong_nan(i2)
+    assert not is_int32_from_longlong_nan(l1)
+    assert is_int32_from_longlong_nan(l2)
+    f1b = longlong2float(l1)
+    assert f1b == f1 or (isnan(f1b) and isnan(f1))
+    assert decode_int32_from_longlong_nan(l2) == i2
+    return 42
+
+def test_compiled_encode_nan():
+    fn2 = compile(fn_encode_nan, [float, int])
+    ints = [-2**31, 2**31-1, 42]
+    for x in enum_floats():
+        y = ints.pop()
+        ints.insert(0, y)
+        fn_encode_nan(x, y)
+        res = fn2(x, y)
+        assert res == 42

From noreply at buildbot.pypy.org  Thu Jul  2 12:46:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 12:46:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: Tweak 'sys.version' to actually contain
 the compiler used to compile
Message-ID: <20150702104618.1EB831C026F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78407:45fae072cb84
Date: 2015-07-02 12:34 +0200
http://bitbucket.org/pypy/pypy/changeset/45fae072cb84/

Log:	Tweak 'sys.version' to actually contain the compiler used to compile
	the final C sources, instead of whatever gcc was present when we
	produced the C sources. Useful e.g. with pypy-stm where we use a
	different compiler at the end (clang or gcc-seg-gs).

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -66,7 +66,7 @@
 
         'api_version'           : 'version.get_api_version(space)',
         'version_info'          : 'version.get_version_info(space)',
-        'version'               : 'version.get_version(space)',
+        #'version'              : set in startup()
         'pypy_version_info'     : 'version.get_pypy_version_info(space)',
         'subversion'            : 'version.get_subversion_info(space)',
         '_mercurial'            : 'version.get_repo_info(space)',
@@ -108,6 +108,9 @@
             assert self.filesystemencoding is None
 
         else:
+            from pypy.module.sys import version
+            space.setitem(self.w_dict, space.wrap("version"),
+                          space.wrap(version.get_version(space)))
             if _WIN:
                 from pypy.module.sys import vm
                 w_handle = vm.get_dllhandle(space)
diff --git a/pypy/module/sys/test/test_version.py b/pypy/module/sys/test/test_version.py
--- a/pypy/module/sys/test/test_version.py
+++ b/pypy/module/sys/test/test_version.py
@@ -2,13 +2,13 @@
     def test_compiler(self):
         import sys
         assert ("MSC v." in sys.version or
-                "GCC " in sys.version)
+                "GCC " in sys.version or
+                "(untranslated)" in sys.version)
 
-def test_get_version(space, monkeypatch):
+def test_get_version():
     from pypy.module.sys import version
-    monkeypatch.setattr(version, 'PYPY_VERSION', (2,5,0, "final", 1))
-    res = space.unwrap(version.get_version(space))
+    res = version._make_version_template(PYPY_VERSION=(2,5,0, "final", 1))
     assert "[PyPy 2.5.0" in res
-    monkeypatch.setattr(version, 'PYPY_VERSION', (2,6,3, "alpha", 5))
-    res = space.unwrap(version.get_version(space))
+    res = version._make_version_template(PYPY_VERSION=(2,6,3, "alpha", 5))
     assert "[PyPy 2.6.3-alpha5" in res
+    assert res.endswith(' with %s]')
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -2,7 +2,7 @@
 Version numbers exposed by PyPy through the 'sys' module.
 """
 import os
-from rpython.translator.platform import platform
+from rpython.rlib import compilerinfo
 from pypy.interpreter import gateway
 
 #XXX # the release serial 42 is not in range(16)
@@ -12,15 +12,6 @@
 
 PYPY_VERSION               = (2, 7, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
-if platform.name == 'msvc':
-    COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
-elif platform.cc is not None and \
-        os.path.basename(platform.cc).startswith(('gcc', 'clang')):
-    from rpython.rtyper.tool import rffi_platform
-    COMPILER_INFO = 'GCC ' + rffi_platform.getdefinedstring('__VERSION__', '')
-else:
-    COMPILER_INFO = ""
-
 
 import pypy
 pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
@@ -57,19 +48,24 @@
     w_version_info = app.wget(space, "version_info")
     return space.call_function(w_version_info, space.wrap(CPYTHON_VERSION))
 
-def get_version(space):
+def _make_version_template(PYPY_VERSION=PYPY_VERSION):
     ver = "%d.%d.%d" % (PYPY_VERSION[0], PYPY_VERSION[1], PYPY_VERSION[2])
     if PYPY_VERSION[3] != "final":
         ver = ver + "-%s%d" %(PYPY_VERSION[3], PYPY_VERSION[4])
-    return space.wrap("%d.%d.%d (%s, %s, %s)\n[PyPy %s%s]" % (
+    template = "%d.%d.%d (%s, %s, %s)\n[PyPy %s with %%s]" % (
         CPYTHON_VERSION[0],
         CPYTHON_VERSION[1],
         CPYTHON_VERSION[2],
         get_repo_version_info(root=pypyroot)[1],
         date,
         time,
-        ver,
-        compiler_version()))
+        ver)
+    assert template.count('%') == 1     # only for the "%s" near the end
+    return template
+_VERSION_TEMPLATE = _make_version_template()
+
+def get_version(space):
+    return space.wrap(_VERSION_TEMPLATE % compilerinfo.get_compiler_info())
 
 def get_winver(space):
     return space.wrap("%d.%d" % (
@@ -111,8 +107,3 @@
             ver[2] << 8    |
             d[ver[3]] << 4 |
             subver)
-
-def compiler_version():
-    if not COMPILER_INFO:
-        return ""
-    return " with %s" % (COMPILER_INFO,)
diff --git a/rpython/rlib/compilerinfo.py b/rpython/rlib/compilerinfo.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/compilerinfo.py
@@ -0,0 +1,26 @@
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import rffi
+from rpython.translator.platform import platform
+
+
+def get_compiler_info():
+    """Returns a string like 'MSC v.# 32 bit' or 'GCC #.#.#'.
+    Before translation, returns '(untranslated)'.
+
+    Must be called at run-time, not before translation, otherwise
+    you're freezing the string '(untranslated)' into the executable!
+    """
+    if we_are_translated():
+        return rffi.charp2str(COMPILER_INFO)
+    return "(untranslated)"
+
+# ____________________________________________________________
+
+
+if platform.name == 'msvc':
+    # XXX hard-code the MSC version, I don't feel like computing it dynamically
+    _C_COMPILER_INFO = '"MSC v.%d 32 bit"' % (platform.version * 10 + 600)
+else:
+    _C_COMPILER_INFO = '("GCC " __VERSION__)'
+
+COMPILER_INFO = rffi.CConstant(_C_COMPILER_INFO, rffi.CCHARP)
diff --git a/rpython/rlib/test/test_compilerinfo.py b/rpython/rlib/test/test_compilerinfo.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_compilerinfo.py
@@ -0,0 +1,26 @@
+from rpython.rlib.compilerinfo import get_compiler_info
+from rpython.translator.c.test.test_genc import compile
+
+
+def test_untranslated():
+    assert get_compiler_info() == "untranslated"
+
+def fn(index):
+    cc = get_compiler_info()
+    if index < len(cc):
+        return ord(cc[index])
+    return 0
+
+def test_compiled():
+    fn2 = compile(fn, [int])
+    lst = []
+    index = 0
+    while True:
+        c = fn2(index)
+        if c == 0:
+            break
+        lst.append(chr(c))
+        index += 1
+    s = ''.join(lst)
+    print s
+    assert s.startswith('MSC ') or s.startswith('GCC ')

From noreply at buildbot.pypy.org  Thu Jul  2 12:49:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 12:49:56 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Graft 45fae072cb84 from default
Message-ID: <20150702104956.B3F9D1C026F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8
Changeset: r78408:ac38229dcced
Date: 2015-07-02 12:50 +0200
http://bitbucket.org/pypy/pypy/changeset/ac38229dcced/

Log:	Graft 45fae072cb84 from default

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -66,7 +66,7 @@
 
         'api_version'           : 'version.get_api_version(space)',
         'version_info'          : 'version.get_version_info(space)',
-        'version'               : 'version.get_version(space)',
+        #'version'              : set in startup()
         'pypy_version_info'     : 'version.get_pypy_version_info(space)',
         'subversion'            : 'version.get_subversion_info(space)',
         '_mercurial'            : 'version.get_repo_info(space)',
@@ -108,6 +108,9 @@
             assert self.filesystemencoding is None
 
         else:
+            from pypy.module.sys import version
+            space.setitem(self.w_dict, space.wrap("version"),
+                          space.wrap(version.get_version(space)))
             if _WIN:
                 from pypy.module.sys import vm
                 w_handle = vm.get_dllhandle(space)
diff --git a/pypy/module/sys/test/test_version.py b/pypy/module/sys/test/test_version.py
--- a/pypy/module/sys/test/test_version.py
+++ b/pypy/module/sys/test/test_version.py
@@ -2,13 +2,13 @@
     def test_compiler(self):
         import sys
         assert ("MSC v." in sys.version or
-                "GCC " in sys.version)
+                "GCC " in sys.version or
+                "(untranslated)" in sys.version)
 
-def test_get_version(space, monkeypatch):
+def test_get_version():
     from pypy.module.sys import version
-    monkeypatch.setattr(version, 'PYPY_VERSION', (2,5,0, "final", 1))
-    res = space.unwrap(version.get_version(space))
-    assert "[PyPy 2.5.0" in res
-    monkeypatch.setattr(version, 'PYPY_VERSION', (2,6,3, "alpha", 5))
-    res = space.unwrap(version.get_version(space))
-    assert "[PyPy 2.6.3-alpha5" in res
+    res = version._make_version_template(PYPY_VERSION=(2,5,0, "final", 1))
+    assert "[PyPy%s 2.5.0" in res
+    res = version._make_version_template(PYPY_VERSION=(2,6,3, "alpha", 5))
+    assert "[PyPy%s 2.6.3-alpha5" in res
+    assert res.endswith(' with %s]')
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -2,7 +2,7 @@
 Version numbers exposed by PyPy through the 'sys' module.
 """
 import os
-from rpython.translator.platform import platform
+from rpython.rlib import compilerinfo
 from pypy.interpreter import gateway
 
 #XXX # the release serial 42 is not in range(16)
@@ -12,15 +12,6 @@
 
 PYPY_VERSION               = (2, 6, 0, "final", 0)    #XXX # sync patchlevel.h
 
-if platform.name == 'msvc':
-    COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
-elif platform.cc is not None and \
-        os.path.basename(platform.cc).startswith(('gcc', 'clang')):
-    from rpython.rtyper.tool import rffi_platform
-    COMPILER_INFO = 'GCC ' + rffi_platform.getdefinedstring('__VERSION__', '')
-else:
-    COMPILER_INFO = ""
-
 
 import pypy
 pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
@@ -57,23 +48,28 @@
     w_version_info = app.wget(space, "version_info")
     return space.call_function(w_version_info, space.wrap(CPYTHON_VERSION))
 
-def get_version(space):
+def _make_version_template(PYPY_VERSION=PYPY_VERSION):
     ver = "%d.%d.%d" % (PYPY_VERSION[0], PYPY_VERSION[1], PYPY_VERSION[2])
     if PYPY_VERSION[3] != "final":
         ver = ver + "-%s%d" %(PYPY_VERSION[3], PYPY_VERSION[4])
-    extra = ''
-    if space.config.translation.stm:
-        extra = '-STM'
-    return space.wrap("%d.%d.%d (%s, %s, %s)\n[PyPy%s %s%s]" % (
+    template = "%d.%d.%d (%s, %s, %s)\n[PyPy%%s %s with %%s]" % (
         CPYTHON_VERSION[0],
         CPYTHON_VERSION[1],
         CPYTHON_VERSION[2],
         get_repo_version_info(root=pypyroot)[1],
         date,
         time,
-        extra,
-        ver,
-        compiler_version()))
+        ver)
+    assert template.count('%') == 2     # two "%s" should remain
+    return template
+_VERSION_TEMPLATE = _make_version_template()
+
+def get_version(space):
+    extra = ''
+    if space.config.translation.stm:
+        extra = '-STM'
+    return space.wrap(_VERSION_TEMPLATE % (
+        extra, compilerinfo.get_compiler_info()))
 
 def get_winver(space):
     return space.wrap("%d.%d" % (
@@ -115,8 +111,3 @@
             ver[2] << 8    |
             d[ver[3]] << 4 |
             subver)
-
-def compiler_version():
-    if not COMPILER_INFO:
-        return ""
-    return " with %s" % (COMPILER_INFO,)
diff --git a/rpython/rlib/compilerinfo.py b/rpython/rlib/compilerinfo.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/compilerinfo.py
@@ -0,0 +1,26 @@
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import rffi
+from rpython.translator.platform import platform
+
+
+def get_compiler_info():
+    """Returns a string like 'MSC v.# 32 bit' or 'GCC #.#.#'.
+    Before translation, returns '(untranslated)'.
+
+    Must be called at run-time, not before translation, otherwise
+    you're freezing the string '(untranslated)' into the executable!
+    """
+    if we_are_translated():
+        return rffi.charp2str(COMPILER_INFO)
+    return "(untranslated)"
+
+# ____________________________________________________________
+
+
+if platform.name == 'msvc':
+    # XXX hard-code the MSC version, I don't feel like computing it dynamically
+    _C_COMPILER_INFO = '"MSC v.%d 32 bit"' % (platform.version * 10 + 600)
+else:
+    _C_COMPILER_INFO = '("GCC " __VERSION__)'
+
+COMPILER_INFO = rffi.CConstant(_C_COMPILER_INFO, rffi.CCHARP)
diff --git a/rpython/rlib/test/test_compilerinfo.py b/rpython/rlib/test/test_compilerinfo.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_compilerinfo.py
@@ -0,0 +1,26 @@
+from rpython.rlib.compilerinfo import get_compiler_info
+from rpython.translator.c.test.test_genc import compile
+
+
+def test_untranslated():
+    assert get_compiler_info() == "untranslated"
+
+def fn(index):
+    cc = get_compiler_info()
+    if index < len(cc):
+        return ord(cc[index])
+    return 0
+
+def test_compiled():
+    fn2 = compile(fn, [int])
+    lst = []
+    index = 0
+    while True:
+        c = fn2(index)
+        if c == 0:
+            break
+        lst.append(chr(c))
+        index += 1
+    s = ''.join(lst)
+    print s
+    assert s.startswith('MSC ') or s.startswith('GCC ')

From noreply at buildbot.pypy.org  Thu Jul  2 12:50:41 2015
From: noreply at buildbot.pypy.org (timfel)
Date: Thu,  2 Jul 2015 12:50:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Drop to gdb on unix when pdb.set_trace
	is encountered
Message-ID: <20150702105041.7950B1C026F@cobra.cs.uni-duesseldorf.de>

Author: Tim Felgentreff <timfelgentreff at gmail.com>
Branch: 
Changeset: r78409:9864b76f82fb
Date: 2015-07-02 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/9864b76f82fb/

Log:	Drop to gdb on unix when pdb.set_trace is encountered

diff --git a/rpython/rtyper/extfuncregistry.py b/rpython/rtyper/extfuncregistry.py
--- a/rpython/rtyper/extfuncregistry.py
+++ b/rpython/rtyper/extfuncregistry.py
@@ -9,6 +9,7 @@
 from rpython.rtyper.lltypesystem.module import ll_math
 from rpython.rtyper.module import ll_os
 from rpython.rtyper.module import ll_time
+from rpython.rtyper.module import ll_pdb
 from rpython.rlib import rfloat
 
 # the following functions all take one float, return one float
@@ -54,4 +55,3 @@
                           export_name='ll_math.%s' % method_name,
                           sandboxsafe=True,
                           llimpl=getattr(ll_math, method_name))
-
diff --git a/rpython/rtyper/module/ll_pdb.py b/rpython/rtyper/module/ll_pdb.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/module/ll_pdb.py
@@ -0,0 +1,26 @@
+"""
+Low-level implementation for pdb.set_trace()
+"""
+
+import os
+import pdb
+from rpython.rtyper.module.support import _WIN32
+from rpython.rtyper.extfunc import register_external
+
+
+if not _WIN32:
+    def pdb_set_trace():
+        pid = os.getpid()
+        gdbpid = os.fork()
+        if gdbpid == 0:
+            shell = os.environ.get("SHELL") or "/bin/sh"
+            sepidx = shell.rfind(os.sep) + 1
+            if sepidx > 0:
+                argv0 = shell[sepidx:]
+            else:
+                argv0 = shell
+            try:
+                os.execv(shell, [argv0, "-c", "gdb -p %d" % pid])
+            except OSError as e:
+                raise SystemExit('Could not start GDB: %s.' % e)
+    register_external(pdb.set_trace, [], llimpl=pdb_set_trace)

From noreply at buildbot.pypy.org  Thu Jul  2 12:50:42 2015
From: noreply at buildbot.pypy.org (timfel)
Date: Thu,  2 Jul 2015 12:50:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge default
Message-ID: <20150702105042.AA6291C026F@cobra.cs.uni-duesseldorf.de>

Author: Tim Felgentreff <timfelgentreff at gmail.com>
Branch: 
Changeset: r78410:b22f13cfe74d
Date: 2015-07-02 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/b22f13cfe74d/

Log:	merge default

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -349,11 +349,14 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            else:
-                # this only runs if fromlist_w != ['*']
-                for w_name in fromlist_w:
-                    if try_getattr(space, w_mod, w_name) is None:
-                        return None
+                else:
+                    fromlist_w = []
+                    # "from x import *" with x already imported and no x.__all__
+                    # always succeeds without doing more imports.  It will
+                    # just copy everything from x.__dict__ as it is now.
+            for w_name in fromlist_w:
+                if try_getattr(space, w_mod, w_name) is None:
+                    return None
         return w_mod
     return first
 
@@ -391,12 +394,12 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            else:
-                # this only runs if fromlist_w != ['*']
-                for w_name in fromlist_w:
-                    if try_getattr(space, w_mod, w_name) is None:
-                        load_part(space, w_path, prefix, space.str0_w(w_name),
-                                  w_mod, tentative=1)
+                else:
+                    fromlist_w = []
+            for w_name in fromlist_w:
+                if try_getattr(space, w_mod, w_name) is None:
+                    load_part(space, w_path, prefix, space.str0_w(w_name),
+                              w_mod, tentative=1)
         return w_mod
     else:
         return first
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -66,6 +66,14 @@
              b          = "insubpackage = 1",
              )
     setuppkg("pkg.pkg2", a='', b='')
+    setuppkg("pkg.withall",
+             __init__  = "__all__ = ['foobar']",
+             foobar    = "found = 123")
+    setuppkg("pkg.withoutall",
+             __init__  = "",
+             foobar    = "found = 123")
+    setuppkg("pkg.bogusall",
+             __init__  = "__all__ = 42")
     setuppkg("pkg_r", inpkg = "import x.y")
     setuppkg("pkg_r.x")
     setuppkg("x", y='')
@@ -677,6 +685,32 @@
         import imp
         raises(ValueError, imp.load_module, "", "", "", [1, 2, 3, 4])
 
+    def test_import_star_finds_submodules_with___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withall import *" in d
+            assert d["foobar"].found == 123
+
+    def test_import_star_does_not_find_submodules_without___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withoutall import *" in d
+            assert "foobar" not in d
+        import pkg.withoutall.foobar     # <- import it here only
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withoutall import *" in d
+            assert d["foobar"].found == 123
+
+    def test_import_star_with_bogus___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            try:
+                exec "from pkg.bogusall import *" in {}
+            except TypeError:
+                pass    # 'int' object does not support indexing
+            else:
+                raise AssertionError("should have failed")
+
 
 class TestAbi:
     def test_abi_tag(self):

From noreply at buildbot.pypy.org  Thu Jul  2 12:50:43 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu,  2 Jul 2015 12:50:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged in timfel/pypy (pull request
	#327)
Message-ID: <20150702105043.C853B1C026F@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r78411:ead7b35f3abb
Date: 2015-07-02 12:50 +0200
http://bitbucket.org/pypy/pypy/changeset/ead7b35f3abb/

Log:	Merged in timfel/pypy (pull request #327)

	Drop to gdb on unix when pdb.set_trace is encountered

diff --git a/rpython/rtyper/extfuncregistry.py b/rpython/rtyper/extfuncregistry.py
--- a/rpython/rtyper/extfuncregistry.py
+++ b/rpython/rtyper/extfuncregistry.py
@@ -9,6 +9,7 @@
 from rpython.rtyper.lltypesystem.module import ll_math
 from rpython.rtyper.module import ll_os
 from rpython.rtyper.module import ll_time
+from rpython.rtyper.module import ll_pdb
 from rpython.rlib import rfloat
 
 # the following functions all take one float, return one float
@@ -54,4 +55,3 @@
                           export_name='ll_math.%s' % method_name,
                           sandboxsafe=True,
                           llimpl=getattr(ll_math, method_name))
-
diff --git a/rpython/rtyper/module/ll_pdb.py b/rpython/rtyper/module/ll_pdb.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/module/ll_pdb.py
@@ -0,0 +1,26 @@
+"""
+Low-level implementation for pdb.set_trace()
+"""
+
+import os
+import pdb
+from rpython.rtyper.module.support import _WIN32
+from rpython.rtyper.extfunc import register_external
+
+
+if not _WIN32:
+    def pdb_set_trace():
+        pid = os.getpid()
+        gdbpid = os.fork()
+        if gdbpid == 0:
+            shell = os.environ.get("SHELL") or "/bin/sh"
+            sepidx = shell.rfind(os.sep) + 1
+            if sepidx > 0:
+                argv0 = shell[sepidx:]
+            else:
+                argv0 = shell
+            try:
+                os.execv(shell, [argv0, "-c", "gdb -p %d" % pid])
+            except OSError as e:
+                raise SystemExit('Could not start GDB: %s.' % e)
+    register_external(pdb.set_trace, [], llimpl=pdb_set_trace)

From noreply at buildbot.pypy.org  Thu Jul  2 13:10:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 13:10:27 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: hg merge default
Message-ID: <20150702111027.E757E1C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1885:e7c5946182cc
Date: 2015-07-02 11:38 +0100
http://bitbucket.org/pypy/stmgc/changeset/e7c5946182cc/

Log:	hg merge default

diff too long, truncating to 2000 out of 5492 lines

diff --git a/c7/demo/Makefile b/c7/demo/Makefile
--- a/c7/demo/Makefile
+++ b/c7/demo/Makefile
@@ -19,20 +19,18 @@
 
 COMMON = -I.. -pthread -lrt -g -Wall -Werror -DSTM_LARGEMALLOC_TEST
 
-CC = gcc-seg-gs
-
 
 # note that 'build' is partially optimized but still contains all asserts
 debug-%: %.c ${H_FILES} ${C_FILES}
-	$(CC) $(COMMON) -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -O0 \
+	clang $(COMMON) -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -O0 \
         $< -o debug-$* ../stmgc.c
 
 build-%: %.c ${H_FILES} ${C_FILES}
-	$(CC) $(COMMON) -DSTM_GC_NURSERY=128 -O1 $< -o build-$* ../stmgc.c
+	clang $(COMMON) -DSTM_GC_NURSERY=128 -O1 $< -o build-$* ../stmgc.c
 
 release-%: %.c ${H_FILES} ${C_FILES}
-	$(CC) $(COMMON) -DNDEBUG -O2 $< -o release-$* ../stmgc.c
+	clang $(COMMON) -DNDEBUG -O2 $< -o release-$* ../stmgc.c
 
 
 release-htm-%: %.c ../../htm-c7/stmgc.? ../../htm-c7/htm.h
-	$(CC) $(COMMON) -O2 $< -o release-htm-$* ../../htm-c7/stmgc.c -DUSE_HTM
+	clang $(COMMON) -O2 $< -o release-htm-$* ../../htm-c7/stmgc.c -DUSE_HTM
diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c
--- a/c7/demo/demo2.c
+++ b/c7/demo/demo2.c
@@ -216,7 +216,7 @@
 
 void teardown_list(void)
 {
-    STM_POP_ROOT_DROP(stm_thread_local);
+    STM_POP_ROOT_RET(stm_thread_local);
 }
 
 
@@ -256,7 +256,6 @@
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     unregister_thread_local();
     status = sem_post(&done); assert(status == 0);
-    (void)status;
     return NULL;
 }
 
@@ -294,7 +293,6 @@
     rewind_jmp_buf rjbuf;
 
     status = sem_init(&done, 0, 0); assert(status == 0);
-    (void)status;
 
     stm_setup();
     stm_register_thread_local(&stm_thread_local);
diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c
--- a/c7/demo/demo_random.c
+++ b/c7/demo/demo_random.c
@@ -412,7 +412,6 @@
     stm_unregister_thread_local(&stm_thread_local);
 
     status = sem_post(&done); assert(status == 0);
-    (void)status;
     return NULL;
 }
 
diff --git a/c7/demo/demo_random2.c b/c7/demo/demo_random2.c
--- a/c7/demo/demo_random2.c
+++ b/c7/demo/demo_random2.c
@@ -435,7 +435,6 @@
     stm_unregister_thread_local(&stm_thread_local);
 
     status = sem_post(&done); assert(status == 0);
-    (void)status;
     return NULL;
 }
 
diff --git a/c7/demo/test_shadowstack.c b/c7/demo/test_shadowstack.c
--- a/c7/demo/test_shadowstack.c
+++ b/c7/demo/test_shadowstack.c
@@ -54,7 +54,7 @@
        then do a major collection.  It should still be found by the
        tracing logic. */
     stm_start_transaction(&stm_thread_local);
-    STM_POP_ROOT_DROP(stm_thread_local);
+    STM_POP_ROOT_RET(stm_thread_local);
     STM_POP_ROOT(stm_thread_local, node);
     assert(node->value == 129821);
     STM_PUSH_ROOT(stm_thread_local, NULL);
diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -34,6 +34,12 @@
                 raise
     Func(func.__name__)
 
+def int_(x):
+    if isinstance(x, gdb.Value):
+        T = gdb.lookup_type('long')
+        x = x.cast(T)
+    return int(x)
+
 # -------------------------------------------------------
 
 _nb_segments = None
@@ -43,26 +49,26 @@
 def get_nb_segments():
     global _nb_segments
     if _nb_segments is None:
-        _nb_segments = int(gdb.parse_and_eval('_stm_nb_segments'))
+        _nb_segments = int_(gdb.parse_and_eval('_stm_nb_segments'))
         assert 1 < _nb_segments <= 240
     return _nb_segments
 
 def get_segment_size():
     global _segment_size
     if _segment_size is None:
-        nb_pages = int(gdb.parse_and_eval('_stm_segment_nb_pages'))
+        nb_pages = int_(gdb.parse_and_eval('_stm_segment_nb_pages'))
         _segment_size = nb_pages * 4096
     return _segment_size
 
 def get_psegment_ofs():
     global _psegment_ofs
     if _psegment_ofs is None:
-        _psegment_ofs = int(gdb.parse_and_eval('_stm_psegment_ofs'))
+        _psegment_ofs = int_(gdb.parse_and_eval('_stm_psegment_ofs'))
     return _psegment_ofs
 
 def get_segment_base(segment_id):
     assert 0 <= segment_id <= get_nb_segments()
-    base = int(gdb.parse_and_eval('stm_object_pages'))
+    base = int_(gdb.parse_and_eval('stm_object_pages'))
     return base + get_segment_size() * segment_id
 
 def get_psegment(segment_id, field=''):
@@ -72,13 +78,15 @@
         % (get_segment_size() * segment_id + get_psegment_ofs(), field))
 
 def thread_to_segment_id(thread_id):
-    base = int(gdb.parse_and_eval('stm_object_pages'))
+    base = int_(gdb.parse_and_eval('stm_object_pages'))
     for j in range(1, get_nb_segments() + 1):
-        ts = get_psegment(j, '->transaction_state')
-        if int(ts) != 0:
-            ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
-            if int(ti) == thread_id:
-                return j
+        #ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
+        ti = get_psegment(j, '->running_pthread')
+        if int_(ti) == thread_id:
+            ts = get_psegment(j, '->transaction_state')
+            if int_(ts) == 0:
+                print >> sys.stderr, "note: transaction_state == 0"
+            return j
     raise Exception("thread not found: %r" % (thread_id,))
 
 def interactive_segment_base(thread=None):
@@ -92,10 +100,10 @@
         thread_id = int(fields[2], 16)
         segment_id = thread_to_segment_id(thread_id)
     elif thread.type.code == gdb.TYPE_CODE_INT:
-        if 0 <= int(thread) < 256:
-            segment_id = int(thread)
+        if 0 <= int_(thread) < 256:
+            segment_id = int_(thread)
         else:
-            thread_id = int(thread)
+            thread_id = int_(thread)
             segment_id = thread_to_segment_id(thread_id)
     else:
         raise TypeError("'thread' argument must be an int or not given")
@@ -105,12 +113,14 @@
 def gc(p=None, thread=None):
     sb = interactive_segment_base(thread)
     if p is not None and p.type.code == gdb.TYPE_CODE_PTR:
-        return gdb.Value(sb + int(p)).cast(p.type).dereference()
-    elif p is None or int(p) == 0:
+        return gdb.Value(sb + int_(p)).cast(p.type).dereference()
+    else:
+        if p is None:
+            p = 0
+        else:
+            p = int_(p)
         T = gdb.lookup_type('char').pointer()
-        return gdb.Value(sb).cast(T)
-    else:
-        raise TypeError("gc() first argument must be a GC pointer or 0")
+        return gdb.Value(sb + p).cast(T)
 
 @gdb_function
 def psegment(thread=None):
diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -45,6 +45,7 @@
 #endif
 }
 
+__attribute__((always_inline))
 static void write_slowpath_overflow_obj(object_t *obj, bool mark_card)
 {
     /* An overflow object is an object from the same transaction, but
@@ -78,6 +79,7 @@
     }
 }
 
+__attribute__((always_inline))
 static void write_slowpath_common(object_t *obj, bool mark_card)
 {
     assert(_seems_to_be_running_transaction());
@@ -221,7 +223,6 @@
     check_flag_write_barrier(obj);
 }
 
-__attribute__((flatten))
 void _stm_write_slowpath(object_t *obj)
 {
     write_slowpath_common(obj, /*mark_card=*/false);
@@ -240,7 +241,6 @@
     return (size >= _STM_MIN_CARD_OBJ_SIZE);
 }
 
-__attribute__((flatten))
 char _stm_write_slowpath_card_extra(object_t *obj)
 {
     /* the PyPy JIT calls this function directly if it finds that an
diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -58,7 +58,7 @@
     /* Make a new mmap at some other address, but of the same size as
        the standard mmap at stm_object_pages
     */
-    int big_copy_fd = -1;
+    int big_copy_fd;
     char *big_copy = setup_mmap("stmgc's fork support", &big_copy_fd);
 
     /* Copy all the data from the two ranges of objects (large, small)
diff --git a/c7/stm/fprintcolor.c b/c7/stm/fprintcolor.c
--- a/c7/stm/fprintcolor.c
+++ b/c7/stm/fprintcolor.c
@@ -1,5 +1,3 @@
-#include <stdarg.h>
-
 /* ------------------------------------------------------------ */
 #ifdef STM_DEBUGPRINT
 /* ------------------------------------------------------------ */
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -20,15 +20,7 @@
 #endif
 
 
-#ifdef __SEG_GS     /* on a custom patched gcc */
-#  define TLPREFIX __seg_gs
-#  define _STM_RM_SUFFIX  :8
-#elif defined(__clang__)   /* on a clang, hopefully made bug-free */
-#  define TLPREFIX __attribute__((address_space(256)))
-#  define _STM_RM_SUFFIX  /* nothing */
-#else
-#  error "needs either a GCC with __seg_gs support, or a bug-freed clang"
-#endif
+#define TLPREFIX __attribute__((address_space(256)))
 
 typedef TLPREFIX struct object_s object_t;
 typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t;
@@ -42,11 +34,11 @@
        'STM_SEGMENT->transaction_read_version' if and only if the
        object was read in the current transaction.  The nurseries
        also have corresponding read markers, but they are never used. */
-    unsigned char rm _STM_RM_SUFFIX;
+    uint8_t rm;
 };
 
 struct stm_segment_info_s {
-    unsigned int transaction_read_version;
+    uint8_t transaction_read_version;
     int segment_num;
     char *segment_base;
     stm_char *nursery_current;
@@ -296,7 +288,6 @@
 #define STM_PUSH_ROOT(tl, p)   ((tl).shadowstack++->ss = (object_t *)(p))
 #define STM_POP_ROOT(tl, p)    ((p) = (typeof(p))((--(tl).shadowstack)->ss))
 #define STM_POP_ROOT_RET(tl)   ((--(tl).shadowstack)->ss)
-#define STM_POP_ROOT_DROP(tl)  ((void)(--(tl).shadowstack))
 
 
 /* Every thread needs to have a corresponding stm_thread_local_t
@@ -311,12 +302,7 @@
 
 /* At some key places, like the entry point of the thread and in the
    function with the interpreter's dispatch loop, you need to declare
-   a local variable of type 'rewind_jmp_buf' and call these macros.
-   IMPORTANT: a function in which you call stm_rewind_jmp_enterframe()
-   must never change the value of its own arguments!  If they are
-   passed on the stack, gcc can change the value directly there, but
-   we're missing the logic to save/restore this part!
-*/
+   a local variable of type 'rewind_jmp_buf' and call these macros. */
 #define stm_rewind_jmp_enterprepframe(tl, rjbuf)   \
     rewind_jmp_enterprepframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack)
 #define stm_rewind_jmp_enterframe(tl, rjbuf)       \
@@ -520,7 +506,7 @@
 
 #define STM_POP_MARKER(tl)   ({                 \
     object_t *_popped = STM_POP_ROOT_RET(tl);   \
-    STM_POP_ROOT_DROP(tl);                      \
+    STM_POP_ROOT_RET(tl);                       \
     _popped;                                    \
 })
 
diff --git a/c7/test/common.py b/c7/test/common.py
--- a/c7/test/common.py
+++ b/c7/test/common.py
@@ -3,7 +3,7 @@
 assert sys.maxint == 9223372036854775807, "requires a 64-bit environment"
 
 # ----------
-os.environ['CC'] = 'gcc-seg-gs'
+os.environ['CC'] = 'clang'
 
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
diff --git a/c7/test/support.py b/c7/test/support.py
--- a/c7/test/support.py
+++ b/c7/test/support.py
@@ -478,8 +478,7 @@
                     ],
      undef_macros=['NDEBUG'],
      include_dirs=[parent_dir],
-     extra_compile_args=['-g', '-O0', '-Werror',  #, '-ferror-limit=1',  for clang
-                         '-Wfatal-errors'],   # for gcc
+     extra_compile_args=['-g', '-O0', '-Werror', '-ferror-limit=1'],
      extra_link_args=['-g', '-lrt'],
      force_generic_engine=True)
 
diff --git a/c7/test/test_list.py b/c7/test/test_list.py
--- a/c7/test/test_list.py
+++ b/c7/test/test_list.py
@@ -56,7 +56,7 @@
 ''', define_macros=[('STM_TESTS', '1')],
      undef_macros=['NDEBUG'],
      include_dirs=[parent_dir],
-     extra_compile_args=['-g', '-O0', '-Werror'], #, '-ferror-limit=1'],
+     extra_compile_args=['-g', '-O0', '-Werror', '-ferror-limit=1'],
      force_generic_engine=True)
 
 # ____________________________________________________________
diff --git a/c7/test/test_rewind.c b/c7/test/test_rewind.c
--- a/c7/test/test_rewind.c
+++ b/c7/test/test_rewind.c
@@ -174,26 +174,12 @@
 void foo(int *x) { ++*x; }
 
 __attribute__((noinline))
-void f6(int c1, int c2, int c3, int c4, int c5, int c6, int c7,
-        int c8, int c9, int c10, int c11, int c12, int c13)
+void f6(int a1, int a2, int a3, int a4, int a5, int a6, int a7,
+        int a8, int a9, int a10, int a11, int a12, int a13)
 {
     rewind_jmp_buf buf;
     rewind_jmp_enterframe(&gthread, &buf, NULL);
 
-    int a1 = c1;
-    int a2 = c2;
-    int a3 = c3;
-    int a4 = c4;
-    int a5 = c5;
-    int a6 = c6;
-    int a7 = c7;
-    int a8 = c8;
-    int a9 = c9;
-    int a10 = c10;
-    int a11 = c11;
-    int a12 = c12;
-    int a13 = c13;
-
     rewind_jmp_setjmp(&gthread, NULL);
     gevent(a1); gevent(a2); gevent(a3); gevent(a4);
     gevent(a5); gevent(a6); gevent(a7); gevent(a8);
diff --git a/c7/test/test_rewind.py b/c7/test/test_rewind.py
--- a/c7/test/test_rewind.py
+++ b/c7/test/test_rewind.py
@@ -1,11 +1,11 @@
 import os
 
 def run_test(opt):
-    err = os.system("gcc-seg-gs -g -O%s -Werror -DRJBUF_CUSTOM_MALLOC -I../stm"
+    err = os.system("clang -g -O%s -Werror -DRJBUF_CUSTOM_MALLOC -I../stm"
                     " -o test_rewind_O%s test_rewind.c ../stm/rewind_setjmp.c"
                     % (opt, opt))
     if err != 0:
-        raise OSError("gcc-seg-gs failed on test_rewind.c")
+        raise OSError("clang failed on test_rewind.c")
     for testnum in [1, 2, 3, 4, 5, 6, 7, "TL1", "TL2"]:
         print '=== O%s: RUNNING TEST %s ===' % (opt, testnum)
         err = os.system("./test_rewind_O%s %s" % (opt, testnum))
diff --git a/c8/CALL_RELEASE_GIL b/c8/CALL_RELEASE_GIL
new file mode 100644
--- /dev/null
+++ b/c8/CALL_RELEASE_GIL
@@ -0,0 +1,120 @@
+
+c8-gil-like
+===========
+
+A branch to have "GIL-like" behavior for inevitable transactions: one
+not-too-short inevitable transaction that is passed around multiple
+threads.
+
+The goal is to have good fast-case behavior with the PyPy JIT around
+CALL_RELEASE_GIL.  This is how it works in default (with shadowstack):
+
+
+- "rpy_fastgil" is a global variable.  The value 0 means the GIL is
+  definitely unlocked; the value 1 means it is probably locked (it is
+  actually locked only if some mutex object is acquired too).
+
+- before CALL_RELEASE_GIL, we know that we have the GIL and we need to
+  release it.  So we know that "rpy_fastgil" is 1, and we just write 0
+  there.
+
+- then we do the external call.
+
+- after CALL_RELEASE_GIL, two cases:
+
+  - if "rpy_fastgil" has been changed to 1 by some other thread *or*
+    if the (non-thread-local) shadowstack pointer changed, then we
+    call reacqgil_addr();
+
+  - otherwise, we swap rpy_fastgil back to 1 and we're done.
+
+- if the external call is long enough, a different thread will notice
+  that rpy_fastgil == 0 by regular polling, and grab the GIL for
+  itself by swapping it back to 1.  (The changes from 0 to 1 are done
+  with atomic instructions.)
+
+- a different mechanism is used when we voluntarily release the GIL,
+  based on the mutex mentioned above.  The mutex is also used by the
+  the reacqgil_addr() function if it actually needs to wait.
+
+
+Plan for porting this idea to stmgc:
+
+- we add a few macros to stmgc.h which can be used by C code, around
+  external calls; and we also inline these macros manually around
+  CALL_RELEASE_GIL in PyPy's JIT.
+
+- we add the "detached" mode to inevitable transactions: it means that
+  no thread is actively running this inevitable transaction for now,
+  but it was not committed yet.  It is meant to be reattached, by the
+  same or a different thread.
+
+- we add a global variable, "stm_detached_inevitable_from_thread".  It
+  is equal to the stm_thread_local pointer of the thread that detached
+  inevitable transaction (like rpy_fastgil == 0), or NULL if there is
+  no detached inevitable transaction (like rpy_fastgil == 1).
+
+- the macro stm_detach_inevitable_transaction() simply writes the
+  current thread's stm_thread_local pointer into the global variable
+  stm_detached_inevitable_from_thread.  It can only be used if the
+  current transaction is inevitable (and in particular the inevitable
+  transaction was not detached already, because we're running it).
+  After the macro is called, the current thread is assumed not to be
+  running in a transaction any more (no more object or shadowstack
+  access).
+
+- the macro stm_reattach_transaction() does an atomic swap on
+  stm_detached_inevitable_from_thread to change it to NULL.  If the
+  old value was equal to our own stm_thread_local pointer, we are done.  If
+  not, we call a helper, _stm_reattach_transaction().
+
+- we also add the macro stm_detach_transation().  If the current
+  thread is inevitable it calls stm_detach_inevitable_transaction().
+  Otherwise it calls a helper, _stm_detach_noninevitable_transaction().
+
+- _stm_reattach_transaction(old): called with the old value from
+  stm_detached_inevitable_from_thread (which was swapped to be NULL just
+  now).  If old != NULL, this swap had the effect that we took over
+  the inevitable transaction originally detached from a different
+  thread; we need to fix a few things like the stm_thread_local and %gs but
+  then we can continue running this reattached inevitable transaction.
+  If old == NULL, we need to fall back to the current
+  stm_start_transaction().  (A priori, there is no need to wait at
+  this point.  The waiting point is later, in the optional
+  stm_become_inevitable()).
+
+- _stm_detach_noninevitable_transaction(): we try to make the
+  transaction inevitable.  If it works we can then use
+  stm_detach_inevitable_transaction().  On the other hand, if we can't
+  make it inevitable without waiting, then instead we just commit it
+  and continue.  In the latter case,
+  stm_detached_inevitable_from_thread is still NULL.
+
+- other place to fix: major collections.  Maybe simply look inside
+  stm_detached_inevitable_from_thread, and if not NULL, grab the
+  inevitable transaction and commit it now.  Or maybe not.  The point
+  is that we need to prevent a thread from asynchronously grabbing it
+  by an atomic swap of stm_detached_inevitable_from_thread; instead,
+  the parallel threads that finish their external calls should all
+  find NULL in this variable and call _stm_reattach_transaction()
+  which will wait for the major GC to end.
+
+- stm_become_inevitable(): if it finds a detached inevitable
+  transaction, it should attach and commit it as a way to get rid of
+  it.  This is why it might be better to call directly
+  stm_start_inevitable_transaction() when possible: that one is
+  allowed to attach to a detached inevitable transaction and simply
+  return, unlike stm_become_inevitable() which must continue running
+  the existing transaction.
+
+- commit logic of a non-inevitable transaction: we wait if there is
+  an inevitable transaction.  Here too, if the inevitable transaction
+  is found to be detached, we could just commit it now.  Or, a better
+  approach: if we find a detached inevitable transaction we grab it
+  temporarily, and commit only the *non-inevitable* transaction if it
+  doesn't conflict.  The inevitable transaction is then detached
+  again.  (Note that the conflict detection is: we don't commit any
+  write to any of the objects in the inevitable transaction's
+  read-set.  This relies on inevitable threads maintaining their
+  read-set correctly, which should be the case in PyPy, but needs to
+  be checked.)
diff --git a/c8/LOCKS b/c8/LOCKS
new file mode 100644
--- /dev/null
+++ b/c8/LOCKS
@@ -0,0 +1,83 @@
+
+
+main lock-free operation
+========================
+
+The main lock-free operation is at commit time: the compare-and-swap
+that attaches a new log entry after 'last_commit_log_entry'.
+
+
+
+modification_lock
+=================
+
+one per segment.
+
+acquired on segment N when we want to read or write the segment N's
+copy of 'modified_old_objects', the backup copies, etc.
+
+an important user is _stm_validate(): it locks the current segment,
+and all the other segments out of which it is going to read data
+
+could be improved, because _stm_validate() writes into the current
+segment but only reads the other ones.  So far it mostly serializes
+calls to _stm_validate(): if we have two of them starting at roughly
+the same time, they need both to acquire the modification_lock of at
+least the segment that did the most recent commit --- even though it
+could proceed in parallel if they could both realize that they only
+want to read from that same segment.
+
+same, handle_segfault_in_page() acquires two modification_locks: the
+current segment (which needs to be written to), and the
+'copy_from_segnum' (which only needs to be read from).
+
+the current segment modification_lock is also acquired briefly
+whenever we change our segment's 'modified_old_objects'.
+
+_validate_and_attach() needs to have its own segment's
+modification_lock *around* the compare-and-swap, so that
+_stm_validate() sees either the commit not done and the backup copies
+still in modified_old_objects, or the commit done and no backup copies
+any more.
+
+
+--- UPDATE: modification_lock is now done with pthread_rwlock_xxx().
+
+
+
+privatization_lock
+==================
+
+one per segment.  Works like a single "reader-writer" lock: each
+segment acquires either only its copy ("reader") or all of them
+("writer").
+
+"Reader" status is needed to call get_page_status_in().
+"Writer" status is needed to call set_page_status_in/page_mark_(in)accessible.
+
+Essential "writers":
+- handle_segfault_in_page(), but it only writes the status for the current seg
+
+Essential "readers":
+- _stm_validate()
+- push_large_overflow_objects_to_other_segments()
+- nursery.c calling synchronize_object_enqueue()
+
+
+
+mutex and conditions
+====================
+
+There is also one global mutex and a few condition codes.  It's
+unclear if these are still the best solution.
+
+The mutex is acquired in stm_start_transaction() and in
+stm_commit_transaction().  The main purpose is to wait for or signal
+the C_SEGMENT_FREE condition code.
+
+The C_AT_SAFE_POINT and C_REQUEST_REMOVED condition codes are used by
+synchronize_all_threads().  That's used only in rare cases, for
+example because we want to start a major collection.
+
+The mutex also needs to be acquired for rewind_longjmp's setjmp() and
+longjmp() equivalent.
diff --git a/c8/TODO b/c8/TODO
--- a/c8/TODO
+++ b/c8/TODO
@@ -1,17 +1,11 @@
+
+- fix markers (e.g. become_inevitable doesn't seem to show up)
 
 - improve sync of small objs on commit (see FLAG_SYNC_LARGE in nursery.c)
 
-- non-zeroed nursery:
-  read-the-docs benchmark shows 8% time spent in memset of throw_away_nursery
-
 - reshare pages:
   make seg0 MAP_SHARED in order to re-share private pages during major GC
 
-- avoid usleep(10) when waiting for an inevitable transaction:
-  we do this sleep when we try to commit and another inev transaction is
-  currently running. idea: signal the inev transaction to do the commit
-  for us
-
 - maybe re-implement the "please commit soon" signal
 
 - the highest_overflow_number can overflow after 2**30 non-collect-time
@@ -28,3 +22,9 @@
 
 - avoid __builtin_frame_address(0) in precisely the performance-critical
   functions like the interpreter main loop
+
+
+---------------------------
+DONE:
+- non-zeroed nursery:
+  read-the-docs benchmark shows 8% time spent in memset of throw_away_nursery
diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -8,6 +8,8 @@
 #include <sys/wait.h>
 
 #include "stmgc.h"
+#include "stm/fprintcolor.h"
+#include "stm/fprintcolor.c"
 
 #define NUMTHREADS 2
 #define STEPS_PER_THREAD 500
@@ -48,8 +50,10 @@
     int num_roots;
     int num_roots_at_transaction_start;
     int steps_left;
+    long globally_unique;
 };
 __thread struct thread_data td;
+static long progress = 1;
 
 struct thread_data *_get_td(void)
 {
@@ -57,9 +61,16 @@
 }
 
 
+long check_size(long size)
+{
+    assert(size >= sizeof(struct node_s));
+    assert(size <= sizeof(struct node_s) + 4096*70);
+    return size;
+}
+
 ssize_t stmcb_size_rounded_up(struct object_s *ob)
 {
-    return ((struct node_s*)ob)->my_size;
+    return check_size(((struct node_s*)ob)->my_size);
 }
 
 void stmcb_trace(struct object_s *obj, void visit(object_t **))
@@ -69,7 +80,8 @@
 
     /* and the same value at the end: */
     /* note, ->next may be the same as last_next */
-    nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*));
+    nodeptr_t *last_next = (nodeptr_t*)((char*)n + check_size(n->my_size)
+                                        - sizeof(void*));
 
     assert(n->next == *last_next);
 
@@ -113,36 +125,36 @@
     }
 }
 
-void reload_roots()
-{
-    int i;
-    assert(td.num_roots == td.num_roots_at_transaction_start);
-    for (i = td.num_roots_at_transaction_start - 1; i >= 0; i--) {
-        if (td.roots[i])
-            STM_POP_ROOT(stm_thread_local, td.roots[i]);
-    }
-
-    for (i = 0; i < td.num_roots_at_transaction_start; i++) {
-        if (td.roots[i])
-            STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
-    }
-}
-
 void push_roots()
 {
     int i;
+    assert(td.num_roots_at_transaction_start <= td.num_roots);
     for (i = td.num_roots_at_transaction_start; i < td.num_roots; i++) {
         if (td.roots[i])
             STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
     }
+    STM_SEGMENT->no_safe_point_here = 0;
 }
 
 void pop_roots()
 {
     int i;
-    for (i = td.num_roots - 1; i >= td.num_roots_at_transaction_start; i--) {
-        if (td.roots[i])
+    STM_SEGMENT->no_safe_point_here = 1;
+
+    assert(td.num_roots_at_transaction_start <= td.num_roots);
+    for (i = td.num_roots - 1; i >= 0; i--) {
+        if (td.roots[i]) {
             STM_POP_ROOT(stm_thread_local, td.roots[i]);
+            assert(td.roots[i]);
+        }
+    }
+
+    dprintf(("stm_is_inevitable() = %d\n", (int)stm_is_inevitable()));
+    for (i = 0; i < td.num_roots_at_transaction_start; i++) {
+        if (td.roots[i]) {
+            dprintf(("root %d: %p\n", i, td.roots[i]));
+            STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
+        }
     }
 }
 
@@ -150,6 +162,7 @@
 {
     int i;
     assert(idx >= td.num_roots_at_transaction_start);
+    assert(idx < td.num_roots);
 
     for (i = idx; i < td.num_roots - 1; i++)
         td.roots[i] = td.roots[i + 1];
@@ -158,6 +171,7 @@
 
 void add_root(objptr_t r)
 {
+    assert(td.num_roots_at_transaction_start <= td.num_roots);
     if (r && td.num_roots < MAXROOTS) {
         td.roots[td.num_roots++] = r;
     }
@@ -184,7 +198,8 @@
         nodeptr_t n = (nodeptr_t)p;
 
         /* and the same value at the end: */
-        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
         assert(n->next == *last_next);
         n->next = (nodeptr_t)v;
         *last_next = (nodeptr_t)v;
@@ -196,7 +211,8 @@
     nodeptr_t n = (nodeptr_t)p;
 
     /* and the same value at the end: */
-    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
     OPT_ASSERT(n->next == *last_next);
 
     return n->next;
@@ -229,7 +245,7 @@
                            sizeof(struct node_s) + (get_rand(100000) & ~15),
                            sizeof(struct node_s) + 4096,
                            sizeof(struct node_s) + 4096*70};
-        size_t size = sizes[get_rand(4)];
+        size_t size = check_size(sizes[get_rand(4)]);
         p = stm_allocate(size);
         nodeptr_t n = (nodeptr_t)p;
         n->sig = SIGNATURE;
@@ -240,7 +256,6 @@
         n->next = NULL;
         *last_next = NULL;
         pop_roots();
-        /* reload_roots not necessary, all are old after start_transaction */
         break;
     case 4:  // read and validate 'p'
         read_barrier(p);
@@ -288,6 +303,15 @@
     return p;
 }
 
+static void end_gut(void)
+{
+    if (td.globally_unique != 0) {
+        fprintf(stderr, "[GUT END]");
+        assert(progress == td.globally_unique);
+        td.globally_unique = 0;
+        stm_resume_all_other_threads();
+    }
+}
 
 objptr_t do_step(objptr_t p)
 {
@@ -308,8 +332,14 @@
         return NULL;
     } else if (get_rand(240) == 1) {
         push_roots();
-        stm_become_globally_unique_transaction(&stm_thread_local, "really");
-        fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        if (td.globally_unique == 0) {
+            stm_stop_all_other_threads();
+            td.globally_unique = progress;
+            fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        }
+        else {
+            end_gut();
+        }
         pop_roots();
         return NULL;
     }
@@ -347,37 +377,53 @@
 
     objptr_t p;
 
-    stm_start_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
     assert(td.num_roots >= td.num_roots_at_transaction_start);
     td.num_roots = td.num_roots_at_transaction_start;
     p = NULL;
     pop_roots();                /* does nothing.. */
-    reload_roots();
 
     while (td.steps_left-->0) {
         if (td.steps_left % 8 == 0)
             fprintf(stdout, "#");
 
-        assert(p == NULL || ((nodeptr_t)p)->sig == SIGNATURE);
+        int local_seg = STM_SEGMENT->segment_num;
+        int p_sig = p == NULL ? 0 : ((nodeptr_t)p)->sig;
+
+        assert(p == NULL || p_sig == SIGNATURE);
+        (void)local_seg;
+        (void)p_sig;
+
+        if (!td.globally_unique)
+            ++progress;   /* racy, but good enough */
 
         p = do_step(p);
 
         if (p == (objptr_t)-1) {
             push_roots();
+            end_gut();
 
             long call_fork = (arg != NULL && *(long *)arg);
             if (call_fork == 0) {   /* common case */
-                stm_commit_transaction();
-                td.num_roots_at_transaction_start = td.num_roots;
-                if (get_rand(100) < 98) {
-                    stm_start_transaction(&stm_thread_local);
-                } else {
-                    stm_start_inevitable_transaction(&stm_thread_local);
+                if (get_rand(100) < 50) {
+                    stm_leave_transactional_zone(&stm_thread_local);
+                    /* Nothing here; it's unlikely that a different thread
+                       manages to steal the detached inev transaction.
+                       Give them a little chance with a usleep(). */
+                    dprintf(("sleep...\n"));
+                    usleep(1);
+                    dprintf(("sleep done\n"));
+                    td.num_roots_at_transaction_start = td.num_roots;
+                    stm_enter_transactional_zone(&stm_thread_local);
+                }
+                else {
+                    _stm_commit_transaction();
+                    td.num_roots_at_transaction_start = td.num_roots;
+                    _stm_start_transaction(&stm_thread_local);
                 }
                 td.num_roots = td.num_roots_at_transaction_start;
                 p = NULL;
                 pop_roots();
-                reload_roots();
             }
             else {
                 /* run a fork() inside the transaction */
@@ -401,16 +447,17 @@
         }
     }
     push_roots();
-    stm_commit_transaction();
+    end_gut();
+    stm_force_transaction_break(&stm_thread_local);
 
     /* even out the shadow stack before leaveframe: */
-    stm_start_inevitable_transaction(&stm_thread_local);
+    stm_become_inevitable(&stm_thread_local, "before leaveframe");
     while (td.num_roots > 0) {
         td.num_roots--;
         objptr_t t;
         STM_POP_ROOT(stm_thread_local, t);
     }
-    stm_commit_transaction();
+    stm_leave_transactional_zone(&stm_thread_local);
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c
--- a/c8/demo/demo_random2.c
+++ b/c8/demo/demo_random2.c
@@ -8,6 +8,8 @@
 #include <sys/wait.h>
 
 #include "stmgc.h"
+#include "stm/fprintcolor.h"
+#include "stm/fprintcolor.c"
 
 #define NUMTHREADS 3
 #define STEPS_PER_THREAD 50000
@@ -52,8 +54,10 @@
     int active_roots_num;
     long roots_on_ss;
     long roots_on_ss_at_tr_start;
+    long globally_unique;
 };
 __thread struct thread_data td;
+static long progress = 1;
 
 struct thread_data *_get_td(void)
 {
@@ -61,9 +65,16 @@
 }
 
 
+long check_size(long size)
+{
+    assert(size >= sizeof(struct node_s));
+    assert(size <= sizeof(struct node_s) + 4096*70);
+    return size;
+}
+
 ssize_t stmcb_size_rounded_up(struct object_s *ob)
 {
-    return ((struct node_s*)ob)->my_size;
+    return check_size(((struct node_s*)ob)->my_size);
 }
 
 void stmcb_trace(struct object_s *obj, void visit(object_t **))
@@ -73,7 +84,8 @@
 
     /* and the same value at the end: */
     /* note, ->next may be the same as last_next */
-    nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*));
+    nodeptr_t *last_next = (nodeptr_t*)((char*)n + check_size(n->my_size)
+                                        - sizeof(void*));
 
     assert(n->next == *last_next);
 
@@ -193,7 +205,8 @@
         nodeptr_t n = (nodeptr_t)p;
 
         /* and the same value at the end: */
-        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+        nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
         assert(n->next == *last_next);
         n->next = (nodeptr_t)v;
         *last_next = (nodeptr_t)v;
@@ -205,7 +218,8 @@
     nodeptr_t n = (nodeptr_t)p;
 
     /* and the same value at the end: */
-    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+    nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+                                       check_size(n->my_size) - sizeof(void*));
     OPT_ASSERT(n->next == *last_next);
 
     return n->next;
@@ -239,6 +253,7 @@
             sizeof(struct node_s)+32, sizeof(struct node_s)+48,
             sizeof(struct node_s) + (get_rand(100000) & ~15)};
         size_t size = sizes[get_rand(sizeof(sizes) / sizeof(size_t))];
+        size = check_size(size);
         p = stm_allocate(size);
         nodeptr_t n = (nodeptr_t)p;
         n->sig = SIGNATURE;
@@ -296,6 +311,16 @@
     return p;
 }
 
+static void end_gut(void)
+{
+    if (td.globally_unique != 0) {
+        fprintf(stderr, "[GUT END]");
+        assert(progress == td.globally_unique);
+        td.globally_unique = 0;
+        stm_resume_all_other_threads();
+    }
+}
+
 void frame_loop();
 objptr_t do_step(objptr_t p)
 {
@@ -309,13 +334,22 @@
         p = simple_events(p, _r);
     } else if (get_rand(20) == 1) {
         long pushed = push_roots();
-        stm_commit_transaction();
-        td.roots_on_ss_at_tr_start = td.roots_on_ss;
-
-        if (get_rand(100) < 98) {
-            stm_start_transaction(&stm_thread_local);
-        } else {
-            stm_start_inevitable_transaction(&stm_thread_local);
+        end_gut();
+        if (get_rand(100) < 95) {
+            stm_leave_transactional_zone(&stm_thread_local);
+            /* Nothing here; it's unlikely that a different thread
+               manages to steal the detached inev transaction.
+               Give them a little chance with a usleep(). */
+            dprintf(("sleep...\n"));
+            usleep(1);
+            dprintf(("sleep done\n"));
+            td.roots_on_ss_at_tr_start = td.roots_on_ss;
+            stm_enter_transactional_zone(&stm_thread_local);
+        }
+        else {
+            _stm_commit_transaction();
+            td.roots_on_ss_at_tr_start = td.roots_on_ss;
+            _stm_start_transaction(&stm_thread_local);
         }
         td.roots_on_ss = td.roots_on_ss_at_tr_start;
         td.active_roots_num = 0;
@@ -331,15 +365,21 @@
     } else if (get_rand(20) == 1) {
         long pushed = push_roots();
         stm_become_inevitable(&stm_thread_local, "please");
-        assert(stm_is_inevitable());
+        assert(stm_is_inevitable(&stm_thread_local));
         pop_roots(pushed);
         p= NULL;
     } else if (get_rand(20) == 1) {
         p = (objptr_t)-1; // possibly fork
-    } else if (get_rand(20) == 1) {
+    } else if (get_rand(100) == 1) {
         long pushed = push_roots();
-        stm_become_globally_unique_transaction(&stm_thread_local, "really");
-        fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        if (td.globally_unique == 0) {
+            stm_stop_all_other_threads();
+            td.globally_unique = progress;
+            fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+        }
+        else {
+            end_gut();
+        }
         pop_roots(pushed);
         p = NULL;
     }
@@ -364,6 +404,8 @@
 
         p = do_step(p);
 
+        if (!td.globally_unique)
+            ++progress;   /* racy, but good enough */
 
         if (p == (objptr_t)-1) {
             p = NULL;
@@ -371,6 +413,7 @@
             long call_fork = (thread_may_fork != NULL && *(long *)thread_may_fork);
             if (call_fork) {   /* common case */
                 long pushed = push_roots();
+                end_gut();
                 /* run a fork() inside the transaction */
                 printf("==========   FORK  =========\n");
                 *(long*)thread_may_fork = 0;
@@ -426,7 +469,7 @@
     setup_thread();
 
     td.roots_on_ss_at_tr_start = 0;
-    stm_start_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
     td.roots_on_ss = td.roots_on_ss_at_tr_start;
     td.active_roots_num = 0;
 
@@ -435,7 +478,8 @@
         frame_loop();
     }
 
-    stm_commit_transaction();
+    end_gut();
+    stm_leave_transactional_zone(&stm_thread_local);
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/demo_simple.c b/c8/demo/demo_simple.c
--- a/c8/demo/demo_simple.c
+++ b/c8/demo/demo_simple.c
@@ -70,18 +70,20 @@
 
     object_t *tmp;
     int i = 0;
+
+    stm_enter_transactional_zone(&stm_thread_local);
     while (i < ITERS) {
-        stm_start_transaction(&stm_thread_local);
         tl_counter++;
         if (i % 500 < 250)
             STM_PUSH_ROOT(stm_thread_local, stm_allocate(16));//gl_counter++;
         else
             STM_POP_ROOT(stm_thread_local, tmp);
-        stm_commit_transaction();
+        stm_force_transaction_break(&stm_thread_local);
         i++;
     }
 
     OPT_ASSERT(org == (char *)stm_thread_local.shadowstack);
+    stm_leave_transactional_zone(&stm_thread_local);
 
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/test_shadowstack.c b/c8/demo/test_shadowstack.c
--- a/c8/demo/test_shadowstack.c
+++ b/c8/demo/test_shadowstack.c
@@ -43,17 +43,16 @@
     stm_register_thread_local(&stm_thread_local);
     stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
 
-    stm_start_transaction(&stm_thread_local);
+    stm_enter_transactional_zone(&stm_thread_local);
     node_t *node = (node_t *)stm_allocate(sizeof(struct node_s));
     node->value = 129821;
     STM_PUSH_ROOT(stm_thread_local, node);
     STM_PUSH_ROOT(stm_thread_local, 333);  /* odd value */
-    stm_commit_transaction();
 
     /* now in a new transaction, pop the node off the shadowstack, but
        then do a major collection.  It should still be found by the
        tracing logic. */
-    stm_start_transaction(&stm_thread_local);
+    stm_force_transaction_break(&stm_thread_local);
     STM_POP_ROOT_RET(stm_thread_local);
     STM_POP_ROOT(stm_thread_local, node);
     assert(node->value == 129821);
diff --git a/c8/stm/atomic.h b/c8/stm/atomic.h
--- a/c8/stm/atomic.h
+++ b/c8/stm/atomic.h
@@ -24,24 +24,37 @@
 
 #if defined(__i386__) || defined(__amd64__)
 
-# define HAVE_FULL_EXCHANGE_INSN
   static inline void spin_loop(void) { asm("pause" : : : "memory"); }
   static inline void write_fence(void) { asm("" : : : "memory"); }
+/*# define atomic_exchange(ptr, old, new)  do {         \
+          (old) = __sync_lock_test_and_set(ptr, new);   \
+      } while (0)*/
 
 #else
 
   static inline void spin_loop(void) { asm("" : : : "memory"); }
   static inline void write_fence(void) { __sync_synchronize(); }
 
+/*# define atomic_exchange(ptr, old, new)  do {           \
+          (old) = *(ptr);                                 \
+      } while (UNLIKELY(!__sync_bool_compare_and_swap(ptr, old, new))); */
+
 #endif
 
 
-#define spinlock_acquire(lock)                                          \
-    do { if (LIKELY(__sync_lock_test_and_set(&(lock), 1) == 0)) break;  \
-         spin_loop(); } while (1)
-#define spinlock_release(lock)                                          \
-    do { assert((lock) == 1);                                           \
-         __sync_lock_release(&(lock)); } while (0)
+static inline void _spinlock_acquire(uint8_t *plock) {
+ retry:
+    if (__builtin_expect(__sync_lock_test_and_set(plock, 1) != 0, 0)) {
+        spin_loop();
+        goto retry;
+    }
+}
+static inline void _spinlock_release(uint8_t *plock) {
+    assert(*plock == 1);
+    __sync_lock_release(plock);
+}
+#define spinlock_acquire(lock) _spinlock_acquire(&(lock))
+#define spinlock_release(lock) _spinlock_release(&(lock))
 
 
 #endif  /* _STM_ATOMIC_H */
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -50,8 +50,8 @@
     char *src_segment_base = (from_segnum >= 0 ? get_segment_base(from_segnum)
                                                : NULL);
 
-    assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock));
-    assert(STM_PSEGMENT->modification_lock);
+    assert(IMPLY(from_segnum >= 0, modification_lock_check_rdlock(from_segnum)));
+    assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
 
     long my_segnum = STM_SEGMENT->segment_num;
     DEBUG_EXPECT_SEGFAULT(false);
@@ -131,7 +131,7 @@
                            struct stm_commit_log_entry_s *from,
                            struct stm_commit_log_entry_s *to)
 {
-    assert(STM_PSEGMENT->modification_lock);
+    assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
     assert(from->rev_num >= to->rev_num);
     /* walk BACKWARDS the commit log and update the page 'pagenum',
        initially at revision 'from', until we reach the revision 'to'. */
@@ -199,8 +199,8 @@
 
     /* before copying anything, acquire modification locks from our and
        the other segment */
-    uint64_t to_lock = (1UL << copy_from_segnum)| (1UL << my_segnum);
-    acquire_modification_lock_set(to_lock);
+    uint64_t to_lock = (1UL << copy_from_segnum);
+    acquire_modification_lock_set(to_lock, my_segnum);
     pagecopy(get_virtual_page(my_segnum, pagenum),
              get_virtual_page(copy_from_segnum, pagenum));
 
@@ -223,7 +223,7 @@
     if (src_version->rev_num > target_version->rev_num)
         go_to_the_past(pagenum, src_version, target_version);
 
-    release_modification_lock_set(to_lock);
+    release_modification_lock_set(to_lock, my_segnum);
     release_all_privatization_locks();
 }
 
@@ -324,10 +324,7 @@
     /* Don't check this 'cl'. This entry is already checked */
 
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
-        //assert(first_cl->next == INEV_RUNNING);
-        /* the above assert may fail when running a major collection
-           while the commit of the inevitable transaction is in progress
-           and the element is already attached */
+        assert(first_cl->next == INEV_RUNNING);
         return true;
     }
 
@@ -357,7 +354,7 @@
         }
 
         /* Find the set of segments we need to copy from and lock them: */
-        uint64_t segments_to_lock = 1UL << my_segnum;
+        uint64_t segments_to_lock = 0;
         cl = first_cl;
         while ((next_cl = cl->next) != NULL) {
             if (next_cl == INEV_RUNNING) {
@@ -375,8 +372,8 @@
 
         /* HERE */
 
-        acquire_privatization_lock(STM_SEGMENT->segment_num);
-        acquire_modification_lock_set(segments_to_lock);
+        acquire_privatization_lock(my_segnum);
+        acquire_modification_lock_set(segments_to_lock, my_segnum);
 
 
         /* import objects from first_cl to last_cl: */
@@ -466,8 +463,8 @@
         }
 
         /* done with modifications */
-        release_modification_lock_set(segments_to_lock);
-        release_privatization_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_set(segments_to_lock, my_segnum);
+        release_privatization_lock(my_segnum);
     }
 
     return !needs_abort;
@@ -494,115 +491,121 @@
 }
 
 
-static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old)
-{
-    timing_event(STM_SEGMENT->running_thread, STM_WAIT_OTHER_INEVITABLE);
-
-    while (old->next == INEV_RUNNING && !safe_point_requested()) {
-        spin_loop();
-        usleep(10);    /* XXXXXX */
-    }
-    timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
-}
-
 static void reset_wb_executed_flags(void);
 static void readd_wb_executed_flags(void);
 static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
 
+static void wait_for_inevitable(void)
+{
+    intptr_t detached = 0;
+
+    s_mutex_lock();
+ wait_some_more:
+    if (safe_point_requested()) {
+        /* XXXXXX if the safe point below aborts, in
+           _validate_and_attach(), 'new' leaks */
+        enter_safe_point_if_requested();
+    }
+    else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+        /* loop until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled, but
+           try to detach an inevitable transaction regularly */
+        detached = fetch_detached_transaction();
+        if (detached == 0) {
+            EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE);
+            if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001))
+                goto wait_some_more;
+        }
+    }
+    EMIT_WAIT_DONE();
+    s_mutex_unlock();
+
+    if (detached != 0)
+        commit_fetched_detached_transaction(detached);
+}
+
+/* This is called to do stm_validate() and then attach 'new' at the
+   head of the 'commit_log_root' chained list.  This function sleeps
+   and retries until it succeeds or aborts.
+*/
 static void _validate_and_attach(struct stm_commit_log_entry_s *new)
 {
     struct stm_commit_log_entry_s *old;
 
     OPT_ASSERT(new != NULL);
-    /* we are attaching a real CL entry: */
-    bool is_commit = new != INEV_RUNNING;
+    OPT_ASSERT(new != INEV_RUNNING);
 
-    while (1) {
-        if (!_stm_validate()) {
-            if (new != INEV_RUNNING)
-                free_cle((struct stm_commit_log_entry_s*)new);
-            stm_abort_transaction();
-        }
+    soon_finished_or_inevitable_thread_segment();
+
+ retry_from_start:
+    if (!_stm_validate()) {
+        free_cle(new);
+        stm_abort_transaction();
+    }
 
 #if STM_TESTS
-        if (STM_PSEGMENT->transaction_state != TS_INEVITABLE
-            && STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-            /* abort for tests... */
-            stm_abort_transaction();
-        }
+    if (STM_PSEGMENT->transaction_state != TS_INEVITABLE
+        && STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+        /* abort for tests... */
+        stm_abort_transaction();
+    }
 #endif
 
-        if (is_commit) {
-            /* we must not remove the WB_EXECUTED flags before validation as
-               it is part of a condition in import_objects() called by
-               copy_bk_objs_in_page_from to not overwrite our modifications.
-               So we do it here: */
-            reset_wb_executed_flags();
-            check_all_write_barrier_flags(STM_SEGMENT->segment_base,
-                                          STM_PSEGMENT->modified_old_objects);
-
-            /* need to remove the entries in modified_old_objects "at the same
-               time" as the attach to commit log. Otherwise, another thread may
-               see the new CL entry, import it, look for backup copies in this
-               segment and find the old backup copies! */
-            acquire_modification_lock(STM_SEGMENT->segment_num);
-        }
-
-        /* try to attach to commit log: */
-        old = STM_PSEGMENT->last_commit_log_entry;
-        if (old->next == NULL) {
-            if (new != INEV_RUNNING) /* INEVITABLE */
-                new->rev_num = old->rev_num + 1;
-
-            if (__sync_bool_compare_and_swap(&old->next, NULL, new))
-                break;   /* success! */
-        }
-
-        if (is_commit) {
-            release_modification_lock(STM_SEGMENT->segment_num);
-            /* XXX: unfortunately, if we failed to attach our CL entry,
-               we have to re-add the WB_EXECUTED flags before we try to
-               validate again because of said condition (s.a) */
-            readd_wb_executed_flags();
-        }
-
-        if (old->next == INEV_RUNNING && !safe_point_requested()) {
-            /* we failed because there is an INEV transaction running */
-            /* XXXXXX for now just sleep.  We should really ask to inev
-               transaction to do the commit for us, and then we can
-               continue running. */
-            dprintf(("_validate_and_attach(%p) failed, "
-                     "waiting for inevitable\n", new));
-            wait_for_other_inevitable(old);
-        }
-
-        dprintf(("_validate_and_attach(%p) failed, enter safepoint\n", new));
-
-        /* check for requested safe point. otherwise an INEV transaction
-           may try to commit but cannot because of the busy-loop here. */
-        /* minor gc is fine here because we did one immediately before, so
-           there are no young objs anyway. major gc is fine because the
-           modified_old_objects list is still populated with the same
-           cl-entry objs */
-        /* XXXXXXXX: memory leak if we happen to do a major gc, we get aborted
-           in major_do_validation_and_minor_collections, and don't free 'new' */
-        _stm_collectable_safe_point();
+    if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+        wait_for_inevitable();
+        goto retry_from_start;   /* redo _stm_validate() now */
     }
 
-    if (is_commit) {
+    /* we must not remove the WB_EXECUTED flags before validation as
+       it is part of a condition in import_objects() called by
+       copy_bk_objs_in_page_from to not overwrite our modifications.
+       So we do it here: */
+    reset_wb_executed_flags();
+    check_all_write_barrier_flags(STM_SEGMENT->segment_base,
+                                  STM_PSEGMENT->modified_old_objects);
+
+    /* need to remove the entries in modified_old_objects "at the same
+       time" as the attach to commit log. Otherwise, another thread may
+       see the new CL entry, import it, look for backup copies in this
+       segment and find the old backup copies! */
+    acquire_modification_lock_wr(STM_SEGMENT->segment_num);
+
+    /* try to attach to commit log: */
+    old = STM_PSEGMENT->last_commit_log_entry;
+    new->rev_num = old->rev_num + 1;
+    if (__sync_bool_compare_and_swap(&old->next, NULL, new)) {
+        /* success! */
         /* compare with _validate_and_add_to_commit_log */
-        STM_PSEGMENT->transaction_state = TS_NONE;
-        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
-
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
-        release_modification_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
+    }
+    else {
+        /* fail */
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
+        /* XXX: unfortunately, if we failed to attach our CL entry,
+           we have to re-add the WB_EXECUTED flags before we try to
+           validate again because of said condition (s.a) */
+        readd_wb_executed_flags();
+
+        dprintf(("_validate_and_attach(%p) failed, retrying\n", new));
+        goto retry_from_start;
     }
 }
 
-static void _validate_and_turn_inevitable(void)
+/* This is called to do stm_validate() and then attach INEV_RUNNING to
+   the head of the 'commit_log_root' chained list.  This function
+   may succeed or fail (or abort).
+*/
+static bool _validate_and_turn_inevitable(void)
 {
-    _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING);
+    struct stm_commit_log_entry_s *old;
+
+    if (!_stm_validate())
+        stm_abort_transaction();
+
+    /* try to attach to commit log: */
+    old = STM_PSEGMENT->last_commit_log_entry;
+    return __sync_bool_compare_and_swap(&old->next, NULL, INEV_RUNNING);
 }
 
 static void _validate_and_add_to_commit_log(void)
@@ -611,6 +614,8 @@
 
     new = _create_commit_log_entry();
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+        assert(_stm_detached_inevitable_from_thread == 0);  /* running it */
+
         old = STM_PSEGMENT->last_commit_log_entry;
         new->rev_num = old->rev_num + 1;
         OPT_ASSERT(old->next == INEV_RUNNING);
@@ -621,14 +626,15 @@
                                       STM_PSEGMENT->modified_old_objects);
 
         /* compare with _validate_and_attach: */
-        STM_PSEGMENT->transaction_state = TS_NONE;
-        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+        acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
 
         /* do it: */
         bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
         OPT_ASSERT(yes);
+
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
     }
     else {
         _validate_and_attach(new);
@@ -692,7 +698,7 @@
         increment_total_allocated(slice_sz);
         memcpy(bk_slice, realobj + slice_off, slice_sz);
 
-        acquire_modification_lock(STM_SEGMENT->segment_num);
+        acquire_modification_lock_wr(STM_SEGMENT->segment_num);
         /* !! follows layout of "struct stm_undo_s" !! */
         STM_PSEGMENT->modified_old_objects = list_append3(
             STM_PSEGMENT->modified_old_objects,
@@ -700,7 +706,7 @@
             (uintptr_t)bk_slice,  /* bk_addr */
             NEW_SLICE(slice_off, slice_sz));
         dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz));
-        release_modification_lock(STM_SEGMENT->segment_num);
+        release_modification_lock_wr(STM_SEGMENT->segment_num);
 
         slice_off += slice_sz;
     }
@@ -896,6 +902,8 @@
 
 static void touch_all_pages_of_obj(object_t *obj, size_t obj_size)
 {
+    /* XXX should it be simpler, just really trying to read a dummy
+       byte in each page? */
     int my_segnum = STM_SEGMENT->segment_num;
     uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
 
@@ -1121,11 +1129,12 @@
 
 
 
-static void _stm_start_transaction(stm_thread_local_t *tl)
+static void _do_start_transaction(stm_thread_local_t *tl)
 {
     assert(!_stm_in_transaction(tl));
+    tl->wait_event_emitted = 0;
 
-    while (!acquire_thread_segment(tl)) {}
+    acquire_thread_segment(tl);
     /* GS invalid before this point! */
 
     assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
@@ -1138,7 +1147,9 @@
 #endif
     STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack;
     STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj;
-
+    STM_PSEGMENT->total_throw_away_nursery = 0;
+    assert(tl->self_or_0_if_atomic == (intptr_t)tl);   /* not atomic */
+    assert(STM_PSEGMENT->atomic_nesting_levels == 0);
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
@@ -1150,6 +1161,7 @@
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
     assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
     assert(STM_PSEGMENT->finalizers == NULL);
+    assert(STM_PSEGMENT->active_queues == NULL);
 #ifndef NDEBUG
     /* this should not be used when objects_pointing_to_nursery == NULL */
     STM_PSEGMENT->position_markers_len_old = 99999999999999999L;
@@ -1179,35 +1191,34 @@
     stm_validate();
 }
 
-long stm_start_transaction(stm_thread_local_t *tl)
+#ifdef STM_NO_AUTOMATIC_SETJMP
+static int did_abort = 0;
+#endif
+
+long _stm_start_transaction(stm_thread_local_t *tl)
 {
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
-    long repeat_count = 0;    /* test/support.py */
+    long repeat_count = did_abort;    /* test/support.py */
+    did_abort = 0;
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
-    _stm_start_transaction(tl);
+    _do_start_transaction(tl);
+
+    if (repeat_count == 0) {  /* else, 'nursery_mark' was already set
+                                 in abort_data_structures_from_segment_num() */
+        STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
+                                     stm_fill_mark_nursery_bytes);
+    }
     return repeat_count;
 }
 
-void stm_start_inevitable_transaction(stm_thread_local_t *tl)
-{
-    /* used to be more efficient, starting directly an inevitable transaction,
-       but there is no real point any more, I believe */
-    rewind_jmp_buf rjbuf;
-    stm_rewind_jmp_enterframe(tl, &rjbuf);
-
-    stm_start_transaction(tl);
-    stm_become_inevitable(tl, "start_inevitable_transaction");
-
-    stm_rewind_jmp_leaveframe(tl, &rjbuf);
-}
-
 #ifdef STM_NO_AUTOMATIC_SETJMP
 void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn));
-int stm_is_inevitable(void)
+int stm_is_inevitable(stm_thread_local_t *tl)
 {
+    assert(STM_SEGMENT->running_thread == tl);
     switch (STM_PSEGMENT->transaction_state) {
     case TS_REGULAR: return 0;
     case TS_INEVITABLE: return 1;
@@ -1222,6 +1233,7 @@
 {
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
 
+    assert(_has_mutex());
     STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
     STM_PSEGMENT->transaction_state = TS_NONE;
 
@@ -1229,7 +1241,15 @@
     list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
     list_clear(STM_PSEGMENT->old_objects_with_cards_set);
     list_clear(STM_PSEGMENT->large_overflow_objects);
-    timing_event(tl, event);
+    if (tl != NULL)
+        timing_event(tl, event);
+
+    /* If somebody is waiting for us to reach a safe point, we simply
+       signal it now and leave this transaction.  This should be enough
+       for synchronize_all_threads() to retry and notice that we are
+       no longer SP_RUNNING. */
+    if (STM_SEGMENT->nursery_end != NURSERY_END)
+        cond_signal(C_AT_SAFE_POINT);
 
     release_thread_segment(tl);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -1278,24 +1298,62 @@
 }
 
 
-void stm_commit_transaction(void)
+void _stm_commit_transaction(void)
+{
+    assert(STM_PSEGMENT->running_pthread == pthread_self());
+    _core_commit_transaction(/*external=*/ false);
+}
+
+static void _core_commit_transaction(bool external)
 {
     exec_local_finalizers();
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
-    assert(STM_PSEGMENT->running_pthread == pthread_self());
+    assert(STM_PSEGMENT->transaction_state != TS_NONE);
+    if (globally_unique_transaction) {
+        stm_fatalerror("cannot commit between stm_stop_all_other_threads "
+                       "and stm_resume_all_other_threads");
+    }
+    if (STM_PSEGMENT->atomic_nesting_levels > 0) {
+        stm_fatalerror("cannot commit between stm_enable_atomic "
+                       "and stm_disable_atomic");
+    }
+    assert(STM_SEGMENT->running_thread->self_or_0_if_atomic ==
+           (intptr_t)(STM_SEGMENT->running_thread));
+    assert(STM_SEGMENT->running_thread->wait_event_emitted == 0);
 
-    dprintf(("> stm_commit_transaction()\n"));
-    minor_collection(1);
+    dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
+    minor_collection(/*commit=*/ true, external);
+    if (!external && is_major_collection_requested()) {
+        s_mutex_lock();
+        if (is_major_collection_requested()) {   /* if still true */
+            major_collection_with_mutex();
+        }
+        s_mutex_unlock();
+    }
 
     push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
 
+    if (external) {
+        /* from this point on, unlink the original 'stm_thread_local_t *'
+           from its segment.  Better do it as soon as possible, because
+           other threads might be spin-looping, waiting for the -1 to
+           disappear. */
+        STM_SEGMENT->running_thread = NULL;
+        write_fence();
+        assert(_stm_detached_inevitable_from_thread == -1);
+        _stm_detached_inevitable_from_thread = 0;
+    }
+
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
-    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    if (!was_inev) {
+        assert(!external);
+        stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    }
 
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
@@ -1310,25 +1368,15 @@
         STM_PSEGMENT->overflow_number_has_been_used = false;
     }
 
+    if (STM_PSEGMENT->active_queues)
+        queues_deactivate_all(get_priv_segment(STM_SEGMENT->segment_num),
+                              /*at_commit=*/true);
+
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
-    /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/
-    enter_safe_point_if_requested();
-    assert(STM_SEGMENT->nursery_end == NURSERY_END);
-
-    /* if a major collection is required, do it here */
-    if (is_major_collection_requested()) {
-        major_collection_with_mutex();
-    }
-
-    _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
-
-    if (globally_unique_transaction && was_inev) {
-        committed_globally_unique_transaction();
-    }
-
     /* done */
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+    assert(external == (tl == NULL));
     _finish_transaction(STM_TRANSACTION_COMMIT);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 
@@ -1336,7 +1384,8 @@
 
     /* between transactions, call finalizers. this will execute
        a transaction itself */
-    invoke_general_finalizers(tl);
+    if (tl != NULL)
+        invoke_general_finalizers(tl);
 }
 
 static void reset_modified_from_backup_copies(int segment_num)
@@ -1345,7 +1394,7 @@
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
-    assert(get_priv_segment(segment_num)->modification_lock);
+    assert(modification_lock_check_wrlock(segment_num));
 
     struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
     struct list_s *list = pseg->modified_old_objects;
@@ -1397,7 +1446,7 @@
 
     abort_finalizers(pseg);
 
-    long bytes_in_nursery = throw_away_nursery(pseg);
+    throw_away_nursery(pseg);
 
     /* clear CARD_MARKED on objs (don't care about CARD_MARKED_OLD) */
     LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/,
@@ -1407,9 +1456,9 @@
             _reset_object_cards(pseg, item, CARD_CLEAR, false, false);
         });
 
-    acquire_modification_lock(segment_num);
+    acquire_modification_lock_wr(segment_num);
     reset_modified_from_backup_copies(segment_num);
-    release_modification_lock(segment_num);
+    release_modification_lock_wr(segment_num);
     _verify_cards_cleared_in_all_lists(pseg);
 
     stm_thread_local_t *tl = pseg->pub.running_thread;
@@ -1431,7 +1480,29 @@
     assert(tl->shadowstack == pseg->shadowstack_at_start_of_transaction);
 #endif
     tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
-    tl->last_abort__bytes_in_nursery = bytes_in_nursery;
+
+    if (pseg->active_queues)
+        queues_deactivate_all(pseg, /*at_commit=*/false);
+
+
+    /* Set the next nursery_mark: first compute the value that
+       nursery_mark must have had at the start of the aborted transaction */
+    stm_char *old_mark =pseg->pub.nursery_mark + pseg->total_throw_away_nursery;
+
+    /* This means that the limit, in term of bytes, was: */
+    uintptr_t old_limit = old_mark - (stm_char *)_stm_nursery_start;
+
+    /* If 'total_throw_away_nursery' is smaller than old_limit, use that */
+    if (pseg->total_throw_away_nursery < old_limit)
+        old_limit = pseg->total_throw_away_nursery;
+
+    /* Now set the new limit to 90% of the old limit */
+    pseg->pub.nursery_mark = ((stm_char *)_stm_nursery_start +
+                              (uintptr_t)(old_limit * 0.9));
+
+#ifdef STM_NO_AUTOMATIC_SETJMP
+    did_abort = 1;
+#endif
 
     list_clear(pseg->objects_pointing_to_nursery);
     list_clear(pseg->old_objects_with_cards_set);
@@ -1452,6 +1523,8 @@
     abort_data_structures_from_segment_num(STM_SEGMENT->segment_num);
 
     stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+    tl->self_or_0_if_atomic = (intptr_t)tl;   /* clear the 'atomic' flag */
+    STM_PSEGMENT->atomic_nesting_levels = 0;
 
     if (tl->mem_clear_on_abort)
         memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort);
@@ -1500,36 +1573,84 @@
 
 void _stm_become_inevitable(const char *msg)
 {
-    if (STM_PSEGMENT->transaction_state == TS_REGULAR) {
+    int num_waits = 0;
+
+    timing_become_inevitable();
+
+ retry_from_start:
+    assert(STM_PSEGMENT->transaction_state == TS_REGULAR);
+    _stm_collectable_safe_point();
+
+    if (msg != MSG_INEV_DONT_SLEEP) {
         dprintf(("become_inevitable: %s\n", msg));
-        _stm_collectable_safe_point();
-        timing_become_inevitable();
 
-        _validate_and_turn_inevitable();
-        STM_PSEGMENT->transaction_state = TS_INEVITABLE;
+        if (any_soon_finished_or_inevitable_thread_segment() &&
+                num_waits <= NB_SEGMENTS) {
+#if STM_TESTS                           /* for tests: another transaction */
+            stm_abort_transaction();    /*   is already inevitable, abort */
+#endif
 
-        stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
-        invoke_and_clear_user_callbacks(0);   /* for commit */
+            bool timed_out = false;
+
+            s_mutex_lock();
+            if (any_soon_finished_or_inevitable_thread_segment() &&
+                    !safe_point_requested()) {
+
+                /* wait until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled */
+                EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE);
+                if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
+                                       0.000054321))
+                    timed_out = true;
+            }
+            s_mutex_unlock();
+
+            if (timed_out) {
+                /* try to detach another inevitable transaction, but
+                   only after waiting a bit.  This is necessary to avoid
+                   deadlocks in some situations, which are hopefully
+                   not too common.  We don't want two threads constantly
+                   detaching each other. */
+                intptr_t detached = fetch_detached_transaction();
+                if (detached != 0) {
+                    EMIT_WAIT_DONE();
+                    commit_fetched_detached_transaction(detached);
+                }
+            }
+            else {
+                num_waits++;
+            }
+            goto retry_from_start;
+        }
+        EMIT_WAIT_DONE();
+        if (!_validate_and_turn_inevitable())
+            goto retry_from_start;
     }
     else {
-        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        if (!_validate_and_turn_inevitable())
+            return;
     }
+    soon_finished_or_inevitable_thread_segment();
+    STM_PSEGMENT->transaction_state = TS_INEVITABLE;
+
+    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    invoke_and_clear_user_callbacks(0);   /* for commit */
 }
 
+#if 0
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl,
                                             const char *msg)
 {
-    stm_become_inevitable(tl, msg);   /* may still abort */
+    stm_become_inevitable(tl, msg);
 
     s_mutex_lock();
     synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
     s_mutex_unlock();
 }
-
+#endif
 
 void stm_stop_all_other_threads(void)
 {
-    if (!stm_is_inevitable())         /* may still abort */
+    if (!stm_is_inevitable(STM_SEGMENT->running_thread))  /* may still abort */
         _stm_become_inevitable("stop_all_other_threads");
 
     s_mutex_lock();
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -74,11 +74,6 @@
 struct stm_priv_segment_info_s {
     struct stm_segment_info_s pub;
 
-    /* lock protecting from concurrent modification of
-       'modified_old_objects', page-revision-changes, ...
-       Always acquired in global order of segments to avoid deadlocks. */
-    uint8_t modification_lock;
-
     /* All the old objects (older than the current transaction) that
        the current transaction attempts to modify.  This is used to
        track the STM status: these are old objects that where written
@@ -122,13 +117,15 @@
     bool minor_collect_will_commit_now;
 
     struct tree_s *callbacks_on_commit_and_abort[2];
+    struct tree_s *active_queues;
+    uint8_t active_queues_lock;
 
     /* This is the number stored in the overflowed objects (a multiple of
        GCFLAG_OVERFLOW_NUMBER_bit0).  It is incremented when the
        transaction is done, but only if we actually overflowed any
        object; otherwise, no object has got this number. */
+    bool overflow_number_has_been_used;
     uint32_t overflow_number;
-    bool overflow_number_has_been_used;
 
 
     struct stm_commit_log_entry_s *last_commit_log_entry;
@@ -157,6 +154,12 @@
     stm_char *sq_fragments[SYNC_QUEUE_SIZE];
     int sq_fragsizes[SYNC_QUEUE_SIZE];
     int sq_len;
+
+    /* For nursery_mark */
+    uintptr_t total_throw_away_nursery;
+
+    /* For stm_enable_atomic() */
+    uintptr_t atomic_nesting_levels;
 };
 
 enum /* safe_point */ {
@@ -164,6 +167,7 @@
     SP_RUNNING,
     SP_WAIT_FOR_C_REQUEST_REMOVED,
     SP_WAIT_FOR_C_AT_SAFE_POINT,
+    SP_WAIT_FOR_INEV,
 #ifdef STM_TESTS
     SP_WAIT_FOR_OTHER_THREAD,
 #endif
@@ -175,6 +179,12 @@
     TS_INEVITABLE,
 };
 
+#define MSG_INEV_DONT_SLEEP  ((const char *)1)
+

From noreply at buildbot.pypy.org  Thu Jul  2 13:10:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 13:10:29 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: hg backout a49dca73c968: cancel the
 original backout of this branch
Message-ID: <20150702111029.35C771C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1886:da2e7294405a
Date: 2015-07-02 11:46 +0100
http://bitbucket.org/pypy/stmgc/changeset/da2e7294405a/

Log:	hg backout a49dca73c968: cancel the original backout of this branch

diff --git a/c7/demo/Makefile b/c7/demo/Makefile
--- a/c7/demo/Makefile
+++ b/c7/demo/Makefile
@@ -19,18 +19,20 @@
 
 COMMON = -I.. -pthread -lrt -g -Wall -Werror -DSTM_LARGEMALLOC_TEST
 
+CC = gcc-seg-gs
+
 
 # note that 'build' is partially optimized but still contains all asserts
 debug-%: %.c ${H_FILES} ${C_FILES}
-	clang $(COMMON) -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -O0 \
+	$(CC) $(COMMON) -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -O0 \
         $< -o debug-$* ../stmgc.c
 
 build-%: %.c ${H_FILES} ${C_FILES}
-	clang $(COMMON) -DSTM_GC_NURSERY=128 -O1 $< -o build-$* ../stmgc.c
+	$(CC) $(COMMON) -DSTM_GC_NURSERY=128 -O1 $< -o build-$* ../stmgc.c
 
 release-%: %.c ${H_FILES} ${C_FILES}
-	clang $(COMMON) -DNDEBUG -O2 $< -o release-$* ../stmgc.c
+	$(CC) $(COMMON) -DNDEBUG -O2 $< -o release-$* ../stmgc.c
 
 
 release-htm-%: %.c ../../htm-c7/stmgc.? ../../htm-c7/htm.h
-	clang $(COMMON) -O2 $< -o release-htm-$* ../../htm-c7/stmgc.c -DUSE_HTM
+	$(CC) $(COMMON) -O2 $< -o release-htm-$* ../../htm-c7/stmgc.c -DUSE_HTM
diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c
--- a/c7/demo/demo2.c
+++ b/c7/demo/demo2.c
@@ -216,7 +216,7 @@
 
 void teardown_list(void)
 {
-    STM_POP_ROOT_RET(stm_thread_local);
+    STM_POP_ROOT_DROP(stm_thread_local);
 }
 
 
@@ -256,6 +256,7 @@
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     unregister_thread_local();
     status = sem_post(&done); assert(status == 0);
+    (void)status;
     return NULL;
 }
 
@@ -293,6 +294,7 @@
     rewind_jmp_buf rjbuf;
 
     status = sem_init(&done, 0, 0); assert(status == 0);
+    (void)status;
 
     stm_setup();
     stm_register_thread_local(&stm_thread_local);
diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c
--- a/c7/demo/demo_random.c
+++ b/c7/demo/demo_random.c
@@ -412,6 +412,7 @@
     stm_unregister_thread_local(&stm_thread_local);
 
     status = sem_post(&done); assert(status == 0);
+    (void)status;
     return NULL;
 }
 
diff --git a/c7/demo/demo_random2.c b/c7/demo/demo_random2.c
--- a/c7/demo/demo_random2.c
+++ b/c7/demo/demo_random2.c
@@ -435,6 +435,7 @@
     stm_unregister_thread_local(&stm_thread_local);
 
     status = sem_post(&done); assert(status == 0);
+    (void)status;
     return NULL;
 }
 
diff --git a/c7/demo/test_shadowstack.c b/c7/demo/test_shadowstack.c
--- a/c7/demo/test_shadowstack.c
+++ b/c7/demo/test_shadowstack.c
@@ -54,7 +54,7 @@
        then do a major collection.  It should still be found by the
        tracing logic. */
     stm_start_transaction(&stm_thread_local);
-    STM_POP_ROOT_RET(stm_thread_local);
+    STM_POP_ROOT_DROP(stm_thread_local);
     STM_POP_ROOT(stm_thread_local, node);
     assert(node->value == 129821);
     STM_PUSH_ROOT(stm_thread_local, NULL);
diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -45,7 +45,6 @@
 #endif
 }
 
-__attribute__((always_inline))
 static void write_slowpath_overflow_obj(object_t *obj, bool mark_card)
 {
     /* An overflow object is an object from the same transaction, but
@@ -79,7 +78,6 @@
     }
 }
 
-__attribute__((always_inline))
 static void write_slowpath_common(object_t *obj, bool mark_card)
 {
     assert(_seems_to_be_running_transaction());
@@ -223,6 +221,7 @@
     check_flag_write_barrier(obj);
 }
 
+__attribute__((flatten))
 void _stm_write_slowpath(object_t *obj)
 {
     write_slowpath_common(obj, /*mark_card=*/false);
@@ -241,6 +240,7 @@
     return (size >= _STM_MIN_CARD_OBJ_SIZE);
 }
 
+__attribute__((flatten))
 char _stm_write_slowpath_card_extra(object_t *obj)
 {
     /* the PyPy JIT calls this function directly if it finds that an
diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -58,7 +58,7 @@
     /* Make a new mmap at some other address, but of the same size as
        the standard mmap at stm_object_pages
     */
-    int big_copy_fd;
+    int big_copy_fd = -1;
     char *big_copy = setup_mmap("stmgc's fork support", &big_copy_fd);
 
     /* Copy all the data from the two ranges of objects (large, small)
diff --git a/c7/stm/fprintcolor.c b/c7/stm/fprintcolor.c
--- a/c7/stm/fprintcolor.c
+++ b/c7/stm/fprintcolor.c
@@ -1,3 +1,5 @@
+#include <stdarg.h>
+
 /* ------------------------------------------------------------ */
 #ifdef STM_DEBUGPRINT
 /* ------------------------------------------------------------ */
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -20,7 +20,15 @@
 #endif
 
 
-#define TLPREFIX __attribute__((address_space(256)))
+#ifdef __SEG_GS     /* on a custom patched gcc */
+#  define TLPREFIX __seg_gs
+#  define _STM_RM_SUFFIX  :8
+#elif defined(__clang__)   /* on a clang, hopefully made bug-free */
+#  define TLPREFIX __attribute__((address_space(256)))
+#  define _STM_RM_SUFFIX  /* nothing */
+#else
+#  error "needs either a GCC with __seg_gs support, or a bug-freed clang"
+#endif
 
 typedef TLPREFIX struct object_s object_t;
 typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t;
@@ -34,11 +42,11 @@
        'STM_SEGMENT->transaction_read_version' if and only if the
        object was read in the current transaction.  The nurseries
        also have corresponding read markers, but they are never used. */
-    uint8_t rm;
+    unsigned char rm _STM_RM_SUFFIX;
 };
 
 struct stm_segment_info_s {
-    uint8_t transaction_read_version;
+    unsigned int transaction_read_version;
     int segment_num;
     char *segment_base;
     stm_char *nursery_current;
@@ -288,6 +296,7 @@
 #define STM_PUSH_ROOT(tl, p)   ((tl).shadowstack++->ss = (object_t *)(p))
 #define STM_POP_ROOT(tl, p)    ((p) = (typeof(p))((--(tl).shadowstack)->ss))
 #define STM_POP_ROOT_RET(tl)   ((--(tl).shadowstack)->ss)
+#define STM_POP_ROOT_DROP(tl)  ((void)(--(tl).shadowstack))
 
 
 /* Every thread needs to have a corresponding stm_thread_local_t
@@ -302,7 +311,12 @@
 
 /* At some key places, like the entry point of the thread and in the
    function with the interpreter's dispatch loop, you need to declare
-   a local variable of type 'rewind_jmp_buf' and call these macros. */
+   a local variable of type 'rewind_jmp_buf' and call these macros.
+   IMPORTANT: a function in which you call stm_rewind_jmp_enterframe()
+   must never change the value of its own arguments!  If they are
+   passed on the stack, gcc can change the value directly there, but
+   we're missing the logic to save/restore this part!
+*/
 #define stm_rewind_jmp_enterprepframe(tl, rjbuf)   \
     rewind_jmp_enterprepframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack)
 #define stm_rewind_jmp_enterframe(tl, rjbuf)       \
@@ -506,7 +520,7 @@
 
 #define STM_POP_MARKER(tl)   ({                 \
     object_t *_popped = STM_POP_ROOT_RET(tl);   \
-    STM_POP_ROOT_RET(tl);                       \
+    STM_POP_ROOT_DROP(tl);                      \
     _popped;                                    \
 })
 
diff --git a/c7/test/common.py b/c7/test/common.py
--- a/c7/test/common.py
+++ b/c7/test/common.py
@@ -3,7 +3,7 @@
 assert sys.maxint == 9223372036854775807, "requires a 64-bit environment"
 
 # ----------
-os.environ['CC'] = 'clang'
+os.environ['CC'] = 'gcc-seg-gs'
 
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
diff --git a/c7/test/support.py b/c7/test/support.py
--- a/c7/test/support.py
+++ b/c7/test/support.py
@@ -478,7 +478,8 @@
                     ],
      undef_macros=['NDEBUG'],
      include_dirs=[parent_dir],
-     extra_compile_args=['-g', '-O0', '-Werror', '-ferror-limit=1'],
+     extra_compile_args=['-g', '-O0', '-Werror',  #, '-ferror-limit=1',  for clang
+                         '-Wfatal-errors'],   # for gcc
      extra_link_args=['-g', '-lrt'],
      force_generic_engine=True)
 
diff --git a/c7/test/test_list.py b/c7/test/test_list.py
--- a/c7/test/test_list.py
+++ b/c7/test/test_list.py
@@ -56,7 +56,7 @@
 ''', define_macros=[('STM_TESTS', '1')],
      undef_macros=['NDEBUG'],
      include_dirs=[parent_dir],
-     extra_compile_args=['-g', '-O0', '-Werror', '-ferror-limit=1'],
+     extra_compile_args=['-g', '-O0', '-Werror'], #, '-ferror-limit=1'],
      force_generic_engine=True)
 
 # ____________________________________________________________
diff --git a/c7/test/test_rewind.c b/c7/test/test_rewind.c
--- a/c7/test/test_rewind.c
+++ b/c7/test/test_rewind.c
@@ -174,12 +174,26 @@
 void foo(int *x) { ++*x; }
 
 __attribute__((noinline))
-void f6(int a1, int a2, int a3, int a4, int a5, int a6, int a7,
-        int a8, int a9, int a10, int a11, int a12, int a13)
+void f6(int c1, int c2, int c3, int c4, int c5, int c6, int c7,
+        int c8, int c9, int c10, int c11, int c12, int c13)
 {
     rewind_jmp_buf buf;
     rewind_jmp_enterframe(&gthread, &buf, NULL);
 
+    int a1 = c1;
+    int a2 = c2;
+    int a3 = c3;
+    int a4 = c4;
+    int a5 = c5;
+    int a6 = c6;
+    int a7 = c7;
+    int a8 = c8;
+    int a9 = c9;
+    int a10 = c10;
+    int a11 = c11;
+    int a12 = c12;
+    int a13 = c13;
+
     rewind_jmp_setjmp(&gthread, NULL);
     gevent(a1); gevent(a2); gevent(a3); gevent(a4);
     gevent(a5); gevent(a6); gevent(a7); gevent(a8);
diff --git a/c7/test/test_rewind.py b/c7/test/test_rewind.py
--- a/c7/test/test_rewind.py
+++ b/c7/test/test_rewind.py
@@ -1,11 +1,11 @@
 import os
 
 def run_test(opt):
-    err = os.system("clang -g -O%s -Werror -DRJBUF_CUSTOM_MALLOC -I../stm"
+    err = os.system("gcc-seg-gs -g -O%s -Werror -DRJBUF_CUSTOM_MALLOC -I../stm"
                     " -o test_rewind_O%s test_rewind.c ../stm/rewind_setjmp.c"
                     % (opt, opt))
     if err != 0:
-        raise OSError("clang failed on test_rewind.c")
+        raise OSError("gcc-seg-gs failed on test_rewind.c")
     for testnum in [1, 2, 3, 4, 5, 6, 7, "TL1", "TL2"]:
         print '=== O%s: RUNNING TEST %s ===' % (opt, testnum)
         err = os.system("./test_rewind_O%s %s" % (opt, testnum))
diff --git a/gcc-seg-gs/README.txt b/gcc-seg-gs/README.txt
new file mode 100644
--- /dev/null
+++ b/gcc-seg-gs/README.txt
@@ -0,0 +1,34 @@
+Get gcc release 5.1.0 from the download page:
+    
+    https://gcc.gnu.org/mirrors.html
+
+Unpack it.
+
+Apply the patch provided here in the file gcc-5.1.0-patch.diff.
+
+You can either install the 'libmpc-dev' package on your system,
+or else, manually:
+    
+    * unpack 'https://ftp.gnu.org/gnu/gmp/gmp-6.0.0a.tar.xz'
+      and move 'gmp-6.0.0' as 'gcc-5.1.0/gmp'.
+
+    * unpack 'http://www.mpfr.org/mpfr-current/mpfr-3.1.2.tar.xz'
+      and move 'mpfr-3.1.2' as 'gcc-5.1.0/mpfr'
+
+    * unpack 'ftp://ftp.gnu.org/gnu/mpc/mpc-1.0.3.tar.gz'
+      and move 'mpc-1.0.3' as 'gcc-5.1.0/mpc'
+
+Compile gcc as usual:
+    
+    mkdir build
+    cd build
+    ../gcc-5.1.0/configure --enable-languages=c --disable-multilib
+    make    # or maybe only "make all-stage1-gcc"
+
+This patched gcc could be globally installed, but in these instructions
+we assume you don't want that.  Instead, create the following script,
+call it 'gcc-seg-gs', and put it in the $PATH:
+
+    #!/bin/bash
+    BUILD=/..../build      # <- insert full path
+    exec $BUILD/gcc/xgcc -B $BUILD/gcc "$@"
diff --git a/gcc-seg-gs/gcc-5.1.0-patch.diff b/gcc-seg-gs/gcc-5.1.0-patch.diff
new file mode 100644
--- /dev/null
+++ b/gcc-seg-gs/gcc-5.1.0-patch.diff
@@ -0,0 +1,269 @@
+Index: gcc/doc/tm.texi.in
+===================================================================
+--- gcc/doc/tm.texi.in	(revision 223859)
++++ gcc/doc/tm.texi.in	(working copy)
+@@ -7424,6 +7424,8 @@
+ 
+ @hook TARGET_ADDR_SPACE_CONVERT
+ 
++ at hook TARGET_ADDR_SPACE_DEFAULT_POINTER_ADDRESS_MODES_P
++
+ @node Misc
+ @section Miscellaneous Parameters
+ @cindex parameters, miscellaneous
+Index: gcc/doc/tm.texi
+===================================================================
+--- gcc/doc/tm.texi	(revision 223859)
++++ gcc/doc/tm.texi	(working copy)
+@@ -10290,6 +10290,17 @@
+ as determined by the @code{TARGET_ADDR_SPACE_SUBSET_P} target hook.
+ @end deftypefn
+ 
++ at deftypefn {Target Hook} bool TARGET_ADDR_SPACE_DEFAULT_POINTER_ADDRESS_MODES_P (void)
++Some places still assume that all pointer or address modes are the
++standard Pmode and ptr_mode.  These optimizations become invalid if
++the target actually supports multiple different modes.  This hook returns
++true if all pointers and addresses are Pmode and ptr_mode, and false
++otherwise.  Called via target_default_pointer_address_modes_p().  The
++default NULL for the hook makes this function return true if the two hooks
++ at code{TARGET_ADDR_SPACE_POINTER_MODE}, @code{TARGET_ADDR_SPACE_ADDRESS_MODE}
++are undefined, and false otherwise.
++ at end deftypefn
++
+ @node Misc
+ @section Miscellaneous Parameters
+ @cindex parameters, miscellaneous
+Index: gcc/target.def
+===================================================================
+--- gcc/target.def	(revision 223859)
++++ gcc/target.def	(working copy)
+@@ -3164,6 +3164,19 @@
+  rtx, (rtx op, tree from_type, tree to_type),
+  default_addr_space_convert)
+ 
++/* True if all pointer or address modes are the standard Pmode and ptr_mode. */
++DEFHOOK
++(default_pointer_address_modes_p,
++ "Some places still assume that all pointer or address modes are the\n\
++standard Pmode and ptr_mode.  These optimizations become invalid if\n\
++the target actually supports multiple different modes.  This hook returns\n\
++true if all pointers and addresses are Pmode and ptr_mode, and false\n\
++otherwise.  Called via target_default_pointer_address_modes_p().  The\n\
++default NULL for the hook makes this function return true if the two hooks\n\
++ at code{TARGET_ADDR_SPACE_POINTER_MODE}, @code{TARGET_ADDR_SPACE_ADDRESS_MODE}\n\
++are undefined, and false otherwise.",
++ bool, (void), NULL)
++
+ HOOK_VECTOR_END (addr_space)
+ 
+ #undef HOOK_PREFIX
+Index: gcc/targhooks.c
+===================================================================
+--- gcc/targhooks.c	(revision 223859)
++++ gcc/targhooks.c	(working copy)
+@@ -1228,6 +1228,9 @@
+ bool
+ target_default_pointer_address_modes_p (void)
+ {
++  if (targetm.addr_space.default_pointer_address_modes_p != NULL)
++    return targetm.addr_space.default_pointer_address_modes_p();
++
+   if (targetm.addr_space.address_mode != default_addr_space_address_mode)
+     return false;
+   if (targetm.addr_space.pointer_mode != default_addr_space_pointer_mode)
+Index: gcc/config/i386/i386-c.c
+===================================================================
+--- gcc/config/i386/i386-c.c	(revision 223859)
++++ gcc/config/i386/i386-c.c	(working copy)
+@@ -572,6 +572,9 @@
+ 			       ix86_tune,
+ 			       ix86_fpmath,
+ 			       cpp_define);
++
++  cpp_define (parse_in, "__SEG_FS");
++  cpp_define (parse_in, "__SEG_GS");
+ }
+ 
+ 

+@@ -586,6 +589,9 @@
+   /* Update pragma hook to allow parsing #pragma GCC target.  */
+   targetm.target_option.pragma_parse = ix86_pragma_target_parse;
+ 
++  c_register_addr_space ("__seg_fs", ADDR_SPACE_SEG_FS);
++  c_register_addr_space ("__seg_gs", ADDR_SPACE_SEG_GS);
++
+ #ifdef REGISTER_SUBTARGET_PRAGMAS
+   REGISTER_SUBTARGET_PRAGMAS ();
+ #endif
+Index: gcc/config/i386/i386.c
+===================================================================
+--- gcc/config/i386/i386.c	(revision 223859)
++++ gcc/config/i386/i386.c	(working copy)
+@@ -15963,6 +15963,20 @@
+ 	  fputs (" PTR ", file);
+ 	}
+ 
++      /**** <AR> ****/
++      switch (MEM_ADDR_SPACE(x))
++	{
++	case ADDR_SPACE_SEG_FS:
++	  fputs (ASSEMBLER_DIALECT == ASM_ATT ? "%fs:" : "fs:", file);
++	  break;
++	case ADDR_SPACE_SEG_GS:
++	  fputs (ASSEMBLER_DIALECT == ASM_ATT ? "%gs:" : "gs:", file);
++	  break;
++	default:
++	  break;
++	}
++      /**** </AR> ****/
++
+       x = XEXP (x, 0);
+       /* Avoid (%rip) for call operands.  */
+       if (CONSTANT_ADDRESS_P (x) && code == 'P'
+@@ -51816,6 +51830,130 @@
+ }
+ #endif
+ 
++
++/***** <AR> *****/
++
++/*** GS segment register addressing mode ***/
++
++static machine_mode
++ix86_addr_space_pointer_mode (addr_space_t as)
++{
++  gcc_assert (as == ADDR_SPACE_GENERIC ||
++	      as == ADDR_SPACE_SEG_FS ||
++	      as == ADDR_SPACE_SEG_GS);
++  return ptr_mode;
++}
++
++/* Return the appropriate mode for a named address address.  */
++static machine_mode
++ix86_addr_space_address_mode (addr_space_t as)
++{
++  gcc_assert (as == ADDR_SPACE_GENERIC ||
++	      as == ADDR_SPACE_SEG_FS ||
++	      as == ADDR_SPACE_SEG_GS);
++  return Pmode;
++}
++
++/* Named address space version of valid_pointer_mode.  */
++static bool
++ix86_addr_space_valid_pointer_mode (machine_mode mode, addr_space_t as)
++{
++  gcc_assert (as == ADDR_SPACE_GENERIC ||
++	      as == ADDR_SPACE_SEG_FS ||
++	      as == ADDR_SPACE_SEG_GS);
++  return targetm.valid_pointer_mode (mode);
++}
++
++/* Like ix86_legitimate_address_p, except with named addresses.  */
++static bool
++ix86_addr_space_legitimate_address_p (machine_mode mode, rtx x,
++				      bool reg_ok_strict, addr_space_t as)
++{
++  gcc_assert (as == ADDR_SPACE_GENERIC ||
++	      as == ADDR_SPACE_SEG_FS ||
++	      as == ADDR_SPACE_SEG_GS);
++  return ix86_legitimate_address_p (mode, x, reg_ok_strict);
++}
++
++/* Named address space version of LEGITIMIZE_ADDRESS.  */
++static rtx
++ix86_addr_space_legitimize_address (rtx x, rtx oldx,
++				    machine_mode mode, addr_space_t as)
++{
++  gcc_assert (as == ADDR_SPACE_GENERIC ||
++	      as == ADDR_SPACE_SEG_FS ||
++	      as == ADDR_SPACE_SEG_GS);
++  return ix86_legitimize_address (x, oldx, mode);
++}
++
++/* The default, SEG_FS and SEG_GS address spaces are all "subsets" of
++   each other. */
++bool static 
++ix86_addr_space_subset_p (addr_space_t subset, addr_space_t superset)
++{
++  gcc_assert (subset == ADDR_SPACE_GENERIC ||
++	      subset == ADDR_SPACE_SEG_FS ||
++	      subset == ADDR_SPACE_SEG_GS);
++  gcc_assert (superset == ADDR_SPACE_GENERIC ||
++	      superset == ADDR_SPACE_SEG_FS ||
++	      superset == ADDR_SPACE_SEG_GS);
++  return true;
++}
++
++/* Convert from one address space to another: it is a no-op.
++   It is the C code's responsibility to write sensible casts. */
++static rtx
++ix86_addr_space_convert (rtx op, tree from_type, tree to_type)
++{
++  addr_space_t from_as = TYPE_ADDR_SPACE (TREE_TYPE (from_type));
++  addr_space_t to_as = TYPE_ADDR_SPACE (TREE_TYPE (to_type));
++
++  gcc_assert (from_as == ADDR_SPACE_GENERIC ||
++	      from_as == ADDR_SPACE_SEG_FS ||
++	      from_as == ADDR_SPACE_SEG_GS);
++  gcc_assert (to_as == ADDR_SPACE_GENERIC ||
++	      to_as == ADDR_SPACE_SEG_FS ||
++	      to_as == ADDR_SPACE_SEG_GS);
++
++  return op;
++}
++
++static bool
++ix86_addr_space_default_pointer_address_modes_p (void)
++{
++  return true;    /* all pointer and address modes are still Pmode/ptr_mode */
++}
++
++#undef TARGET_ADDR_SPACE_POINTER_MODE
++#define TARGET_ADDR_SPACE_POINTER_MODE ix86_addr_space_pointer_mode
++
++#undef TARGET_ADDR_SPACE_ADDRESS_MODE
++#define TARGET_ADDR_SPACE_ADDRESS_MODE ix86_addr_space_address_mode
++
++#undef TARGET_ADDR_SPACE_VALID_POINTER_MODE
++#define TARGET_ADDR_SPACE_VALID_POINTER_MODE ix86_addr_space_valid_pointer_mode
++
++#undef TARGET_ADDR_SPACE_LEGITIMATE_ADDRESS_P
++#define TARGET_ADDR_SPACE_LEGITIMATE_ADDRESS_P \
++  ix86_addr_space_legitimate_address_p
++
++#undef TARGET_ADDR_SPACE_LEGITIMIZE_ADDRESS
++#define TARGET_ADDR_SPACE_LEGITIMIZE_ADDRESS \
++  ix86_addr_space_legitimize_address
++
++#undef TARGET_ADDR_SPACE_SUBSET_P
++#define TARGET_ADDR_SPACE_SUBSET_P ix86_addr_space_subset_p
++
++#undef TARGET_ADDR_SPACE_CONVERT
++#define TARGET_ADDR_SPACE_CONVERT ix86_addr_space_convert
++
++#undef TARGET_ADDR_SPACE_DEFAULT_POINTER_ADDRESS_MODES_P
++#define TARGET_ADDR_SPACE_DEFAULT_POINTER_ADDRESS_MODES_P \
++  ix86_addr_space_default_pointer_address_modes_p
++
++/***** </AR> *****/
++
++
+ /* Initialize the GCC target structure.  */
+ #undef TARGET_RETURN_IN_MEMORY
+ #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
+Index: gcc/config/i386/i386.h
+===================================================================
+--- gcc/config/i386/i386.h	(revision 223859)
++++ gcc/config/i386/i386.h	(working copy)
+@@ -2568,6 +2568,11 @@
+ /* For switching between functions with different target attributes.  */
+ #define SWITCHABLE_TARGET 1
+ 
++enum {
++  ADDR_SPACE_SEG_FS = 1,
++  ADDR_SPACE_SEG_GS = 2
++};
++
+ /*
+ Local variables:
+ version-control: t

From noreply at buildbot.pypy.org  Thu Jul  2 13:10:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 13:10:30 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: Port the changes from c7
Message-ID: <20150702111030.3B0E71C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1887:33d8c4cbcb32
Date: 2015-07-02 12:06 +0100
http://bitbucket.org/pypy/stmgc/changeset/33d8c4cbcb32/

Log:	Port the changes from c7

diff --git a/c8/demo/Makefile b/c8/demo/Makefile
--- a/c8/demo/Makefile
+++ b/c8/demo/Makefile
@@ -19,18 +19,20 @@
 
 COMMON = -I.. -pthread -lrt -g -Wall -Werror -DSTM_LARGEMALLOC_TEST 
 
+CC = gcc-seg-gs
+
 
 # note that 'build' is partially optimized but still contains all asserts
 debug-%: %.c ${H_FILES} ${C_FILES}
-	clang $(COMMON) -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -O0 \
+	$(CC) $(COMMON) -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -O0 \
         $< -o debug-$* ../stmgc.c
 
 build-%: %.c ${H_FILES} ${C_FILES}
-	clang $(COMMON) -DSTM_GC_NURSERY=128 -O1 $< -o build-$* ../stmgc.c
+	$(CC) $(COMMON) -DSTM_GC_NURSERY=128 -O1 $< -o build-$* ../stmgc.c
 
 release-%: %.c ${H_FILES} ${C_FILES}
-	clang $(COMMON) -DNDEBUG -O2 $< -o release-$* ../stmgc.c
+	$(CC) $(COMMON) -DNDEBUG -O2 $< -o release-$* ../stmgc.c
 
 
 release-htm-%: %.c ../../htm-c7/stmgc.? ../../htm-c7/htm.h
-	clang $(COMMON) -O2 $< -o release-htm-$* ../../htm-c7/stmgc.c -DUSE_HTM
+	$(CC) $(COMMON) -O2 $< -o release-htm-$* ../../htm-c7/stmgc.c -DUSE_HTM
diff --git a/c8/demo/demo2.c b/c8/demo/demo2.c
--- a/c8/demo/demo2.c
+++ b/c8/demo/demo2.c
@@ -214,7 +214,7 @@
 
 void teardown_list(void)
 {
-    STM_POP_ROOT_RET(stm_thread_local);
+    STM_POP_ROOT_DROP(stm_thread_local);
 }
 
 
@@ -255,6 +255,7 @@
     stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
     unregister_thread_local();
     status = sem_post(&done); assert(status == 0);
+    (void)status;
     return NULL;
 }
 
@@ -292,6 +293,7 @@
     rewind_jmp_buf rjbuf;
 
     status = sem_init(&done, 0, 0); assert(status == 0);
+    (void)status;
 
     stm_setup();
     stm_register_thread_local(&stm_thread_local);
@@ -308,6 +310,7 @@
 
     for (i = 1; i <= NTHREADS; i++) {
         status = sem_wait(&done); assert(status == 0);
+        (void)status;
     }
 
     final_check();
diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c
--- a/c8/demo/demo_random2.c
+++ b/c8/demo/demo_random2.c
@@ -485,6 +485,7 @@
     stm_unregister_thread_local(&stm_thread_local);
 
     status = sem_post(&done); assert(status == 0);
+    (void)status;
     return NULL;
 }
 
diff --git a/c8/demo/test_shadowstack.c b/c8/demo/test_shadowstack.c
--- a/c8/demo/test_shadowstack.c
+++ b/c8/demo/test_shadowstack.c
@@ -53,7 +53,7 @@
        then do a major collection.  It should still be found by the
        tracing logic. */
     stm_force_transaction_break(&stm_thread_local);
-    STM_POP_ROOT_RET(stm_thread_local);
+    STM_POP_ROOT_DROP(stm_thread_local);
     STM_POP_ROOT(stm_thread_local, node);
     assert(node->value == 129821);
     STM_PUSH_ROOT(stm_thread_local, NULL);
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -868,7 +868,6 @@
     _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj, false);
 }
 
-__attribute__((always_inline))
 static void write_slowpath_overflow_obj(object_t *obj, bool mark_card)
 {
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
@@ -927,7 +926,6 @@
     release_privatization_lock(STM_SEGMENT->segment_num);
 }
 
-__attribute__((always_inline))
 static void write_slowpath_common(object_t *obj, bool mark_card)
 {
     assert(_seems_to_be_running_transaction());
@@ -1070,6 +1068,7 @@
              obj, index, get_index_to_card_index(index), CARD_MARKED));
 }
 
+__attribute__((flatten))
 void _stm_write_slowpath(object_t *obj) {
     write_slowpath_common(obj,  /* mark_card */ false);
 }
diff --git a/c8/stm/fprintcolor.c b/c8/stm/fprintcolor.c
--- a/c8/stm/fprintcolor.c
+++ b/c8/stm/fprintcolor.c
@@ -1,3 +1,5 @@
+#include <stdarg.h>
+
 /* ------------------------------------------------------------ */
 #ifdef STM_DEBUGPRINT
 /* ------------------------------------------------------------ */
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -21,7 +21,15 @@
 #endif
 
 
-#define TLPREFIX __attribute__((address_space(256)))
+#ifdef __SEG_GS     /* on a custom patched gcc */
+#  define TLPREFIX __seg_gs
+#  define _STM_RM_SUFFIX  :8
+#elif defined(__clang__)   /* on a clang, hopefully made bug-free */
+#  define TLPREFIX __attribute__((address_space(256)))
+#  define _STM_RM_SUFFIX  /* nothing */
+#else
+#  error "needs either a GCC with __seg_gs support, or a bug-freed clang"
+#endif
 
 typedef TLPREFIX struct object_s object_t;
 typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t;
@@ -35,18 +43,18 @@
        'STM_SEGMENT->transaction_read_version' if and only if the
        object was read in the current transaction.  The nurseries
        also have corresponding read markers, but they are never used. */
-    uint8_t rm;
+    unsigned char rm _STM_RM_SUFFIX;
 };
 
 struct stm_segment_info_s {
-    uint8_t transaction_read_version;
-    uint8_t no_safe_point_here;    /* set from outside, triggers an assert */
+    unsigned int transaction_read_version;
     int segment_num;
     char *segment_base;
     stm_char *nursery_current;
     stm_char *nursery_mark;
     uintptr_t nursery_end;
     struct stm_thread_local_s *running_thread;
+    uint8_t no_safe_point_here;    /* set from outside, triggers an assert */
 };
 #define STM_SEGMENT           ((stm_segment_info_t *)4352)
 
@@ -357,6 +365,7 @@
 #define STM_PUSH_ROOT(tl, p)   ((tl).shadowstack++->ss = (object_t *)(p))
 #define STM_POP_ROOT(tl, p)    ((p) = (typeof(p))((--(tl).shadowstack)->ss))
 #define STM_POP_ROOT_RET(tl)   ((--(tl).shadowstack)->ss)
+#define STM_POP_ROOT_DROP(tl)  ((void)(--(tl).shadowstack))
 
 /* Every thread needs to have a corresponding stm_thread_local_t
    structure.  It may be a "__thread" global variable or something else.
@@ -370,7 +379,12 @@
 
 /* At some key places, like the entry point of the thread and in the
    function with the interpreter's dispatch loop, you need to declare
-   a local variable of type 'rewind_jmp_buf' and call these macros. */
+   a local variable of type 'rewind_jmp_buf' and call these macros.
+   IMPORTANT: a function in which you call stm_rewind_jmp_enterframe()
+   must never change the value of its own arguments!  If they are
+   passed on the stack, gcc can change the value directly there, but
+   we're missing the logic to save/restore this part!
+*/
 #define stm_rewind_jmp_enterprepframe(tl, rjbuf)                        \
     rewind_jmp_enterprepframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack)
 #define stm_rewind_jmp_enterframe(tl, rjbuf)       \
@@ -657,7 +671,7 @@
 
 #define STM_POP_MARKER(tl)   ({                 \
     object_t *_popped = STM_POP_ROOT_RET(tl);   \
-    STM_POP_ROOT_RET(tl);                       \
+    STM_POP_ROOT_DROP(tl);                      \
     _popped;                                    \
 })
 
@@ -760,11 +774,11 @@
 
 /* ==================== END ==================== */
 
-static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,
+extern void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,
                             object_t *following_object,
                             char *outputbuf, size_t outputbufsize);
 
-static void (*stmcb_debug_print)(const char *cause, double time,
+extern void (*stmcb_debug_print)(const char *cause, double time,
                           const char *marker);
 
 #endif
diff --git a/c8/test/common.py b/c8/test/common.py
--- a/c8/test/common.py
+++ b/c8/test/common.py
@@ -3,7 +3,7 @@
 assert sys.maxint == 9223372036854775807, "requires a 64-bit environment"
 
 # ----------
-os.environ['CC'] = 'clang'
+os.environ['CC'] = 'gcc-seg-gs'
 
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -574,7 +574,7 @@
                     ],
      undef_macros=['NDEBUG'],
      include_dirs=[parent_dir],
-     extra_compile_args=['-g', '-O0', '-Wall', '-ferror-limit=5'],
+     extra_compile_args=['-g', '-O0', '-Wall'], #, '-ferror-limit=5'],
      extra_link_args=['-g', '-lrt'],
      force_generic_engine=True)
 
diff --git a/c8/test/test_list.py b/c8/test/test_list.py
--- a/c8/test/test_list.py
+++ b/c8/test/test_list.py
@@ -56,7 +56,7 @@
 ''', define_macros=[('STM_TESTS', '1')],
      undef_macros=['NDEBUG'],
      include_dirs=[parent_dir],
-     extra_compile_args=['-g', '-O0', '-Werror', '-ferror-limit=1'],
+     extra_compile_args=['-g', '-O0', '-Werror'],
      force_generic_engine=True)
 
 # ____________________________________________________________
diff --git a/c8/test/test_rewind.c b/c8/test/test_rewind.c
--- a/c8/test/test_rewind.c
+++ b/c8/test/test_rewind.c
@@ -174,12 +174,26 @@
 void foo(int *x) { ++*x; }
 
 __attribute__((noinline))
-void f6(int a1, int a2, int a3, int a4, int a5, int a6, int a7,
-        int a8, int a9, int a10, int a11, int a12, int a13)
+void f6(int c1, int c2, int c3, int c4, int c5, int c6, int c7,
+        int c8, int c9, int c10, int c11, int c12, int c13)
 {
     rewind_jmp_buf buf;
     rewind_jmp_enterframe(&gthread, &buf, NULL);
 
+    int a1 = c1;
+    int a2 = c2;
+    int a3 = c3;
+    int a4 = c4;
+    int a5 = c5;
+    int a6 = c6;
+    int a7 = c7;
+    int a8 = c8;
+    int a9 = c9;
+    int a10 = c10;
+    int a11 = c11;
+    int a12 = c12;
+    int a13 = c13;
+
     rewind_jmp_setjmp(&gthread, NULL);
     gevent(a1); gevent(a2); gevent(a3); gevent(a4);
     gevent(a5); gevent(a6); gevent(a7); gevent(a8);
diff --git a/c8/test/test_rewind.py b/c8/test/test_rewind.py
--- a/c8/test/test_rewind.py
+++ b/c8/test/test_rewind.py
@@ -1,11 +1,11 @@
 import os
 
 def run_test(opt):
-    err = os.system("clang -g -O%s -Werror -DRJBUF_CUSTOM_MALLOC -I../stm"
+    err = os.system("gcc-seg-gs -g -O%s -Werror -DRJBUF_CUSTOM_MALLOC -I../stm"
                     " -o test_rewind_O%s test_rewind.c ../stm/rewind_setjmp.c"
                     % (opt, opt))
     if err != 0:
-        raise OSError("clang failed on test_rewind.c")
+        raise OSError("gcc-seg-gs failed on test_rewind.c")
     for testnum in [1, 2, 3, 4, 5, 6, 7, "TL1", "TL2"]:
         print '=== O%s: RUNNING TEST %s ===' % (opt, testnum)
         err = os.system("./test_rewind_O%s %s" % (opt, testnum))

From noreply at buildbot.pypy.org  Thu Jul  2 13:24:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 13:24:27 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: Add a bug example
Message-ID: <20150702112427.E78151C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1888:4cf19da2ec99
Date: 2015-07-02 13:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/4cf19da2ec99/

Log:	Add a bug example

diff --git a/gcc-seg-gs/bug1.c b/gcc-seg-gs/bug1.c
new file mode 100644
--- /dev/null
+++ b/gcc-seg-gs/bug1.c
@@ -0,0 +1,21 @@
+typedef struct {
+    int a[20];
+} foo_t;
+
+
+int sum1(__seg_gs foo_t *p)
+{
+    int i, total=0;
+    for (i=0; i<20; i++)
+        total += p->a[i];
+    return total;
+}
+
+int sum2(void)
+{
+    __seg_gs foo_t *p = (__seg_gs foo_t *)0x1234;
+    int i, total=0;
+    for (i=0; i<20; i++)
+        total += p->a[i];     // <= this memory read is missing %gs:
+    return total;
+}

From noreply at buildbot.pypy.org  Thu Jul  2 13:36:54 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu,  2 Jul 2015 13:36:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150702113654.93FAD1C0170@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78412:3cb6a7ae2fb7
Date: 2015-07-02 12:40 +0200
http://bitbucket.org/pypy/pypy/changeset/3cb6a7ae2fb7/

Log:	2to3

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1407,15 +1407,15 @@
                     return self
             def load_module(self, fullname):
                 assert fullname == 'meta_path_2_pseudo_module'
-                m = new.module('meta_path_2_pseudo_module')
+                m = types.ModuleType('meta_path_2_pseudo_module')
                 m.__path__ = ['/some/random/dir']
                 sys.modules['meta_path_2_pseudo_module'] = m
                 return m
 
-        import sys, new
+        import sys, types
         sys.meta_path.append(ImportHook())
         try:
-            exec "from meta_path_2_pseudo_module import *" in {}
+            exec("from meta_path_2_pseudo_module import *", {})
         finally:
             sys.meta_path.pop()
 

From noreply at buildbot.pypy.org  Thu Jul  2 16:27:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 16:27:58 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: Avoid referencing arrays from within
 the struct stm_priv_segment_s, 
Message-ID: <20150702142758.AF63A1C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1889:69f1abc8e3fe
Date: 2015-07-02 15:24 +0100
http://bitbucket.org/pypy/stmgc/changeset/69f1abc8e3fe/

Log:	Avoid referencing arrays from within the struct stm_priv_segment_s,
	as a workaround for a gcc issue

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -151,8 +151,10 @@
 
     /* The sync queue used to synchronize newly allocated objs to
        other segments */
-    stm_char *sq_fragments[SYNC_QUEUE_SIZE];
-    int sq_fragsizes[SYNC_QUEUE_SIZE];
+    stm_char *_sq_fragments[SYNC_QUEUE_SIZE];
+    int _sq_fragsizes[SYNC_QUEUE_SIZE];
+    stm_char **sq_fragments;   /* indirection to work around a gcc issue */
+    int *sq_fragsizes;
     int sq_len;
 
     /* For nursery_mark */
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -116,6 +116,9 @@
         pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
         highest_overflow_number = pr->overflow_number;
         pr->pub.transaction_read_version = 0xff;
+
+        pr->sq_fragments = pr->_sq_fragments;
+        pr->sq_fragsizes = pr->_sq_fragsizes;
     }
 
     /* The pages are shared lazily, as remap_file_pages() takes a relatively

From noreply at buildbot.pypy.org  Thu Jul  2 16:27:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 16:27:59 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: Silence more warnings
Message-ID: <20150702142759.C9C7E1C1159@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1890:8fc3a9b12b15
Date: 2015-07-02 15:27 +0100
http://bitbucket.org/pypy/stmgc/changeset/8fc3a9b12b15/

Log:	Silence more warnings

diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -463,6 +463,7 @@
     stm_unregister_thread_local(&stm_thread_local);
 
     status = sem_post(&done); assert(status == 0);
+    (void)status;
     return NULL;
 }
 
diff --git a/c8/stm/locks.h b/c8/stm/locks.h
--- a/c8/stm/locks.h
+++ b/c8/stm/locks.h
@@ -10,11 +10,14 @@
    of modification locks!
 */
 
-typedef struct {
-    pthread_rwlock_t lock;
+typedef union {
+    struct {
+        pthread_rwlock_t lock;
 #ifndef NDEBUG
-    volatile bool write_locked;
+        volatile bool write_locked;
 #endif
+    };
+    char _pad[64];
 } modification_lock_t __attribute__((aligned(64)));
 
 static modification_lock_t _modlocks[NB_SEGMENTS - 1];
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -477,6 +477,7 @@
     /* reset the nursery by zeroing it */
     char *realnursery;
     realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start);
+    (void)realnursery;
 #if _STM_NURSERY_ZEROED
     memset(realnursery, 0, nursery_used);
 
diff --git a/c8/stm/queue.c b/c8/stm/queue.c
--- a/c8/stm/queue.c
+++ b/c8/stm/queue.c
@@ -52,7 +52,7 @@
 
 stm_queue_t *stm_queue_create(void)
 {
-    void *mem;
+    void *mem = NULL;
     int result = posix_memalign(&mem, 64, sizeof(stm_queue_t));
     assert(result == 0);
     (void)result;

From noreply at buildbot.pypy.org  Thu Jul  2 18:30:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 18:30:12 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: Trying to use gcc, 2nd attempt
Message-ID: <20150702163012.622701C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gcc
Changeset: r78413:5a06eb6c5c2f
Date: 2015-07-02 15:32 +0100
http://bitbucket.org/pypy/pypy/changeset/5a06eb6c5c2f/

Log:	Trying to use gcc, 2nd attempt


From noreply at buildbot.pypy.org  Thu Jul  2 18:30:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 18:30:13 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: Import the 'use-gcc' branch of
	stmgc
Message-ID: <20150702163013.8B3B41C11D3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gcc
Changeset: r78414:62f881a8b44d
Date: 2015-07-02 15:35 +0100
http://bitbucket.org/pypy/pypy/changeset/62f881a8b44d/

Log:	Import the 'use-gcc' branch of stmgc

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -440,7 +440,7 @@
             exe_name = targetdir.join(exe_name)
         kwds = {}
         if self.config.translation.stm:
-            kwds['cc'] = 'clang'     # force the use of clang
+            kwds['cc'] = 'gcc-seg-gs'   # use the custom patched version of gcc
         mk = self.translator.platform.gen_makefile(
             cfiles, self.eci,
             path=targetdir, exe_name=exe_name,
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-9f966d34d3be
+8fc3a9b12b15
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -868,7 +868,6 @@
     _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj, false);
 }
 
-__attribute__((always_inline))
 static void write_slowpath_overflow_obj(object_t *obj, bool mark_card)
 {
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
@@ -927,7 +926,6 @@
     release_privatization_lock(STM_SEGMENT->segment_num);
 }
 
-__attribute__((always_inline))
 static void write_slowpath_common(object_t *obj, bool mark_card)
 {
     assert(_seems_to_be_running_transaction());
@@ -1070,6 +1068,7 @@
              obj, index, get_index_to_card_index(index), CARD_MARKED));
 }
 
+__attribute__((flatten))
 void _stm_write_slowpath(object_t *obj) {
     write_slowpath_common(obj,  /* mark_card */ false);
 }
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -151,8 +151,10 @@
 
     /* The sync queue used to synchronize newly allocated objs to
        other segments */
-    stm_char *sq_fragments[SYNC_QUEUE_SIZE];
-    int sq_fragsizes[SYNC_QUEUE_SIZE];
+    stm_char *_sq_fragments[SYNC_QUEUE_SIZE];
+    int _sq_fragsizes[SYNC_QUEUE_SIZE];
+    stm_char **sq_fragments;   /* indirection to work around a gcc issue */
+    int *sq_fragsizes;
     int sq_len;
 
     /* For nursery_mark */
diff --git a/rpython/translator/stm/src_stm/stm/fprintcolor.c b/rpython/translator/stm/src_stm/stm/fprintcolor.c
--- a/rpython/translator/stm/src_stm/stm/fprintcolor.c
+++ b/rpython/translator/stm/src_stm/stm/fprintcolor.c
@@ -1,8 +1,10 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
+#include <stdarg.h>
 /* ------------------------------------------------------------ */
 #ifdef STM_DEBUGPRINT
 /* ------------------------------------------------------------ */
 
+
 static int threadcolor_printf(const char *format, ...)
 {
     char buffer[2048];
diff --git a/rpython/translator/stm/src_stm/stm/locks.h b/rpython/translator/stm/src_stm/stm/locks.h
--- a/rpython/translator/stm/src_stm/stm/locks.h
+++ b/rpython/translator/stm/src_stm/stm/locks.h
@@ -10,11 +10,14 @@
    of modification locks!
 */
 
-typedef struct {
-    pthread_rwlock_t lock;
+typedef union {
+    struct {
+        pthread_rwlock_t lock;
 #ifndef NDEBUG
-    volatile bool write_locked;
+        volatile bool write_locked;
 #endif
+    };
+    char _pad[64];
 } modification_lock_t __attribute__((aligned(64)));
 
 static modification_lock_t _modlocks[NB_SEGMENTS - 1];
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -477,6 +477,7 @@
     /* reset the nursery by zeroing it */
     char *realnursery;
     realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start);
+    (void)realnursery;
 #if _STM_NURSERY_ZEROED
     memset(realnursery, 0, nursery_used);
 
diff --git a/rpython/translator/stm/src_stm/stm/queue.c b/rpython/translator/stm/src_stm/stm/queue.c
--- a/rpython/translator/stm/src_stm/stm/queue.c
+++ b/rpython/translator/stm/src_stm/stm/queue.c
@@ -52,7 +52,7 @@
 
 stm_queue_t *stm_queue_create(void)
 {
-    void *mem;
+    void *mem = NULL;
     int result = posix_memalign(&mem, 64, sizeof(stm_queue_t));
     assert(result == 0);
     (void)result;
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -116,6 +116,9 @@
         pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
         highest_overflow_number = pr->overflow_number;
         pr->pub.transaction_read_version = 0xff;
+
+        pr->sq_fragments = pr->_sq_fragments;
+        pr->sq_fragsizes = pr->_sq_fragsizes;
     }
 
     /* The pages are shared lazily, as remap_file_pages() takes a relatively
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -21,7 +21,15 @@
 #endif
 
 
-#define TLPREFIX __attribute__((address_space(256)))
+#ifdef __SEG_GS     /* on a custom patched gcc */
+#  define TLPREFIX __seg_gs
+#  define _STM_RM_SUFFIX  :8
+#elif defined(__clang__)   /* on a clang, hopefully made bug-free */
+#  define TLPREFIX __attribute__((address_space(256)))
+#  define _STM_RM_SUFFIX  /* nothing */
+#else
+#  error "needs either a GCC with __seg_gs support, or a bug-freed clang"
+#endif
 
 typedef TLPREFIX struct object_s object_t;
 typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t;
@@ -35,18 +43,18 @@
        'STM_SEGMENT->transaction_read_version' if and only if the
        object was read in the current transaction.  The nurseries
        also have corresponding read markers, but they are never used. */
-    uint8_t rm;
+    unsigned char rm _STM_RM_SUFFIX;
 };
 
 struct stm_segment_info_s {
-    uint8_t transaction_read_version;
-    uint8_t no_safe_point_here;    /* set from outside, triggers an assert */
+    unsigned int transaction_read_version;
     int segment_num;
     char *segment_base;
     stm_char *nursery_current;
     stm_char *nursery_mark;
     uintptr_t nursery_end;
     struct stm_thread_local_s *running_thread;
+    uint8_t no_safe_point_here;    /* set from outside, triggers an assert */
 };
 #define STM_SEGMENT           ((stm_segment_info_t *)4352)
 
@@ -357,6 +365,7 @@
 #define STM_PUSH_ROOT(tl, p)   ((tl).shadowstack++->ss = (object_t *)(p))
 #define STM_POP_ROOT(tl, p)    ((p) = (typeof(p))((--(tl).shadowstack)->ss))
 #define STM_POP_ROOT_RET(tl)   ((--(tl).shadowstack)->ss)
+#define STM_POP_ROOT_DROP(tl)  ((void)(--(tl).shadowstack))
 
 /* Every thread needs to have a corresponding stm_thread_local_t
    structure.  It may be a "__thread" global variable or something else.
@@ -370,7 +379,12 @@
 
 /* At some key places, like the entry point of the thread and in the
    function with the interpreter's dispatch loop, you need to declare
-   a local variable of type 'rewind_jmp_buf' and call these macros. */
+   a local variable of type 'rewind_jmp_buf' and call these macros.
+   IMPORTANT: a function in which you call stm_rewind_jmp_enterframe()
+   must never change the value of its own arguments!  If they are
+   passed on the stack, gcc can change the value directly there, but
+   we're missing the logic to save/restore this part!
+*/
 #define stm_rewind_jmp_enterprepframe(tl, rjbuf)                        \
     rewind_jmp_enterprepframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack)
 #define stm_rewind_jmp_enterframe(tl, rjbuf)       \
@@ -657,7 +671,7 @@
 
 #define STM_POP_MARKER(tl)   ({                 \
     object_t *_popped = STM_POP_ROOT_RET(tl);   \
-    STM_POP_ROOT_RET(tl);                       \
+    STM_POP_ROOT_DROP(tl);                      \
     _popped;                                    \
 })
 
@@ -760,11 +774,11 @@
 
 /* ==================== END ==================== */
 
-static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,
+extern void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,
                             object_t *following_object,
                             char *outputbuf, size_t outputbufsize);
 
-static void (*stmcb_debug_print)(const char *cause, double time,
+extern void (*stmcb_debug_print)(const char *cause, double time,
                           const char *marker);
 
 #endif

From noreply at buildbot.pypy.org  Thu Jul  2 20:31:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 20:31:27 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: "Detect" stack overflows by checking
 if a segfault is caused by
Message-ID: <20150702183127.717471C026F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1891:8b922e88252b
Date: 2015-07-02 19:33 +0100
http://bitbucket.org/pypy/stmgc/changeset/8b922e88252b/

Log:	"Detect" stack overflows by checking if a segfault is caused by
	access in the protected page at the end of the shadowstack.

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -238,6 +238,7 @@
         addr >= stm_object_pages+TOTAL_MEMORY) {
         /* actual segfault, unrelated to stmgc */
         fprintf(stderr, "Segmentation fault: accessing %p\n", addr);
+        detect_shadowstack_overflow(addr);
         abort();
     }
 
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -186,12 +186,12 @@
     teardown_modification_locks();
 }
 
-static void _shadowstack_trap_page(char *start, int prot)
+static char *_shadowstack_trap_page(struct stm_shadowentry_s *base)
 {
     size_t bsize = STM_SHADOW_STACK_DEPTH * sizeof(struct stm_shadowentry_s);
-    char *end = start + bsize + 4095;
+    char *end = ((char *)base) + bsize + 4095;
     end -= (((uintptr_t)end) & 4095);
-    mprotect(end, 4096, prot);
+    return end;
 }
 
 static void _init_shadow_stack(stm_thread_local_t *tl)
@@ -203,9 +203,9 @@
 
     /* set up a trap page: if the shadowstack overflows, it will
        crash in a clean segfault */
-    _shadowstack_trap_page(start, PROT_NONE);
+    struct stm_shadowentry_s *s = (struct stm_shadowentry_s *)start;
+    mprotect(_shadowstack_trap_page(s), 4096, PROT_NONE);
 
-    struct stm_shadowentry_s *s = (struct stm_shadowentry_s *)start;
     tl->shadowstack = s;
     tl->shadowstack_base = s;
     STM_PUSH_ROOT(*tl, -1);
@@ -216,8 +216,8 @@
     assert(tl->shadowstack > tl->shadowstack_base);
     assert(tl->shadowstack_base->ss == (object_t *)-1);
 
-    char *start = (char *)tl->shadowstack_base;
-    _shadowstack_trap_page(start, PROT_READ | PROT_WRITE);
+    char *trap = _shadowstack_trap_page(tl->shadowstack_base);
+    mprotect(trap, 4096, PROT_READ | PROT_WRITE);
 
     free(tl->shadowstack_base);
     tl->shadowstack = NULL;
@@ -298,3 +298,19 @@
 {
     return tl->next != NULL;
 }
+
+static void detect_shadowstack_overflow(char *addr)
+{
+    if (addr == NULL)
+        return;
+    stm_thread_local_t *tl = stm_all_thread_locals;
+    while (tl != NULL) {
+        char *trap = _shadowstack_trap_page(tl->shadowstack_base);
+        if (trap <= addr && addr <= trap + 4095) {
+            fprintf(stderr, "This is caused by a stack overflow.\n"
+                "Sorry, proper RuntimeError support is not implemented yet.\n");
+            return;
+        }
+        tl = tl->next;
+    }
+}
diff --git a/c8/stm/setup.h b/c8/stm/setup.h
--- a/c8/stm/setup.h
+++ b/c8/stm/setup.h
@@ -1,6 +1,7 @@
 static void setup_mmap(char *reason);
 static void setup_protection_settings(void);
 static pthread_t *_get_cpth(stm_thread_local_t *);
+static void detect_shadowstack_overflow(char *);
 
 #ifndef NDEBUG
 static __thread long _stm_segfault_expected = 1;

From noreply at buildbot.pypy.org  Thu Jul  2 20:34:00 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 20:34:00 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: update to stmgc/8b922e88252b
Message-ID: <20150702183400.A67481C026F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gcc
Changeset: r78415:c20288c4ec6e
Date: 2015-07-02 19:35 +0100
http://bitbucket.org/pypy/pypy/changeset/c20288c4ec6e/

Log:	update to stmgc/8b922e88252b

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-8fc3a9b12b15
+8b922e88252b
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -238,6 +238,7 @@
         addr >= stm_object_pages+TOTAL_MEMORY) {
         /* actual segfault, unrelated to stmgc */
         fprintf(stderr, "Segmentation fault: accessing %p\n", addr);
+        detect_shadowstack_overflow(addr);
         abort();
     }
 
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -186,12 +186,12 @@
     teardown_modification_locks();
 }
 
-static void _shadowstack_trap_page(char *start, int prot)
+static char *_shadowstack_trap_page(struct stm_shadowentry_s *base)
 {
     size_t bsize = STM_SHADOW_STACK_DEPTH * sizeof(struct stm_shadowentry_s);
-    char *end = start + bsize + 4095;
+    char *end = ((char *)base) + bsize + 4095;
     end -= (((uintptr_t)end) & 4095);
-    mprotect(end, 4096, prot);
+    return end;
 }
 
 static void _init_shadow_stack(stm_thread_local_t *tl)
@@ -203,9 +203,9 @@
 
     /* set up a trap page: if the shadowstack overflows, it will
        crash in a clean segfault */
-    _shadowstack_trap_page(start, PROT_NONE);
+    struct stm_shadowentry_s *s = (struct stm_shadowentry_s *)start;
+    mprotect(_shadowstack_trap_page(s), 4096, PROT_NONE);
 
-    struct stm_shadowentry_s *s = (struct stm_shadowentry_s *)start;
     tl->shadowstack = s;
     tl->shadowstack_base = s;
     STM_PUSH_ROOT(*tl, -1);
@@ -216,8 +216,8 @@
     assert(tl->shadowstack > tl->shadowstack_base);
     assert(tl->shadowstack_base->ss == (object_t *)-1);
 
-    char *start = (char *)tl->shadowstack_base;
-    _shadowstack_trap_page(start, PROT_READ | PROT_WRITE);
+    char *trap = _shadowstack_trap_page(tl->shadowstack_base);
+    mprotect(trap, 4096, PROT_READ | PROT_WRITE);
 
     free(tl->shadowstack_base);
     tl->shadowstack = NULL;
@@ -298,3 +298,19 @@
 {
     return tl->next != NULL;
 }
+
+static void detect_shadowstack_overflow(char *addr)
+{
+    if (addr == NULL)
+        return;
+    stm_thread_local_t *tl = stm_all_thread_locals;
+    while (tl != NULL) {
+        char *trap = _shadowstack_trap_page(tl->shadowstack_base);
+        if (trap <= addr && addr <= trap + 4095) {
+            fprintf(stderr, "This is caused by a stack overflow.\n"
+                "Sorry, proper RuntimeError support is not implemented yet.\n");
+            return;
+        }
+        tl = tl->next;
+    }
+}
diff --git a/rpython/translator/stm/src_stm/stm/setup.h b/rpython/translator/stm/src_stm/stm/setup.h
--- a/rpython/translator/stm/src_stm/stm/setup.h
+++ b/rpython/translator/stm/src_stm/stm/setup.h
@@ -2,6 +2,7 @@
 static void setup_mmap(char *reason);
 static void setup_protection_settings(void);
 static pthread_t *_get_cpth(stm_thread_local_t *);
+static void detect_shadowstack_overflow(char *);
 #ifndef NDEBUG
 static __thread long _stm_segfault_expected = 1;
 #define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--; assert(_stm_segfault_expected <= 1);} while (0)

From noreply at buildbot.pypy.org  Thu Jul  2 20:41:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 20:41:03 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: Right now,
 it doesn't make sense to print the warning when forking, 
Message-ID: <20150702184103.E51331C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1892:f29c44d46d58
Date: 2015-07-02 19:43 +0100
http://bitbucket.org/pypy/stmgc/changeset/f29c44d46d58/

Log:	Right now, it doesn't make sense to print the warning when forking,
	because forking is fast enough

diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -20,7 +20,7 @@
     s_mutex_lock();
 
     dprintf(("forksupport_prepare\n"));
-    fprintf(stderr, "[forking: for now, this operation can take some time]\n");
+    //fprintf(stderr, "[forking: for now, this operation can take some time]\n");
 
     stm_thread_local_t *this_tl = NULL;
     stm_thread_local_t *tl = stm_all_thread_locals;

From noreply at buildbot.pypy.org  Thu Jul  2 22:17:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  2 Jul 2015 22:17:01 +0200 (CEST)
Subject: [pypy-commit] cffi default: Mention there are more args
Message-ID: <20150702201701.6BA5D1C026F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2198:af1fb2b8ce3b
Date: 2015-07-02 22:17 +0200
http://bitbucket.org/cffi/cffi/changeset/af1fb2b8ce3b/

Log:	Mention there are more args

diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -133,6 +133,8 @@
             #include <pwd.h>
         """,
         libraries=[])   # or a list of libraries to link with
+        # (more arguments like setup.py's Extension class:
+        # include_dirs=[..], extra_objects=[..], and so on)
 
     ffi.cdef("""     // some declarations from the man page
         struct passwd {

From noreply at buildbot.pypy.org  Fri Jul  3 08:50:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  3 Jul 2015 08:50:28 +0200 (CEST)
Subject: [pypy-commit] cffi default: Write a test for issue #212
Message-ID: <20150703065028.0BD4D1C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2199:0f549bccae62
Date: 2015-07-03 08:51 +0200
http://bitbucket.org/cffi/cffi/changeset/0f549bccae62/

Log:	Write a test for issue #212

diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -2230,3 +2230,15 @@
     #
     ffi.cdef("typedef int... foo_t;")
     py.test.raises(VerificationError, ffi.verify, "typedef float foo_t;")
+
+def test_windows_dllimport_data():
+    if sys.platform != 'win32':
+        py.test.skip("Windows only")
+    from testing.udir import udir
+    tmpfile = udir.join('dllimport_data.c')
+    tmpfile.write('int my_value = 42;\n')
+    ffi = FFI()
+    ffi.cdef("int my_value;")
+    lib = ffi.verify("extern __declspec(dllimport) int my_value;",
+                     sources = [str(tmpfile)])
+    assert lib.my_value == 42

From noreply at buildbot.pypy.org  Fri Jul  3 09:35:17 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri,  3 Jul 2015 09:35:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: add get_translation_config function to
 get at the (implicitly anyway global)
Message-ID: <20150703073517.2B3341C0170@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r78417:cadcfc72e119
Date: 2015-07-02 18:34 +0200
http://bitbucket.org/pypy/pypy/changeset/cadcfc72e119/

Log:	add get_translation_config function to get at the (implicitly anyway
	global) translation config object

diff --git a/rpython/config/test/test_translationoption.py b/rpython/config/test/test_translationoption.py
--- a/rpython/config/test/test_translationoption.py
+++ b/rpython/config/test/test_translationoption.py
@@ -2,6 +2,8 @@
 import py
 from rpython.config.translationoption import get_combined_translation_config
 from rpython.config.translationoption import set_opt_level
+from rpython.config.translationoption import get_translation_config
+from rpython.config import translationoption
 from rpython.config.config import ConflictConfigError, ConfigError
 from rpython.translator.platform import platform as compiler
 
@@ -16,3 +18,23 @@
         config = get_combined_translation_config()
         config.translation.gcrootfinder = "asmgcc"
         py.test.raises(ConfigError, set_opt_level, config, 'jit') 
+
+
+def test_get_translation_config():
+    from rpython.translator.interactive import Translation
+    from rpython.config import config
+    def f(x):
+        config = get_translation_config()
+        if config is not None:
+            return config.translating
+        return False
+
+    t = Translation(f, [int])
+    config = t.config
+
+    # do the patching
+    t.annotate()
+    retvar = t.context.graphs[0].returnblock.inputargs[0]
+    assert t.context.annotator.binding(retvar).const
+
+    assert get_translation_config() is config # check during import time
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -6,6 +6,7 @@
 from rpython.config.support import detect_number_of_processors
 from rpython.translator.platform import platform as compiler
 
+
 DEFL_INLINE_THRESHOLD = 32.4    # just enough to inline add__Int_Int()
 # and just small enough to prevend inlining of some rlist functions.
 
@@ -392,3 +393,16 @@
     opt = config.translation.platform
     cc = config.translation.cc
     return pick_platform(opt, cc)
+
+
+
+# when running a translation, this is patched
+# XXX evil global variable
+_GLOBAL_TRANSLATIONCONFIG = None
+
+
+def get_translation_config():
+    """ Return the translation config when translating. When running
+    un-translated returns None """
+    return _GLOBAL_TRANSLATIONCONFIG
+
diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -67,14 +67,16 @@
                  disable=[],
                  exe_name=None, extmod_name=None,
                  config=None, overrides=None):
+        from rpython.config import translationoption
         self.timer = Timer()
         SimpleTaskEngine.__init__(self)
 
         self.log = log
 
         if config is None:
-            from rpython.config.translationoption import get_combined_translation_config
-            config = get_combined_translation_config(translating=True)
+            config = translationoption.get_combined_translation_config(translating=True)
+        # XXX patch global variable with translation config
+        translationoption._GLOBAL_TRANSLATIONCONFIG = config
         self.config = config
         if overrides is not None:
             self.config.override(overrides)

From noreply at buildbot.pypy.org  Fri Jul  3 09:35:18 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri,  3 Jul 2015 09:35:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: allow GDB to attach and make set_trace
 an error if --lldebug isn't given to
Message-ID: <20150703073518.609F01C0170@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r78418:c2ef4714164a
Date: 2015-07-03 09:35 +0200
http://bitbucket.org/pypy/pypy/changeset/c2ef4714164a/

Log:	allow GDB to attach and make set_trace an error if --lldebug isn't
	given to translation

diff --git a/rpython/rtyper/module/ll_pdb.py b/rpython/rtyper/module/ll_pdb.py
--- a/rpython/rtyper/module/ll_pdb.py
+++ b/rpython/rtyper/module/ll_pdb.py
@@ -4,12 +4,32 @@
 
 import os
 import pdb
+import signal
 from rpython.rtyper.module.support import _WIN32
-from rpython.rtyper.extfunc import register_external
+from rpython.rtyper.extfunc import register_external, ExtFuncEntry
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.annotator.model import s_None
+from rpython.config.translationconfig import get_translation_config
+
+import time
 
 
 if not _WIN32:
+    eci = ExternalCompilationInfo(includes=['string.h', 'assert.h', 'sys/prctl.h'],
+                                  post_include_bits=["""
+static void pypy__allow_attach(void) {
+    prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY);
+    return;
+}
+    """])
+
+    allow_attach= rffi.llexternal(
+        "pypy__allow_attach", [], lltype.Void,
+        compilation_info=eci, _nowrapper=True)
+
     def pdb_set_trace():
+        allow_attach()
         pid = os.getpid()
         gdbpid = os.fork()
         if gdbpid == 0:
@@ -22,5 +42,21 @@
             try:
                 os.execv(shell, [argv0, "-c", "gdb -p %d" % pid])
             except OSError as e:
-                raise SystemExit('Could not start GDB: %s.' % e)
-    register_external(pdb.set_trace, [], llimpl=pdb_set_trace)
+                os.write(2, "Could not start GDB: %s" % (os.strerror(e.errno)))
+                raise SystemExit
+        else:
+            time.sleep(1) # give the GDB time to attach
+
+
+    class FunEntry(ExtFuncEntry):
+        _about_ = pdb.set_trace
+        signature_args = []
+        lltypeimpl = staticmethod(pdb_set_trace)
+        name = "pdb_set_trace"
+
+        def compute_result_annotation(self, *args_s):
+            config = self.bookkeeper.annotator.translator.config.translation
+            if config.lldebug or config.lldebug0:
+                return s_None
+            raise Exception("running pdb.set_trace() without also providing "
+                            "--lldebug when translating is not supported")

From noreply at buildbot.pypy.org  Fri Jul  3 11:36:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  3 Jul 2015 11:36:31 +0200 (CEST)
Subject: [pypy-commit] cffi default: Refactor the way global variables are
 accessed. Now, every access to
Message-ID: <20150703093631.AD9361C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2200:bdd39fccaf33
Date: 2015-07-03 11:37 +0200
http://bitbucket.org/cffi/cffi/changeset/bdd39fccaf33/

Log:	Refactor the way global variables are accessed. Now, every access
	to "lib.var" calls a generated C function to get "&var". This fixes
	issue #212 and also make it work in more cases, like __thread
	variables.

diff --git a/c/cglob.c b/c/cglob.c
--- a/c/cglob.c
+++ b/c/cglob.c
@@ -1,9 +1,12 @@
+
+typedef void *(*gs_fetch_addr_fn)(void);
 
 typedef struct {
     PyObject_HEAD
 
     CTypeDescrObject *gs_type;
     char             *gs_data;
+    gs_fetch_addr_fn  gs_fetch_addr;
 
 } GlobSupportObject;
 
@@ -38,7 +41,8 @@
 
 #define GlobSupport_Check(ob)  (Py_TYPE(ob) == &GlobSupport_Type)
 
-static PyObject *make_global_var(CTypeDescrObject *type, char *addr)
+static PyObject *make_global_var(CTypeDescrObject *type, char *addr,
+                                 gs_fetch_addr_fn fetch_addr)
 {
     GlobSupportObject *gs = PyObject_New(GlobSupportObject, &GlobSupport_Type);
     if (gs == NULL)
@@ -47,26 +51,47 @@
     Py_INCREF(type);
     gs->gs_type = type;
     gs->gs_data = addr;
+    gs->gs_fetch_addr = fetch_addr;
     return (PyObject *)gs;
 }
 
+static void *fetch_global_var_addr(GlobSupportObject *gs)
+{
+    void *data;
+    if (gs->gs_data != NULL) {
+        data = gs->gs_data;
+    }
+    else {
+        Py_BEGIN_ALLOW_THREADS
+        restore_errno();
+        data = gs->gs_fetch_addr();
+        save_errno();
+        Py_END_ALLOW_THREADS
+    }
+    return data;
+}
+
 static PyObject *read_global_var(GlobSupportObject *gs)
 {
-    return convert_to_object(gs->gs_data, gs->gs_type);
+    void *data = fetch_global_var_addr(gs);
+    return convert_to_object(data, gs->gs_type);
 }
 
 static int write_global_var(GlobSupportObject *gs, PyObject *obj)
 {
-    return convert_from_object(gs->gs_data, gs->gs_type, obj);
+    void *data = fetch_global_var_addr(gs);
+    return convert_from_object(data, gs->gs_type, obj);
 }
 
 static PyObject *cg_addressof_global_var(GlobSupportObject *gs)
 {
+    void *data;
     PyObject *x, *ptrtype = new_pointer_type(gs->gs_type);
     if (ptrtype == NULL)
         return NULL;
 
-    x = new_simple_cdata(gs->gs_data, (CTypeDescrObject *)ptrtype);
+    data = fetch_global_var_addr(gs);
+    x = new_simple_cdata(data, (CTypeDescrObject *)ptrtype);
     Py_DECREF(ptrtype);
     return x;
 }
diff --git a/c/lib_obj.c b/c/lib_obj.c
--- a/c/lib_obj.c
+++ b/c/lib_obj.c
@@ -267,7 +267,8 @@
             return NULL;
 
         if (ct->ct_size <= 0) {
-            PyErr_SetString(PyExc_SystemError, "constant has no known size");
+            PyErr_Format(FFIError, "constant '%s' is of type '%s', "
+                         "whose size is not known", s, ct->ct_name);
             return NULL;
         }
         if (g->address == NULL) {
@@ -299,7 +300,10 @@
 
     case _CFFI_OP_GLOBAL_VAR:
     {
-        /* global variable of the exact type specified here */
+        /* global variable of the exact type specified here
+           (nowadays, only used by the ABI mode or backward
+           compatibility; see _CFFI_OP_GLOBAL_VAR_F for the API mode)
+         */
         Py_ssize_t g_size = (Py_ssize_t)g->size_or_direct_fn;
         ct = realize_c_type(types_builder, types_builder->ctx.types,
                             _CFFI_GETARG(g->type_op));
@@ -320,12 +324,21 @@
                 if (address == NULL)
                     return NULL;
             }
-            x = make_global_var(ct, address);
+            x = make_global_var(ct, address, NULL);
         }
         Py_DECREF(ct);
         break;
     }
 
+    case _CFFI_OP_GLOBAL_VAR_F:
+        ct = realize_c_type(types_builder, types_builder->ctx.types,
+                            _CFFI_GETARG(g->type_op));
+        if (ct == NULL)
+            return NULL;
+        x = make_global_var(ct, NULL, (gs_fetch_addr_fn)g->address);
+        Py_DECREF(ct);
+        break;
+
     case _CFFI_OP_DLOPEN_FUNC:
     {
         /* For dlopen(): the function of the given 'name'.  We use
@@ -378,22 +391,25 @@
         }                                               \
     } while (0)
 
-static PyObject *_lib_dir1(LibObject *lib, int ignore_type)
+static PyObject *_lib_dir1(LibObject *lib, int ignore_global_vars)
 {
     const struct _cffi_global_s *g = lib->l_types_builder->ctx.globals;
     int i, count = 0, total = lib->l_types_builder->ctx.num_globals;
-    PyObject *lst = PyList_New(total);
+    PyObject *s, *lst = PyList_New(total);
     if (lst == NULL)
         return NULL;
 
     for (i = 0; i < total; i++) {
-        if (_CFFI_GETOP(g[i].type_op) != ignore_type) {
-            PyObject *s = PyText_FromString(g[i].name);
-            if (s == NULL)
-                goto error;
-            PyList_SET_ITEM(lst, count, s);
-            count++;
+        if (ignore_global_vars) {
+            int op = _CFFI_GETOP(g[i].type_op);
+            if (op == _CFFI_OP_GLOBAL_VAR || op == _CFFI_OP_GLOBAL_VAR_F)
+                continue;
         }
+        s = PyText_FromString(g[i].name);
+        if (s == NULL)
+            goto error;
+        PyList_SET_ITEM(lst, count, s);
+        count++;
     }
     if (PyList_SetSlice(lst, count, total, NULL) < 0)
         goto error;
@@ -445,7 +461,7 @@
  missing:
     if (strcmp(PyText_AsUTF8(name), "__all__") == 0) {
         PyErr_Clear();
-        return _lib_dir1(lib, _CFFI_OP_GLOBAL_VAR);
+        return _lib_dir1(lib, 1);
     }
     if (strcmp(PyText_AsUTF8(name), "__dict__") == 0) {
         PyErr_Clear();
@@ -481,7 +497,7 @@
 
 static PyObject *lib_dir(PyObject *self, PyObject *noarg)
 {
-    return _lib_dir1((LibObject *)self, -1);
+    return _lib_dir1((LibObject *)self, 0);
 }
 
 static PyMethodDef lib_methods[] = {
diff --git a/cffi/cffi_opcode.py b/cffi/cffi_opcode.py
--- a/cffi/cffi_opcode.py
+++ b/cffi/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -35,9 +35,6 @@
     def is_integer_type(self):
         return False
 
-    def sizeof_enabled(self):
-        return False
-
     def get_cached_btype(self, ffi, finishlist, can_delay=False):
         try:
             BType = ffi._cached_btypes[self]
@@ -80,8 +77,7 @@
 
 
 class BasePrimitiveType(BaseType):
-    def sizeof_enabled(self):
-        return True
+    pass
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -205,9 +201,6 @@
 class FunctionPtrType(BaseFunctionType):
     _base_pattern = '(*&)(%s)'
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         result = self.result.get_cached_btype(ffi, finishlist)
         args = []
@@ -233,9 +226,6 @@
             extra = self._base_pattern
         self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra)
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True)
         return global_cache(self, ffi, 'new_pointer_type', BItem)
@@ -276,9 +266,6 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def sizeof_enabled(self):
-        return self.item.sizeof_enabled() and self.length is not None
-
     def resolve_length(self, newlength):
         return ArrayType(self.item, newlength)
 
@@ -433,9 +420,6 @@
             from . import ffiplatform
             raise ffiplatform.VerificationMissing(self._get_c_name())
 
-    def sizeof_enabled(self):
-        return self.fldtypes is not None
-
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
@@ -464,9 +448,6 @@
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
-    def sizeof_enabled(self):
-        return True     # not strictly true, but external enums are obscure
-
     def force_the_name(self, forcename):
         StructOrUnionOrEnum.force_the_name(self, forcename)
         if self.forcename is None:
diff --git a/cffi/parse_c_type.h b/cffi/parse_c_type.h
--- a/cffi/parse_c_type.h
+++ b/cffi/parse_c_type.h
@@ -26,6 +26,7 @@
 #define _CFFI_OP_GLOBAL_VAR     33
 #define _CFFI_OP_DLOPEN_FUNC    35
 #define _CFFI_OP_DLOPEN_CONST   37
+#define _CFFI_OP_GLOBAL_VAR_F   39
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -981,10 +981,6 @@
         if not self.target_is_python and tp.is_integer_type():
             type_op = CffiOp(OP_CONSTANT_INT, -1)
         else:
-            if not tp.sizeof_enabled():
-                raise ffiplatform.VerificationError(
-                    "constant '%s' is of type '%s', whose size is not known"
-                    % (name, tp._get_c_name()))
             if self.target_is_python:
                 const_kind = OP_DLOPEN_CONST
             else:
@@ -1069,18 +1065,36 @@
         self._do_collect_type(self._global_type(tp, name))
 
     def _generate_cpy_variable_decl(self, tp, name):
-        pass
+        prnt = self._prnt
+        tp = self._global_type(tp, name)
+        if isinstance(tp, model.ArrayType) and tp.length is None:
+            tp = tp.item
+            ampersand = ''
+        else:
+            ampersand = '&'
+        # This code assumes that casts from "tp *" to "void *" is a
+        # no-op, i.e. a function that returns a "tp *" can be called
+        # as if it returned a "void *".  This should be generally true
+        # on any modern machine.  The only exception to that rule (on
+        # uncommon architectures, and as far as I can tell) might be
+        # if 'tp' were a function type, but that is not possible here.
+        # (If 'tp' is a function _pointer_ type, then casts from "fn_t
+        # **" to "void *" are again no-ops, as far as I can tell.)
+        prnt('static ' + tp.get_c_name('*_cffi_var_%s(void)' % (name,)))
+        prnt('{')
+        prnt('  return %s(%s);' % (ampersand, name))
+        prnt('}')
+        prnt()
 
     def _generate_cpy_variable_ctx(self, tp, name):
         tp = self._global_type(tp, name)
         type_index = self._typesdict[tp]
-        type_op = CffiOp(OP_GLOBAL_VAR, type_index)
-        if tp.sizeof_enabled():
-            size = "sizeof(%s)" % (name,)
+        if self.target_is_python:
+            op = OP_GLOBAL_VAR
         else:
-            size = 0
+            op = OP_GLOBAL_VAR_F
         self._lsts["global"].append(
-            GlobalExpr(name, '&%s' % name, type_op, size))
+            GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index)))
 
     # ----------
     # emitting the opcodes for individual types
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -562,8 +562,6 @@
   foo_wrapper(struct my_complex c) { foo(c.real + c.imag*1j); }``, and
   call ``foo_wrapper`` rather than ``foo`` directly.
 
-* Thread-local variables (access them via getter/setter functions)
-
 * Function pointers with non-default calling conventions (e.g. on
   Windows, "stdcall").
 
@@ -577,6 +575,11 @@
 get reduced safety checks: for example, you risk overwriting the
 following fields by passing too many array items in the constructor.
 
+.. versionadded:: 1.2
+   Thread-local variables (``__thread``) can be accessed, as well as
+   variables defined as dynamic macros (``#define myvar  (*fetchme())``).
+   Before version 1.2, you need to write getter/setter functions.
+
 
 Debugging dlopen'ed C libraries
 -------------------------------
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -30,6 +30,12 @@
   dict---assuming that ``lib`` has got no symbol called precisely
   ``__dict__``.  (In general, it is safer to use ``dir(lib)``.)
 
+* Out-of-line API mode: global variables are now fetched on demand at
+  every access.  It fixes issue #212 (Windows DLL variables), and also
+  allows variables that are defined as dynamic macros (like ``errno``)
+  or ``__thread`` -local variables.  (This change might also tighten
+  the C compiler's check on the variables' type.)
+
 * Issue #209: dereferencing NULL pointers now raises RuntimeError
   instead of segfaulting.  Meant as a debugging aid.  The check is
   only for NULL: if you dereference random or dead pointers you might
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -511,22 +511,38 @@
 def test_bad_size_of_global_1():
     ffi = FFI()
     ffi.cdef("short glob;")
-    lib = verify(ffi, "test_bad_size_of_global_1", "long glob;")
-    py.test.raises(ffi.error, "lib.glob")
+    py.test.raises(VerificationError, verify, ffi,
+                   "test_bad_size_of_global_1", "long glob;")
 
 def test_bad_size_of_global_2():
     ffi = FFI()
     ffi.cdef("int glob[10];")
-    lib = verify(ffi, "test_bad_size_of_global_2", "int glob[9];")
-    e = py.test.raises(ffi.error, "lib.glob")
-    assert str(e.value) == ("global variable 'glob' should be 40 bytes "
-                            "according to the cdef, but is actually 36")
+    py.test.raises(VerificationError, verify, ffi,
+                   "test_bad_size_of_global_2", "int glob[9];")
 
-def test_unspecified_size_of_global():
+def test_unspecified_size_of_global_1():
     ffi = FFI()
     ffi.cdef("int glob[];")
-    lib = verify(ffi, "test_unspecified_size_of_global", "int glob[10];")
-    lib.glob    # does not crash
+    lib = verify(ffi, "test_unspecified_size_of_global_1", "int glob[10];")
+    assert ffi.typeof(lib.glob) == ffi.typeof("int *")
+
+def test_unspecified_size_of_global_2():
+    ffi = FFI()
+    ffi.cdef("int glob[][5];")
+    lib = verify(ffi, "test_unspecified_size_of_global_2", "int glob[10][5];")
+    assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+def test_unspecified_size_of_global_3():
+    ffi = FFI()
+    ffi.cdef("int glob[][...];")
+    lib = verify(ffi, "test_unspecified_size_of_global_3", "int glob[10][5];")
+    assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+def test_unspecified_size_of_global_4():
+    ffi = FFI()
+    ffi.cdef("int glob[...][...];")
+    lib = verify(ffi, "test_unspecified_size_of_global_4", "int glob[10][5];")
+    assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
 def test_include_1():
     ffi1 = FFI()
@@ -888,9 +904,11 @@
         typedef ... opaque_t;
         const opaque_t CONSTANT;
     """)
-    e = py.test.raises(VerificationError, verify, ffi,
-                       'test_constant_of_unknown_size', "stuff")
-    assert str(e.value) == ("constant CONSTANT: constant 'CONSTANT' is of "
+    lib = verify(ffi, 'test_constant_of_unknown_size',
+                 "typedef int opaque_t;"
+                 "const int CONSTANT = 42;")
+    e = py.test.raises(ffi.error, getattr, lib, 'CONSTANT')
+    assert str(e.value) == ("constant 'CONSTANT' is of "
                             "type 'opaque_t', whose size is not known")
 
 def test_variable_of_unknown_size():
@@ -899,7 +917,7 @@
         typedef ... opaque_t;
         opaque_t globvar;
     """)
-    lib = verify(ffi, 'test_constant_of_unknown_size', """
+    lib = verify(ffi, 'test_variable_of_unknown_size', """
         typedef char opaque_t[6];
         opaque_t globvar = "hello";
     """)
@@ -1081,3 +1099,32 @@
     assert hasattr(lib, '__dict__')
     assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
     assert lib.__name__ == repr(lib)
+
+def test_macro_var_callback():
+    ffi = FFI()
+    ffi.cdef("int my_value; int *(*get_my_value)(void);")
+    lib = verify(ffi, 'test_macro_var_callback',
+                 "int *(*get_my_value)(void);\n"
+                 "#define my_value (*get_my_value())")
+    #
+    values = ffi.new("int[50]")
+    def it():
+        for i in range(50):
+            yield i
+    it = it()
+    #
+    @ffi.callback("int *(*)(void)")
+    def get_my_value():
+        return values + it.next()
+    lib.get_my_value = get_my_value
+    #
+    values[0] = 41
+    assert lib.my_value == 41            # [0]
+    p = ffi.addressof(lib, 'my_value')   # [1]
+    assert p == values + 1
+    assert p[-1] == 41
+    assert p[+1] == 0
+    lib.my_value = 42                    # [2]
+    assert values[2] == 42
+    assert p[-1] == 41
+    assert p[+1] == 42
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -2242,3 +2242,27 @@
     lib = ffi.verify("extern __declspec(dllimport) int my_value;",
                      sources = [str(tmpfile)])
     assert lib.my_value == 42
+
+def test_macro_var():
+    ffi = FFI()
+    ffi.cdef("int myarray[50], my_value;")
+    lib = ffi.verify("""
+        int myarray[50];
+        int *get_my_value(void) {
+            static int index = 0;
+            return &myarray[index++];
+        }
+        #define my_value (*get_my_value())
+    """)
+    assert lib.my_value == 0             # [0]
+    lib.my_value = 42                    # [1]
+    assert lib.myarray[1] == 42
+    assert lib.my_value == 0             # [2]
+    lib.myarray[3] = 63
+    assert lib.my_value == 63            # [3]
+    p = ffi.addressof(lib, 'my_value')   # [4]
+    assert p[-1] == 63
+    assert p[0] == 0
+    assert p == lib.myarray + 4
+    p[1] = 82
+    assert lib.my_value == 82            # [5]

From noreply at buildbot.pypy.org  Fri Jul  3 11:48:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  3 Jul 2015 11:48:53 +0200 (CEST)
Subject: [pypy-commit] cffi default: Bump the version number to 1.2.0
Message-ID: <20150703094853.89B061C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2201:5fd800db658c
Date: 2015-07-03 11:49 +0200
http://bitbucket.org/cffi/cffi/changeset/5fd800db658c/

Log:	Bump the version number to 1.2.0

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6073,7 +6073,7 @@
     if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0)
         INITERROR;
 
-    v = PyText_FromString("1.1.2");
+    v = PyText_FromString("1.2.0");
     if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0)
         INITERROR;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3354,4 +3354,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.2"
+    assert __version__ == "1.2.0"
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.2"
-__version_info__ = (1, 1, 2)
+__version__ = "1.2.0"
+__version_info__ = (1, 2, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -45,9 +45,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.1'
+version = '1.2'
 # The full version, including alpha/beta/rc tags.
-release = '1.1.2'
+release = '1.2.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.0.tar.gz
 
    - Or grab the most current version by following the instructions below.
 
-   - MD5: ca6e6c45b45caa87aee9adc7c796eaea
+   - MD5: ...
 
-   - SHA: 6d6203bf7d390560ac50943da4a3d2c96ab29756
+   - SHA: ...
 
 * Or get it from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.1.2',
+        version='1.2.0',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']}
                      if cpython else {},

From noreply at buildbot.pypy.org  Fri Jul  3 16:51:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  3 Jul 2015 16:51:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update to cffi/5fd800db658c
Message-ID: <20150703145147.311B91C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78419:3968d9b3c744
Date: 2015-07-03 16:51 +0200
http://bitbucket.org/pypy/pypy/changeset/3968d9b3c744/

Log:	Update to cffi/5fd800db658c

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.1.2
+Version: 1.2.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.2"
-__version_info__ = (1, 1, 2)
+__version__ = "1.2.0"
+__version_info__ = (1, 2, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -428,6 +428,8 @@
             raise TypeError("ffi.include() expects an argument that is also of"
                             " type cffi.FFI, not %r" % (
                                 type(ffi_to_include).__name__,))
+        if ffi_to_include is self:
+            raise ValueError("self.include(self)")
         with ffi_to_include._lock:
             with self._lock:
                 self._parser.include(ffi_to_include._parser)
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -633,6 +633,8 @@
 
     def include(self, other):
         for name, tp in other._declarations.items():
+            if name.startswith('anonymous $enum_$'):
+                continue   # fix for test_anonymous_enum_include
             kind = name.split(' ', 1)[0]
             if kind in ('struct', 'union', 'enum', 'anonymous'):
                 self._declare(name, tp, included=True)
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -35,9 +35,6 @@
     def is_integer_type(self):
         return False
 
-    def sizeof_enabled(self):
-        return False
-
     def get_cached_btype(self, ffi, finishlist, can_delay=False):
         try:
             BType = ffi._cached_btypes[self]
@@ -80,8 +77,7 @@
 
 
 class BasePrimitiveType(BaseType):
-    def sizeof_enabled(self):
-        return True
+    pass
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -205,9 +201,6 @@
 class FunctionPtrType(BaseFunctionType):
     _base_pattern = '(*&)(%s)'
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         result = self.result.get_cached_btype(ffi, finishlist)
         args = []
@@ -233,9 +226,6 @@
             extra = self._base_pattern
         self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra)
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True)
         return global_cache(self, ffi, 'new_pointer_type', BItem)
@@ -276,9 +266,6 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def sizeof_enabled(self):
-        return self.item.sizeof_enabled() and self.length is not None
-
     def resolve_length(self, newlength):
         return ArrayType(self.item, newlength)
 
@@ -433,9 +420,6 @@
             from . import ffiplatform
             raise ffiplatform.VerificationMissing(self._get_c_name())
 
-    def sizeof_enabled(self):
-        return self.fldtypes is not None
-
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
@@ -464,9 +448,6 @@
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
-    def sizeof_enabled(self):
-        return True     # not strictly true, but external enums are obscure
-
     def force_the_name(self, forcename):
         StructOrUnionOrEnum.force_the_name(self, forcename)
         if self.forcename is None:
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -26,6 +26,7 @@
 #define _CFFI_OP_GLOBAL_VAR     33
 #define _CFFI_OP_DLOPEN_FUNC    35
 #define _CFFI_OP_DLOPEN_CONST   37
+#define _CFFI_OP_GLOBAL_VAR_F   39
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -981,10 +981,6 @@
         if not self.target_is_python and tp.is_integer_type():
             type_op = CffiOp(OP_CONSTANT_INT, -1)
         else:
-            if not tp.sizeof_enabled():
-                raise ffiplatform.VerificationError(
-                    "constant '%s' is of type '%s', whose size is not known"
-                    % (name, tp._get_c_name()))
             if self.target_is_python:
                 const_kind = OP_DLOPEN_CONST
             else:
@@ -1069,18 +1065,36 @@
         self._do_collect_type(self._global_type(tp, name))
 
     def _generate_cpy_variable_decl(self, tp, name):
-        pass
+        prnt = self._prnt
+        tp = self._global_type(tp, name)
+        if isinstance(tp, model.ArrayType) and tp.length is None:
+            tp = tp.item
+            ampersand = ''
+        else:
+            ampersand = '&'
+        # This code assumes that casts from "tp *" to "void *" is a
+        # no-op, i.e. a function that returns a "tp *" can be called
+        # as if it returned a "void *".  This should be generally true
+        # on any modern machine.  The only exception to that rule (on
+        # uncommon architectures, and as far as I can tell) might be
+        # if 'tp' were a function type, but that is not possible here.
+        # (If 'tp' is a function _pointer_ type, then casts from "fn_t
+        # **" to "void *" are again no-ops, as far as I can tell.)
+        prnt('static ' + tp.get_c_name('*_cffi_var_%s(void)' % (name,)))
+        prnt('{')
+        prnt('  return %s(%s);' % (ampersand, name))
+        prnt('}')
+        prnt()
 
     def _generate_cpy_variable_ctx(self, tp, name):
         tp = self._global_type(tp, name)
         type_index = self._typesdict[tp]
-        type_op = CffiOp(OP_GLOBAL_VAR, type_index)
-        if tp.sizeof_enabled():
-            size = "sizeof(%s)" % (name,)
+        if self.target_is_python:
+            op = OP_GLOBAL_VAR
         else:
-            size = 0
+            op = OP_GLOBAL_VAR_F
         self._lsts["global"].append(
-            GlobalExpr(name, '&%s' % name, type_op, size))
+            GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index)))
 
     # ----------
     # emitting the opcodes for individual types
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.2"
+VERSION = "1.2.0"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -36,7 +36,10 @@
                         self.libname)
         try:
             cdata = dlsym(self.libhandle, name)
+            found = bool(cdata)
         except KeyError:
+            found = False
+        if not found:
             raise oefmt(self.ffi.w_FFIError,
                         "symbol '%s' not found in library '%s'",
                         name, self.libname)
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/pypy/module/_cffi_backend/cglob.py b/pypy/module/_cffi_backend/cglob.py
--- a/pypy/module/_cffi_backend/cglob.py
+++ b/pypy/module/_cffi_backend/cglob.py
@@ -2,23 +2,38 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend import newtype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+FNPTR = rffi.CCallback([], rffi.VOIDP)
 
 
 class W_GlobSupport(W_Root):
-    def __init__(self, space, w_ctype, ptr):
+    _immutable_fields_ = ['w_ctype', 'ptr', 'fetch_addr']
+
+    def __init__(self, space, w_ctype, ptr=lltype.nullptr(rffi.CCHARP.TO),
+                 fetch_addr=lltype.nullptr(rffi.VOIDP.TO)):
         self.space = space
         self.w_ctype = w_ctype
         self.ptr = ptr
+        self.fetch_addr = rffi.cast(FNPTR, fetch_addr)
+
+    def fetch_global_var_addr(self):
+        if self.ptr:
+            return self.ptr
+        result = self.fetch_addr()
+        return rffi.cast(rffi.CCHARP, result)
 
     def read_global_var(self):
-        return self.w_ctype.convert_to_object(self.ptr)
+        return self.w_ctype.convert_to_object(self.fetch_global_var_addr())
 
     def write_global_var(self, w_newvalue):
-        self.w_ctype.convert_from_object(self.ptr, w_newvalue)
+        self.w_ctype.convert_from_object(self.fetch_global_var_addr(),
+                                         w_newvalue)
 
     def address(self):
         w_ctypeptr = newtype.new_pointer_type(self.space, self.w_ctype)
-        return W_CData(self.space, self.ptr, w_ctypeptr)
+        return W_CData(self.space, self.fetch_global_var_addr(), w_ctypeptr)
 
 W_GlobSupport.typedef = TypeDef("FFIGlobSupport")
 W_GlobSupport.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -102,6 +102,8 @@
                 #
             elif op == cffi_opcode.OP_GLOBAL_VAR:
                 # A global variable of the exact type specified here
+                # (nowadays, only used by the ABI mode or backend
+                # compatibility; see OP_GLOBAL_F for the API mode
                 w_ct = realize_c_type.realize_c_type(
                     self.ffi, self.ctx.c_types, getarg(g.c_type_op))
                 g_size = rffi.cast(lltype.Signed, g.c_size_or_direct_fn)
@@ -113,7 +115,13 @@
                 ptr = rffi.cast(rffi.CCHARP, g.c_address)
                 if not ptr:   # for dlopen() style
                     ptr = self.cdlopen_fetch(attr)
-                w_result = cglob.W_GlobSupport(space, w_ct, ptr)
+                w_result = cglob.W_GlobSupport(space, w_ct, ptr=ptr)
+                #
+            elif op == cffi_opcode.OP_GLOBAL_VAR_F:
+                w_ct = realize_c_type.realize_c_type(
+                    self.ffi, self.ctx.c_types, getarg(g.c_type_op))
+                w_result = cglob.W_GlobSupport(space, w_ct,
+                                               fetch_addr=g.c_address)
                 #
             elif (op == cffi_opcode.OP_CONSTANT_INT or
                   op == cffi_opcode.OP_ENUM):
@@ -131,6 +139,9 @@
                     realize_c_type.FUNCPTR_FETCH_CHARP,
                     g.c_address)
                 if w_ct.size <= 0:
+                    raise oefmt(self.ffi.w_FFIError,
+                                "constant '%s' is of type '%s', "
+                                "whose size is not known", attr, w_ct.name)
                     raise oefmt(space.w_SystemError,
                                 "constant has no known size")
                 if not fetch_funcptr:   # for dlopen() style
@@ -172,7 +183,7 @@
             w_value = self._build_attr(attr)
             if w_value is None:
                 if is_getattr and attr == '__all__':
-                    return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
+                    return self.dir1(ignore_global_vars=True)
                 if is_getattr and attr == '__dict__':
                     return self.full_dict_copy()
                 if is_getattr and attr == '__name__':
@@ -206,14 +217,18 @@
     def descr_dir(self):
         return self.dir1()
 
-    def dir1(self, ignore_type=-1):
+    def dir1(self, ignore_global_vars=False):
         space = self.space
         total = rffi.getintfield(self.ctx, 'c_num_globals')
         g = self.ctx.c_globals
         names_w = []
         for i in range(total):
-            if getop(g[i].c_type_op) != ignore_type:
-                names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
+            if ignore_global_vars:
+                op = getop(g[i].c_type_op)
+                if (op == cffi_opcode.OP_GLOBAL_VAR or
+                    op == cffi_opcode.OP_GLOBAL_VAR_F):
+                    continue
+            names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
     def full_dict_copy(self):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3343,4 +3343,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.2"
+    assert __version__ == "1.2.0"
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -16,8 +16,8 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 0, 4):
-        py.test.skip("system cffi module needs to be at least 1.0.4")
+    if cffi.__version_info__ < (1, 2, 0):
+        py.test.skip("system cffi module needs to be at least 1.2.0")
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -500,28 +500,33 @@
             "int foo(int x) { return x + 32; }")
         assert lib.foo(10) == 42
 
-    def test_bad_size_of_global_1(self):
-        ffi, lib = self.prepare(
-            "short glob;",
-            "test_bad_size_of_global_1",
-            "long glob;")
-        raises(ffi.error, getattr, lib, "glob")
-
-    def test_bad_size_of_global_2(self):
-        ffi, lib = self.prepare(
-            "int glob[10];",
-            "test_bad_size_of_global_2",
-            "int glob[9];")
-        e = raises(ffi.error, getattr, lib, "glob")
-        assert str(e.value) == ("global variable 'glob' should be 40 bytes "
-                                "according to the cdef, but is actually 36")
-
-    def test_unspecified_size_of_global(self):
+    def test_unspecified_size_of_global_1(self):
         ffi, lib = self.prepare(
             "int glob[];",
-            "test_unspecified_size_of_global",
+            "test_unspecified_size_of_global_1",
             "int glob[10];")
-        lib.glob    # does not crash
+        assert ffi.typeof(lib.glob) == ffi.typeof("int *")
+
+    def test_unspecified_size_of_global_2(self):
+        ffi, lib = self.prepare(
+            "int glob[][5];",
+            "test_unspecified_size_of_global_2",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+    def test_unspecified_size_of_global_3(self):
+        ffi, lib = self.prepare(
+            "int glob[][...];",
+            "test_unspecified_size_of_global_3",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+    def test_unspecified_size_of_global_4(self):
+        ffi, lib = self.prepare(
+            "int glob[...][...];",
+            "test_unspecified_size_of_global_4",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
     def test_include_1(self):
         ffi1, lib1 = self.prepare(
@@ -869,11 +874,22 @@
             """)
         assert lib.almost_forty_two == 42.25
 
+    def test_constant_of_unknown_size(self):
+        ffi, lib = self.prepare(
+            "typedef ... opaque_t;"
+            "const opaque_t CONSTANT;",
+            'test_constant_of_unknown_size',
+            "typedef int opaque_t;"
+            "const int CONSTANT = 42;")
+        e = raises(ffi.error, getattr, lib, 'CONSTANT')
+        assert str(e.value) == ("constant 'CONSTANT' is of "
+                                "type 'opaque_t', whose size is not known")
+
     def test_variable_of_unknown_size(self):
         ffi, lib = self.prepare("""
             typedef ... opaque_t;
             opaque_t globvar;
-        """, 'test_constant_of_unknown_size', """
+        """, 'test_variable_of_unknown_size', """
             typedef char opaque_t[6];
             opaque_t globvar = "hello";
         """)
@@ -1012,3 +1028,32 @@
         assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
         assert lib.__name__ == repr(lib)
+
+    def test_macro_var_callback(self):
+        ffi, lib = self.prepare(
+            "int my_value; int *(*get_my_value)(void);",
+            'test_macro_var_callback',
+            "int *(*get_my_value)(void);\n"
+            "#define my_value (*get_my_value())")
+        #
+        values = ffi.new("int[50]")
+        def it():
+            for i in range(50):
+                yield i
+        it = it()
+        #
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            return values + it.next()
+        lib.get_my_value = get_my_value
+        #
+        values[0] = 41
+        assert lib.my_value == 41            # [0]
+        p = ffi.addressof(lib, 'my_value')   # [1]
+        assert p == values + 1
+        assert p[-1] == 41
+        assert p[+1] == 0
+        lib.my_value = 42                    # [2]
+        assert values[2] == 42
+        assert p[-1] == 41
+        assert p[+1] == 42
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1771,3 +1771,18 @@
         py.test.raises(TypeError, ffi.new, "struct foo_s *")
         ffi.cdef("struct foo_s { int x; };")
         ffi.new("struct foo_s *")
+
+    def test_ffi_self_include(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(ValueError, ffi.include, ffi)
+
+    def test_anonymous_enum_include(self):
+        ffi1 = FFI()
+        ffi1.cdef("enum { EE1 };")
+        ffi = FFI()
+        ffi.include(ffi1)
+        ffi.cdef("enum { EE2, EE3 };")
+        lib = ffi.dlopen(None)
+        assert lib.EE1 == 0
+        assert lib.EE2 == 0
+        assert lib.EE3 == 1
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1715,6 +1715,6 @@
         assert myvar == -5     # but can't be changed, so not very useful
         py.test.raises(ImportError, "from _test_import_from_lib.lib import bar")
         d = {}
-        exec "from _test_import_from_lib.lib import *" in d
+        exec("from _test_import_from_lib.lib import *", d)
         assert (set(key for key in d if not key.startswith('_')) ==
                 set(['myfunc', 'MYFOO']))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1,4 +1,5 @@
 # Generated by pypy/tool/import_cffi.py
+
 import sys, os, py
 from cffi import FFI, VerificationError
 from cffi import recompiler
@@ -511,22 +512,38 @@
 def test_bad_size_of_global_1():
     ffi = FFI()
     ffi.cdef("short glob;")
-    lib = verify(ffi, "test_bad_size_of_global_1", "long glob;")
-    py.test.raises(ffi.error, "lib.glob")
+    py.test.raises(VerificationError, verify, ffi,
+                   "test_bad_size_of_global_1", "long glob;")
 
 def test_bad_size_of_global_2():
     ffi = FFI()
     ffi.cdef("int glob[10];")
-    lib = verify(ffi, "test_bad_size_of_global_2", "int glob[9];")
-    e = py.test.raises(ffi.error, "lib.glob")
-    assert str(e.value) == ("global variable 'glob' should be 40 bytes "
-                            "according to the cdef, but is actually 36")
+    py.test.raises(VerificationError, verify, ffi,
+                   "test_bad_size_of_global_2", "int glob[9];")
 
-def test_unspecified_size_of_global():
+def test_unspecified_size_of_global_1():
     ffi = FFI()
     ffi.cdef("int glob[];")
-    lib = verify(ffi, "test_unspecified_size_of_global", "int glob[10];")
-    lib.glob    # does not crash
+    lib = verify(ffi, "test_unspecified_size_of_global_1", "int glob[10];")
+    assert ffi.typeof(lib.glob) == ffi.typeof("int *")
+
+def test_unspecified_size_of_global_2():
+    ffi = FFI()
+    ffi.cdef("int glob[][5];")
+    lib = verify(ffi, "test_unspecified_size_of_global_2", "int glob[10][5];")
+    assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+def test_unspecified_size_of_global_3():
+    ffi = FFI()
+    ffi.cdef("int glob[][...];")
+    lib = verify(ffi, "test_unspecified_size_of_global_3", "int glob[10][5];")
+    assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+def test_unspecified_size_of_global_4():
+    ffi = FFI()
+    ffi.cdef("int glob[...][...];")
+    lib = verify(ffi, "test_unspecified_size_of_global_4", "int glob[10][5];")
+    assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
 def test_include_1():
     ffi1 = FFI()
@@ -839,6 +856,22 @@
     assert isinstance(addr, ffi.CData)
     assert ffi.typeof(addr) == ffi.typeof("long(*)(long)")
 
+def test_address_of_function_with_struct():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int x; }; long myfunc(struct foo_s);")
+    lib = verify(ffi, "test_addressof_function_with_struct", """
+        struct foo_s { int x; };
+        char myfunc(struct foo_s input) { return (char)(input.x + 42); }
+    """)
+    s = ffi.new("struct foo_s *", [5])[0]
+    assert lib.myfunc(s) == 47
+    assert not isinstance(lib.myfunc, ffi.CData)
+    assert ffi.typeof(lib.myfunc) == ffi.typeof("long(*)(struct foo_s)")
+    addr = ffi.addressof(lib, 'myfunc')
+    assert addr(s) == 47
+    assert isinstance(addr, ffi.CData)
+    assert ffi.typeof(addr) == ffi.typeof("long(*)(struct foo_s)")
+
 def test_issue198():
     ffi = FFI()
     ffi.cdef("""
@@ -872,9 +905,11 @@
         typedef ... opaque_t;
         const opaque_t CONSTANT;
     """)
-    e = py.test.raises(VerificationError, verify, ffi,
-                       'test_constant_of_unknown_size', "stuff")
-    assert str(e.value) == ("constant CONSTANT: constant 'CONSTANT' is of "
+    lib = verify(ffi, 'test_constant_of_unknown_size',
+                 "typedef int opaque_t;"
+                 "const int CONSTANT = 42;")
+    e = py.test.raises(ffi.error, getattr, lib, 'CONSTANT')
+    assert str(e.value) == ("constant 'CONSTANT' is of "
                             "type 'opaque_t', whose size is not known")
 
 def test_variable_of_unknown_size():
@@ -883,7 +918,7 @@
         typedef ... opaque_t;
         opaque_t globvar;
     """)
-    lib = verify(ffi, 'test_constant_of_unknown_size', """
+    lib = verify(ffi, 'test_variable_of_unknown_size', """
         typedef char opaque_t[6];
         opaque_t globvar = "hello";
     """)
@@ -1064,3 +1099,33 @@
     assert MYFOO == 42
     assert hasattr(lib, '__dict__')
     assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
+    assert lib.__name__ == repr(lib)
+
+def test_macro_var_callback():
+    ffi = FFI()
+    ffi.cdef("int my_value; int *(*get_my_value)(void);")
+    lib = verify(ffi, 'test_macro_var_callback',
+                 "int *(*get_my_value)(void);\n"
+                 "#define my_value (*get_my_value())")
+    #
+    values = ffi.new("int[50]")
+    def it():
+        for i in range(50):
+            yield i
+    it = it()
+    #
+    @ffi.callback("int *(*)(void)")
+    def get_my_value():
+        return values + it.next()
+    lib.get_my_value = get_my_value
+    #
+    values[0] = 41
+    assert lib.my_value == 41            # [0]
+    p = ffi.addressof(lib, 'my_value')   # [1]
+    assert p == values + 1
+    assert p[-1] == 41
+    assert p[+1] == 0
+    lib.my_value = 42                    # [2]
+    assert values[2] == 42
+    assert p[-1] == 41
+    assert p[+1] == 42
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -2231,3 +2231,39 @@
     #
     ffi.cdef("typedef int... foo_t;")
     py.test.raises(VerificationError, ffi.verify, "typedef float foo_t;")
+
+def test_windows_dllimport_data():
+    if sys.platform != 'win32':
+        py.test.skip("Windows only")
+    from pypy.module.test_lib_pypy.cffi_tests.udir import udir
+    tmpfile = udir.join('dllimport_data.c')
+    tmpfile.write('int my_value = 42;\n')
+    ffi = FFI()
+    ffi.cdef("int my_value;")
+    lib = ffi.verify("extern __declspec(dllimport) int my_value;",
+                     sources = [str(tmpfile)])
+    assert lib.my_value == 42
+
+def test_macro_var():
+    ffi = FFI()
+    ffi.cdef("int myarray[50], my_value;")
+    lib = ffi.verify("""
+        int myarray[50];
+        int *get_my_value(void) {
+            static int index = 0;
+            return &myarray[index++];
+        }
+        #define my_value (*get_my_value())
+    """)
+    assert lib.my_value == 0             # [0]
+    lib.my_value = 42                    # [1]
+    assert lib.myarray[1] == 42
+    assert lib.my_value == 0             # [2]
+    lib.myarray[3] = 63
+    assert lib.my_value == 63            # [3]
+    p = ffi.addressof(lib, 'my_value')   # [4]
+    assert p[-1] == 63
+    assert p[0] == 0
+    assert p == lib.myarray + 4
+    p[1] = 82
+    assert lib.my_value == 82            # [5]
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -68,6 +68,8 @@
                 name += '.SO'
             if name.startswith('pycparser') and name.endswith('.egg'):
                 continue    # no clue why this shows up sometimes and not others
+            if name == '.eggs':
+                continue    # seems new in 3.5, ignore it
             assert name in content, "found unexpected file %r" % (
                 os.path.join(curdir, name),)
             value = content.pop(name)

From noreply at buildbot.pypy.org  Sat Jul  4 00:06:41 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 00:06:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix test
Message-ID: <20150703220641.A67231C120E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78420:b1635c72d2d4
Date: 2015-07-04 00:06 +0200
http://bitbucket.org/pypy/pypy/changeset/b1635c72d2d4/

Log:	Fix test

diff --git a/rpython/rlib/test/test_compilerinfo.py b/rpython/rlib/test/test_compilerinfo.py
--- a/rpython/rlib/test/test_compilerinfo.py
+++ b/rpython/rlib/test/test_compilerinfo.py
@@ -3,7 +3,7 @@
 
 
 def test_untranslated():
-    assert get_compiler_info() == "untranslated"
+    assert get_compiler_info() == "(untranslated)"
 
 def fn(index):
     cc = get_compiler_info()

From noreply at buildbot.pypy.org  Sat Jul  4 00:06:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 00:06:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: Uh,
 ImportError here.  I guess no test runs,
 so interrupting and restarting the tests.
Message-ID: <20150703220642.BBB6F1C1277@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78421:dc6539a30036
Date: 2015-07-04 00:06 +0200
http://bitbucket.org/pypy/pypy/changeset/dc6539a30036/

Log:	Uh, ImportError here. I guess no test runs, so interrupting and
	restarting the tests.

diff --git a/rpython/rtyper/module/ll_pdb.py b/rpython/rtyper/module/ll_pdb.py
--- a/rpython/rtyper/module/ll_pdb.py
+++ b/rpython/rtyper/module/ll_pdb.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.annotator.model import s_None
-from rpython.config.translationconfig import get_translation_config
+from rpython.config.translationoption import get_translation_config
 
 import time
 

From noreply at buildbot.pypy.org  Sat Jul  4 08:49:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat,  4 Jul 2015 08:49:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix the weird case of pickled finished
	generator
Message-ID: <20150704064939.1CCEF1C0170@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78422:a357ba14fe22
Date: 2015-07-04 08:49 +0200
http://bitbucket.org/pypy/pypy/changeset/a357ba14fe22/

Log:	fix the weird case of pickled finished generator

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -15,7 +15,10 @@
         self.running = False
 
     def descr__repr__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = 'finished'
+        else:
+            code_name = self.pycode.co_name
         addrstring = self.getaddrstring(space)
         return space.wrap("<generator object %s at 0x%s>" %
                           (code_name, addrstring))
@@ -45,6 +48,8 @@
         w_framestate, w_running = args_w
         if space.is_w(w_framestate, space.w_None):
             self.frame = None
+            self.space = space
+            self.pycode = None
         else:
             frame = instantiate(space.FrameClass)   # XXX fish
             frame.descr__setstate__(space, w_framestate)
@@ -62,9 +67,10 @@
 
     def send_ex(self, w_arg, operr=None):
         pycode = self.pycode
-        if jit.we_are_jitted() and should_not_inline(pycode):
-            generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
-                                                  operr=operr, pycode=pycode)
+        if pycode is not None:
+            if jit.we_are_jitted() and should_not_inline(pycode):
+                generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
+                                                    operr=operr, pycode=pycode)
         return self._send_ex(w_arg, operr)
 
     def _send_ex(self, w_arg, operr):
@@ -158,7 +164,10 @@
         return self.pycode
 
     def descr__name__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = 'finished'
+        else:
+            code_name = self.pycode.co_name
         return space.wrap(code_name)
 
     # Results can be either an RPython list of W_Root, or it can be an
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -491,6 +491,22 @@
         assert pack.mod is result
 
 
+    def test_pickle_generator_crash(self):
+        import pickle
+
+        def f():
+            yield 0
+
+        x = f()
+        x.next()
+        try:
+            x.next()
+        except StopIteration:
+            y = pickle.loads(pickle.dumps(x))
+        assert 'finished' in y.__name__
+        assert 'finished' in repr(y)
+        assert y.gi_code is None
+
 class AppTestGeneratorCloning:
 
     def setup_class(cls):

From noreply at buildbot.pypy.org  Sat Jul  4 09:25:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 09:25:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Use angular brackets on the artificial
	name
Message-ID: <20150704072559.93D311C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78423:613c07b2b3ad
Date: 2015-07-04 09:26 +0200
http://bitbucket.org/pypy/pypy/changeset/613c07b2b3ad/

Log:	Use angular brackets on the artificial name

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -16,7 +16,7 @@
 
     def descr__repr__(self, space):
         if self.pycode is None:
-            code_name = 'finished'
+            code_name = '<finished>'
         else:
             code_name = self.pycode.co_name
         addrstring = self.getaddrstring(space)
@@ -165,7 +165,7 @@
 
     def descr__name__(self, space):
         if self.pycode is None:
-            code_name = 'finished'
+            code_name = '<finished>'
         else:
             code_name = self.pycode.co_name
         return space.wrap(code_name)

From noreply at buildbot.pypy.org  Sat Jul  4 11:19:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 11:19:32 +0200 (CEST)
Subject: [pypy-commit] pypy int-float-list-strategy: Close branch ready to
	merge
Message-ID: <20150704091932.9BE801C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-float-list-strategy
Changeset: r78424:4e91fd7c06db
Date: 2015-07-04 11:17 +0200
http://bitbucket.org/pypy/pypy/changeset/4e91fd7c06db/

Log:	Close branch ready to merge


From noreply at buildbot.pypy.org  Sat Jul  4 11:19:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 11:19:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: hg merge int-float-list-strategy
Message-ID: <20150704091934.573511C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78425:d369501b8f6d
Date: 2015-07-04 11:19 +0200
http://bitbucket.org/pypy/pypy/changeset/d369501b8f6d/

Log:	hg merge int-float-list-strategy

	Add a list strategy for lists that store both floats and 32-bit
	integers. The latter are encoded as nonstandard NaNs. Benchmarks
	show that the speed of such lists is now very close to the speed of
	purely-int or purely-float lists.

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -14,6 +14,7 @@
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
     import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize)
+from rpython.rlib import longlong2float
 from rpython.tool.sourcetools import func_with_new_name
 
 from pypy.interpreter.baseobjspace import W_Root
@@ -73,33 +74,56 @@
             return SizeListStrategy(space, sizehint)
         return space.fromcache(EmptyListStrategy)
 
-    # check for ints
-    for w_obj in list_w:
-        if not type(w_obj) is W_IntObject:
+    w_firstobj = list_w[0]
+    check_int_or_float = False
+
+    if type(w_firstobj) is W_IntObject:
+        # check for all-ints
+        for i in range(1, len(list_w)):
+            w_obj = list_w[i]
+            if type(w_obj) is not W_IntObject:
+                check_int_or_float = (type(w_obj) is W_FloatObject)
+                break
+        else:
+            return space.fromcache(IntegerListStrategy)
+
+    elif type(w_firstobj) is W_BytesObject:
+        # check for all-strings
+        for i in range(1, len(list_w)):
+            if type(list_w[i]) is not W_BytesObject:
+                break
+        else:
+            return space.fromcache(BytesListStrategy)
+
+    elif type(w_firstobj) is W_UnicodeObject:
+        # check for all-unicodes
+        for i in range(1, len(list_w)):
+            if type(list_w[i]) is not W_UnicodeObject:
+                break
+        else:
+            return space.fromcache(UnicodeListStrategy)
+
+    elif type(w_firstobj) is W_FloatObject:
+        # check for all-floats
+        for i in range(1, len(list_w)):
+            w_obj = list_w[i]
+            if type(w_obj) is not W_FloatObject:
+                check_int_or_float = (type(w_obj) is W_IntObject)
+                break
+        else:
+            return space.fromcache(FloatListStrategy)
+
+    if check_int_or_float:
+        for w_obj in list_w:
+            if type(w_obj) is W_IntObject:
+                if longlong2float.can_encode_int32(space.int_w(w_obj)):
+                    continue    # ok
+            elif type(w_obj) is W_FloatObject:
+                if longlong2float.can_encode_float(space.float_w(w_obj)):
+                    continue    # ok
             break
-    else:
-        return space.fromcache(IntegerListStrategy)
-
-    # check for strings
-    for w_obj in list_w:
-        if not type(w_obj) is W_BytesObject:
-            break
-    else:
-        return space.fromcache(BytesListStrategy)
-
-    # check for unicode
-    for w_obj in list_w:
-        if not type(w_obj) is W_UnicodeObject:
-            break
-    else:
-        return space.fromcache(UnicodeListStrategy)
-
-    # check for floats
-    for w_obj in list_w:
-        if not type(w_obj) is W_FloatObject:
-            break
-    else:
-        return space.fromcache(FloatListStrategy)
+        else:
+            return space.fromcache(IntOrFloatListStrategy)
 
     return space.fromcache(ObjectListStrategy)
 
@@ -1382,12 +1406,15 @@
             return W_ListObject.from_storage_and_strategy(
                     self.space, storage, self)
 
+    def switch_to_next_strategy(self, w_list, w_sample_item):
+        w_list.switch_to_object_strategy()
+
     def append(self, w_list, w_item):
         if self.is_correct_type(w_item):
             self.unerase(w_list.lstorage).append(self.unwrap(w_item))
             return
 
-        w_list.switch_to_object_strategy()
+        self.switch_to_next_strategy(w_list, w_item)
         w_list.append(w_item)
 
     def insert(self, w_list, index, w_item):
@@ -1397,7 +1424,7 @@
             l.insert(index, self.unwrap(w_item))
             return
 
-        w_list.switch_to_object_strategy()
+        self.switch_to_next_strategy(w_list, w_item)
         w_list.insert(index, w_item)
 
     def _extend_from_list(self, w_list, w_other):
@@ -1421,7 +1448,7 @@
             except IndexError:
                 raise
         else:
-            w_list.switch_to_object_strategy()
+            self.switch_to_next_strategy(w_list, w_item)
             w_list.setitem(index, w_item)
 
     def setslice(self, w_list, start, step, slicelength, w_other):
@@ -1586,6 +1613,9 @@
     def getitems(self, w_list):
         return self.unerase(w_list.lstorage)
 
+    # no sort() method here: W_ListObject.descr_sort() handles this
+    # case explicitly
+
 
 class IntegerListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
@@ -1628,6 +1658,11 @@
             assert other is not None
             l += other
             return
+        if (w_other.strategy is self.space.fromcache(FloatListStrategy) or
+            w_other.strategy is self.space.fromcache(IntOrFloatListStrategy)):
+            if self.switch_to_int_or_float_strategy(w_list):
+                w_list.extend(w_other)
+                return
         return self._base_extend_from_list(w_list, w_other)
 
 
@@ -1638,8 +1673,46 @@
             storage = self.erase(w_other.getitems_int())
             w_other = W_ListObject.from_storage_and_strategy(
                     self.space, storage, self)
+        if (w_other.strategy is self.space.fromcache(FloatListStrategy) or
+            w_other.strategy is self.space.fromcache(IntOrFloatListStrategy)):
+            if self.switch_to_int_or_float_strategy(w_list):
+                w_list.setslice(start, step, slicelength, w_other)
+                return
         return self._base_setslice(w_list, start, step, slicelength, w_other)
 
+
+    @staticmethod
+    def int_2_float_or_int(w_list):
+        l = IntegerListStrategy.unerase(w_list.lstorage)
+        if not longlong2float.CAN_ALWAYS_ENCODE_INT32:
+            for intval in l:
+                if not longlong2float.can_encode_int32(intval):
+                    raise ValueError
+        return [longlong2float.encode_int32_into_longlong_nan(intval)
+                for intval in l]
+
+    def switch_to_int_or_float_strategy(self, w_list):
+        try:
+            generalized_list = self.int_2_float_or_int(w_list)
+        except ValueError:
+            return False
+        strategy = self.space.fromcache(IntOrFloatListStrategy)
+        w_list.strategy = strategy
+        w_list.lstorage = strategy.erase(generalized_list)
+        return True
+
+    def switch_to_next_strategy(self, w_list, w_sample_item):
+        if type(w_sample_item) is W_FloatObject:
+            if self.switch_to_int_or_float_strategy(w_list):
+                # yes, we can switch to IntOrFloatListStrategy
+                # (ignore here the extremely unlikely case where
+                # w_sample_item is just the wrong nonstandard NaN float;
+                # it will caught later and yet another switch will occur)
+                return
+        # no, fall back to ObjectListStrategy
+        w_list.switch_to_object_strategy()
+
+
 class FloatListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
@@ -1671,9 +1744,34 @@
     def getitems_float(self, w_list):
         return self.unerase(w_list.lstorage)
 
+
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if (w_other.strategy is self.space.fromcache(IntegerListStrategy) or
+            w_other.strategy is self.space.fromcache(IntOrFloatListStrategy)):
+            # xxx a case that we don't optimize: [3.4].extend([9999999999999])
+            # will cause a switch to int-or-float, followed by another
+            # switch to object
+            if self.switch_to_int_or_float_strategy(w_list):
+                w_list.extend(w_other)
+                return
+        return self._base_extend_from_list(w_list, w_other)
+
+
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if (w_other.strategy is self.space.fromcache(IntegerListStrategy) or
+            w_other.strategy is self.space.fromcache(IntOrFloatListStrategy)):
+            if self.switch_to_int_or_float_strategy(w_list):
+                w_list.setslice(start, step, slicelength, w_other)
+                return
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+
     def _safe_find(self, w_list, obj, start, stop):
         from rpython.rlib.rfloat import isnan
-        from rpython.rlib.longlong2float import float2longlong
         #
         l = self.unerase(w_list.lstorage)
         stop = min(stop, len(l))
@@ -1683,13 +1781,156 @@
                 if val == obj:
                     return i
         else:
-            search = float2longlong(obj)
+            search = longlong2float.float2longlong(obj)
             for i in range(start, stop):
                 val = l[i]
-                if float2longlong(val) == search:
+                if longlong2float.float2longlong(val) == search:
                     return i
         raise ValueError
 
+    @staticmethod
+    def float_2_float_or_int(w_list):
+        l = FloatListStrategy.unerase(w_list.lstorage)
+        generalized_list = []
+        for floatval in l:
+            if not longlong2float.can_encode_float(floatval):
+                raise ValueError
+            generalized_list.append(
+                longlong2float.float2longlong(floatval))
+        return generalized_list
+
+    def switch_to_int_or_float_strategy(self, w_list):
+        # xxx we should be able to use the same lstorage, but
+        # there is a typing issue (float vs longlong)...
+        try:
+            generalized_list = self.float_2_float_or_int(w_list)
+        except ValueError:
+            return False
+        strategy = self.space.fromcache(IntOrFloatListStrategy)
+        w_list.strategy = strategy
+        w_list.lstorage = strategy.erase(generalized_list)
+        return True
+
+    def switch_to_next_strategy(self, w_list, w_sample_item):
+        if type(w_sample_item) is W_IntObject:
+            sample_intval = self.space.int_w(w_sample_item)
+            if longlong2float.can_encode_int32(sample_intval):
+                if self.switch_to_int_or_float_strategy(w_list):
+                    # yes, we can switch to IntOrFloatListStrategy
+                    return
+        # no, fall back to ObjectListStrategy
+        w_list.switch_to_object_strategy()
+
+
+class IntOrFloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
+    _none_value = longlong2float.float2longlong(0.0)
+
+    def wrap(self, llval):
+        if longlong2float.is_int32_from_longlong_nan(llval):
+            intval = longlong2float.decode_int32_from_longlong_nan(llval)
+            return self.space.wrap(intval)
+        else:
+            floatval = longlong2float.longlong2float(llval)
+            return self.space.wrap(floatval)
+
+    def unwrap(self, w_int_or_float):
+        if type(w_int_or_float) is W_IntObject:
+            intval = self.space.int_w(w_int_or_float)
+            return longlong2float.encode_int32_into_longlong_nan(intval)
+        else:
+            floatval = self.space.float_w(w_int_or_float)
+            return longlong2float.float2longlong(floatval)
+
+    erase, unerase = rerased.new_erasing_pair("longlong")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def is_correct_type(self, w_obj):
+        if type(w_obj) is W_IntObject:
+            intval = self.space.int_w(w_obj)
+            return longlong2float.can_encode_int32(intval)
+        elif type(w_obj) is W_FloatObject:
+            floatval = self.space.float_w(w_obj)
+            return longlong2float.can_encode_float(floatval)
+        else:
+            return False
+
+    def list_is_correct_type(self, w_list):
+        return w_list.strategy is self.space.fromcache(IntOrFloatListStrategy)
+
+    def sort(self, w_list, reverse):
+        l = self.unerase(w_list.lstorage)
+        sorter = IntOrFloatSort(l, len(l))
+        # Reverse sort stability achieved by initially reversing the list,
+        # applying a stable forward sort, then reversing the final result.
+        if reverse:
+            l.reverse()
+        sorter.sort()
+        if reverse:
+            l.reverse()
+
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_longlong(self, w_list, longlong_list):
+        l = self.unerase(w_list.lstorage)
+        l += longlong_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(IntegerListStrategy):
+            try:
+                longlong_list = IntegerListStrategy.int_2_float_or_int(w_other)
+            except ValueError:
+                pass
+            else:
+                return self._extend_longlong(w_list, longlong_list)
+        if w_other.strategy is self.space.fromcache(FloatListStrategy):
+            try:
+                longlong_list = FloatListStrategy.float_2_float_or_int(w_other)
+            except ValueError:
+                pass
+            else:
+                return self._extend_longlong(w_list, longlong_list)
+        return self._base_extend_from_list(w_list, w_other)
+
+    _base_setslice = setslice
+
+    def _temporary_longlong_list(self, longlong_list):
+        storage = self.erase(longlong_list)
+        return W_ListObject.from_storage_and_strategy(self.space, storage, self)
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(IntegerListStrategy):
+            try:
+                longlong_list = IntegerListStrategy.int_2_float_or_int(w_other)
+            except ValueError:
+                pass
+            else:
+                w_other = self._temporary_longlong_list(longlong_list)
+        elif w_other.strategy is self.space.fromcache(FloatListStrategy):
+            try:
+                longlong_list = FloatListStrategy.float_2_float_or_int(w_other)
+            except ValueError:
+                pass
+            else:
+                w_other = self._temporary_longlong_list(longlong_list)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+    def _safe_find(self, w_list, obj, start, stop):
+        l = self.unerase(w_list.lstorage)
+        # careful: we must consider that 0.0 == -0.0 == 0, but also
+        # NaN == NaN if they have the same bit pattern.
+        fobj = longlong2float.maybe_decode_longlong_as_float(obj)
+        for i in range(start, min(stop, len(l))):
+            llval = l[i]
+            if llval == obj:     # equal as longlongs: includes NaN == NaN
+                return i
+            fval = longlong2float.maybe_decode_longlong_as_float(llval)
+            if fval == fobj:     # cases like 0.0 == -0.0 or 42 == 42.0
+                return i
+        raise ValueError
+
 
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
@@ -1786,6 +2027,7 @@
 TimSort = make_timsort_class()
 IntBaseTimSort = make_timsort_class()
 FloatBaseTimSort = make_timsort_class()
+IntOrFloatBaseTimSort = make_timsort_class()
 StringBaseTimSort = make_timsort_class()
 UnicodeBaseTimSort = make_timsort_class()
 
@@ -1816,6 +2058,13 @@
         return a < b
 
 
+class IntOrFloatSort(IntOrFloatBaseTimSort):
+    def lt(self, a, b):
+        fa = longlong2float.maybe_decode_longlong_as_float(a)
+        fb = longlong2float.maybe_decode_longlong_as_float(b)
+        return fa < fb
+
+
 class StringSort(StringBaseTimSort):
     def lt(self, a, b):
         return a < b
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1575,7 +1575,19 @@
         L2 = [N, 0.0]         # float strategy
         assert N in L2
         assert L2.index(N) == 0
+        assert L2.index(-0.0) == 1
         assert L2 == [N, -0.0]
+        # same with the int-or-float list strategy
+        L3 = [N, 0.0, -0.0, 0]
+        assert N in L3
+        assert L3.index(N) == 0
+        for i in [1, 2, 3]:
+            assert L3[i] == 0
+            assert L3[i] == 0.0
+            assert L3[i] == -0.0
+            assert L3.index(0, i) == i
+            assert L3.index(0.0, i) == i
+            assert L3.index(-0.0, i) == i
 
 
 class AppTestListObjectWithRangeList(AppTestListObject):
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -1,8 +1,10 @@
 import sys
+import py
 from pypy.objspace.std.listobject import (
     W_ListObject, EmptyListStrategy, ObjectListStrategy, IntegerListStrategy,
     FloatListStrategy, BytesListStrategy, RangeListStrategy,
-    SimpleRangeListStrategy, make_range_list, UnicodeListStrategy)
+    SimpleRangeListStrategy, make_range_list, UnicodeListStrategy,
+    IntOrFloatListStrategy)
 from pypy.objspace.std import listobject
 from pypy.objspace.std.test.test_listobject import TestW_ListObject
 
@@ -166,7 +168,6 @@
         assert isinstance(l.strategy, IntegerListStrategy)
 
     def test_list_empty_after_delete(self):
-        import py
         py.test.skip("return to emptyliststrategy is not supported anymore")
         l = W_ListObject(self.space, [self.space.wrap(3)])
         assert isinstance(l.strategy, IntegerListStrategy)
@@ -317,7 +318,7 @@
 
         l = W_ListObject(space, [w(1.1), w(2.2), w(3.3)])
         assert isinstance(l.strategy, FloatListStrategy)
-        l.extend(W_ListObject(space, [w(4), w(5), w(6)]))
+        l.extend(W_ListObject(space, [w("abc"), w("def"), w("ghi")]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
     def test_empty_extend_with_any(self):
@@ -733,6 +734,306 @@
         list_copy[0] = 42
         assert list_orig == [1, 2, 3]
 
+    def test_int_or_float_special_nan(self):
+        from rpython.rlib import longlong2float, rarithmetic
+        space = self.space
+        ll = rarithmetic.r_longlong(0xfffffffe12345678 - 2**64)
+        specialnan = longlong2float.longlong2float(ll)
+        w_l = W_ListObject(space, [space.wrap(1), space.wrap(specialnan)])
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+
+    def test_int_or_float_int_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        ok1 = 2**31 - 1
+        ok2 = -2**31
+        ovf1 = ok1 + 1
+        ovf2 = ok2 - 1
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(ovf1)])
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(ovf2)])
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(ok1)])
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(ok2)])
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+
+    def test_int_or_float_base(self):
+        from rpython.rlib.rfloat import INFINITY, NAN
+        space = self.space
+        w = space.wrap
+        w_l = W_ListObject(space, [space.wrap(1), space.wrap(2.3)])
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(w(int(2**31-1)))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(w(-5.1))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        assert space.int_w(w_l.getitem(2)) == 2**31-1
+        assert space.float_w(w_l.getitem(3)) == -5.1
+        w_l.append(w(INFINITY))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(w(NAN))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(w(-NAN))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        w_l.append(space.newlist([]))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+
+    def test_int_or_float_from_integer(self):
+        space = self.space
+        w = space.wrap
+        w_l = W_ListObject(space, [space.wrap(int(-2**31))])
+        assert isinstance(w_l.strategy, IntegerListStrategy)
+        w_l.append(w(-5.1))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        assert space.int_w(w_l.getitem(0)) == -2**31
+        assert space.float_w(w_l.getitem(1)) == -5.1
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_integer_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        w = space.wrap
+        ovf1 = -2**31 - 1
+        w_l = W_ListObject(space, [space.wrap(ovf1)])
+        assert isinstance(w_l.strategy, IntegerListStrategy)
+        w_l.append(w(-5.1))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert space.int_w(w_l.getitem(0)) == ovf1
+        assert space.float_w(w_l.getitem(1)) == -5.1
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_integer_special_nan(self):
+        from rpython.rlib import longlong2float, rarithmetic
+        space = self.space
+        w = space.wrap
+        w_l = W_ListObject(space, [space.wrap(int(-2**31))])
+        assert isinstance(w_l.strategy, IntegerListStrategy)
+        ll = rarithmetic.r_longlong(0xfffffffe12345678 - 2**64)
+        specialnan = longlong2float.longlong2float(ll)
+        w_l.append(w(specialnan))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert space.int_w(w_l.getitem(0)) == -2**31
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_float(self):
+        space = self.space
+        w = space.wrap
+        w_l = W_ListObject(space, [space.wrap(-42.5)])
+        assert isinstance(w_l.strategy, FloatListStrategy)
+        w_l.append(w(-15))
+        assert isinstance(w_l.strategy, IntOrFloatListStrategy)
+        assert space.float_w(w_l.getitem(0)) == -42.5
+        assert space.int_w(w_l.getitem(1)) == -15
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_float_int_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        w = space.wrap
+        ovf1 = 2 ** 31
+        w_l = W_ListObject(space, [space.wrap(1.2)])
+        assert isinstance(w_l.strategy, FloatListStrategy)
+        w_l.append(w(ovf1))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert space.float_w(w_l.getitem(0)) == 1.2
+        assert space.int_w(w_l.getitem(1)) == ovf1
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_from_float_special_nan(self):
+        from rpython.rlib import longlong2float, rarithmetic
+        space = self.space
+        w = space.wrap
+        ll = rarithmetic.r_longlong(0xfffffffe12345678 - 2**64)
+        specialnan = longlong2float.longlong2float(ll)
+        w_l = W_ListObject(space, [space.wrap(specialnan)])
+        assert isinstance(w_l.strategy, FloatListStrategy)
+        w_l.append(w(42))
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert space.int_w(w_l.getitem(1)) == 42
+        assert space.len_w(w_l) == 2
+
+    def test_int_or_float_extend(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3), space.wrap(4.5)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [0, 1.2, 3, 4.5]
+
+    def test_int_or_float_extend_mixed_1(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert [(type(x), x) for x in space.unwrap(w_l1)] == [
+            (int, 0), (float, 1.2), (int, 3)]
+
+    def test_int_or_float_extend_mixed_2(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3.4)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, FloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [0, 1.2, 3.4]
+
+    def test_int_or_float_extend_mixed_3(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0)])
+        w_l2 = W_ListObject(space, [space.wrap(3.4)])
+        assert isinstance(w_l1.strategy, IntegerListStrategy)
+        assert isinstance(w_l2.strategy, FloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [0, 3.4]
+
+    def test_int_or_float_extend_mixed_4(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0)])
+        w_l2 = W_ListObject(space, [space.wrap(3.4), space.wrap(-2)])
+        assert isinstance(w_l1.strategy, IntegerListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [0, 3.4, -2]
+
+    def test_int_or_float_extend_mixed_5(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(-2.5)])
+        w_l2 = W_ListObject(space, [space.wrap(42)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [-2.5, 42]
+
+    def test_int_or_float_extend_mixed_5_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        ovf1 = 2 ** 35
+        w_l1 = W_ListObject(space, [space.wrap(-2.5)])
+        w_l2 = W_ListObject(space, [space.wrap(ovf1)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, ObjectListStrategy)
+        assert space.unwrap(w_l1) == [-2.5, ovf1]
+
+    def test_int_or_float_extend_mixed_6(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(-2.5)])
+        w_l2 = W_ListObject(space, [space.wrap(3.4), space.wrap(-2)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.extend(w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [-2.5, 3.4, -2]
+
+    def test_int_or_float_setslice(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3), space.wrap(4.5)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3, 4.5, 1.2]
+
+    def test_int_or_float_setslice_mixed_1(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(12)])
+        w_l2 = W_ListObject(space, [space.wrap(3.2), space.wrap(4.5)])
+        assert isinstance(w_l1.strategy, IntegerListStrategy)
+        assert isinstance(w_l2.strategy, FloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3.2, 4.5, 12]
+
+    def test_int_or_float_setslice_mixed_2(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(12)])
+        w_l2 = W_ListObject(space, [space.wrap(3.2), space.wrap(45)])
+        assert isinstance(w_l1.strategy, IntegerListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3.2, 45, 12]
+
+    def test_int_or_float_setslice_mixed_3(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0.1), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(32), space.wrap(45)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [32, 45, 1.2]
+
+    def test_int_or_float_setslice_mixed_4(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0.1), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3.2), space.wrap(45)])
+        assert isinstance(w_l1.strategy, FloatListStrategy)
+        assert isinstance(w_l2.strategy, IntOrFloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3.2, 45, 1.2]
+
+    def test_int_or_float_setslice_mixed_5(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(32), space.wrap(45)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [32, 45, 1.2]
+
+    def test_int_or_float_setslice_mixed_5_overflow(self):
+        if sys.maxint == 2147483647:
+            py.test.skip("only on 64-bit")
+        space = self.space
+        ovf1 = 2 ** 35
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(32), space.wrap(ovf1)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, IntegerListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, ObjectListStrategy)
+        assert space.unwrap(w_l1) == [32, ovf1, 1.2]
+
+    def test_int_or_float_setslice_mixed_6(self):
+        space = self.space
+        w_l1 = W_ListObject(space, [space.wrap(0), space.wrap(1.2)])
+        w_l2 = W_ListObject(space, [space.wrap(3.2), space.wrap(4.5)])
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert isinstance(w_l2.strategy, FloatListStrategy)
+        w_l1.setslice(0, 1, 1, w_l2)
+        assert isinstance(w_l1.strategy, IntOrFloatListStrategy)
+        assert space.unwrap(w_l1) == [3.2, 4.5, 1.2]
+
+    def test_int_or_float_sort(self):
+        space = self.space
+        w_l = W_ListObject(space, [space.wrap(1.2), space.wrap(1),
+                                   space.wrap(1.0), space.wrap(5)])
+        w_l.sort(False)
+        assert [(type(x), x) for x in space.unwrap(w_l)] == [
+            (int, 1), (float, 1.0), (float, 1.2), (int, 5)]
+        w_l.sort(True)
+        assert [(type(x), x) for x in space.unwrap(w_l)] == [
+            (int, 5), (float, 1.2), (int, 1), (float, 1.0)]
+
 
 class TestW_ListStrategiesDisabled:
     spaceconfig = {"objspace.std.withliststrategies": False}
diff --git a/rpython/rlib/longlong2float.py b/rpython/rlib/longlong2float.py
--- a/rpython/rlib/longlong2float.py
+++ b/rpython/rlib/longlong2float.py
@@ -7,8 +7,9 @@
 """
 
 from __future__ import with_statement
+import sys
 from rpython.annotator import model as annmodel
-from rpython.rlib.rarithmetic import r_int64
+from rpython.rlib.rarithmetic import r_int64, intmask
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -99,3 +100,46 @@
         [v_longlong] = hop.inputargs(lltype.SignedLongLong)
         hop.exception_cannot_occur()
         return hop.genop("convert_longlong_bytes_to_float", [v_longlong], resulttype=lltype.Float)
+
+# ____________________________________________________________
+
+
+# For encoding integers inside nonstandard NaN bit patterns.
+#   ff ff ff fe xx xx xx xx    (signed 32-bit int)
+nan_high_word_int32 = -2       # -2 == (int)0xfffffffe
+nan_encoded_zero = r_int64(nan_high_word_int32 << 32)
+
+def encode_int32_into_longlong_nan(value):
+    return (nan_encoded_zero +
+            rffi.cast(rffi.LONGLONG, rffi.cast(rffi.UINT, value)))
+
+def decode_int32_from_longlong_nan(value):
+    return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+
+def is_int32_from_longlong_nan(value):
+    return intmask(value >> 32) == nan_high_word_int32
+
+CAN_ALWAYS_ENCODE_INT32 = (sys.maxint == 2147483647)
+
+def can_encode_int32(value):
+    if CAN_ALWAYS_ENCODE_INT32:
+        return True
+    return value == rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+
+def can_encode_float(value):
+    return intmask(float2longlong(value) >> 32) != nan_high_word_int32
+
+def maybe_decode_longlong_as_float(value):
+    # Decode a longlong value.  If a float, just return it as a float.
+    # If an encoded integer, return a float that has the (exact) same
+    # value.  This relies on the fact that casting from a decoded int
+    # to a float is an exact operation.  If we had full 64-bit
+    # integers, this cast would loose precision.  But this works
+    # because the integers are only 32-bit.  This would also work even
+    # if we encoded larger integers: as long as they are encoded
+    # inside a subset of the mantissa of a float, then the
+    # cast-to-float will be exact.
+    if is_int32_from_longlong_nan(value):
+        return float(decode_int32_from_longlong_nan(value))
+    else:
+        return longlong2float(value)
diff --git a/rpython/rlib/test/test_longlong2float.py b/rpython/rlib/test/test_longlong2float.py
--- a/rpython/rlib/test/test_longlong2float.py
+++ b/rpython/rlib/test/test_longlong2float.py
@@ -1,7 +1,7 @@
 from rpython.translator.c.test.test_genc import compile
 from rpython.rlib.longlong2float import longlong2float, float2longlong
 from rpython.rlib.longlong2float import uint2singlefloat, singlefloat2uint
-from rpython.rlib.rarithmetic import r_singlefloat
+from rpython.rlib.rarithmetic import r_singlefloat, r_longlong
 from rpython.rtyper.test.test_llinterp import interpret
 
 
@@ -21,6 +21,9 @@
     yield -inf
     yield inf / inf     # nan
 
+def test_float2longlong():
+    assert float2longlong(0.0) == r_longlong(0)
+
 def test_longlong_as_float():
     for x in enum_floats():
         res = fn(x)
@@ -63,3 +66,32 @@
     for x in enum_floats():
         res = fn2(x)
         assert repr(res) == repr(float(r_singlefloat(x)))
+
+# ____________________________________________________________
+
+def fn_encode_nan(f1, i2):
+    from rpython.rlib.longlong2float import can_encode_float, can_encode_int32
+    from rpython.rlib.longlong2float import encode_int32_into_longlong_nan
+    from rpython.rlib.longlong2float import decode_int32_from_longlong_nan
+    from rpython.rlib.longlong2float import is_int32_from_longlong_nan
+    from rpython.rlib.rfloat import isnan
+    assert can_encode_float(f1)
+    assert can_encode_int32(i2)
+    l1 = float2longlong(f1)
+    l2 = encode_int32_into_longlong_nan(i2)
+    assert not is_int32_from_longlong_nan(l1)
+    assert is_int32_from_longlong_nan(l2)
+    f1b = longlong2float(l1)
+    assert f1b == f1 or (isnan(f1b) and isnan(f1))
+    assert decode_int32_from_longlong_nan(l2) == i2
+    return 42
+
+def test_compiled_encode_nan():
+    fn2 = compile(fn_encode_nan, [float, int])
+    ints = [-2**31, 2**31-1, 42]
+    for x in enum_floats():
+        y = ints.pop()
+        ints.insert(0, y)
+        fn_encode_nan(x, y)
+        res = fn2(x, y)
+        assert res == 42

From noreply at buildbot.pypy.org  Sat Jul  4 16:05:55 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sat,  4 Jul 2015 16:05:55 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: hg merge py3k
Message-ID: <20150704140555.23E7A1C1050@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78426:d9dca9ea01f3
Date: 2015-07-04 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/d9dca9ea01f3/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 2155 lines

diff --git a/lib-python/2.7/test/test_urllib2.py b/lib-python/2.7/test/test_urllib2.py
--- a/lib-python/2.7/test/test_urllib2.py
+++ b/lib-python/2.7/test/test_urllib2.py
@@ -291,6 +291,7 @@
         self.req_headers = []
         self.data = None
         self.raise_on_endheaders = False
+        self.sock = None
         self._tunnel_headers = {}
 
     def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
diff --git a/lib-python/2.7/urllib2.py b/lib-python/2.7/urllib2.py
--- a/lib-python/2.7/urllib2.py
+++ b/lib-python/2.7/urllib2.py
@@ -1200,6 +1200,12 @@
                 r = h.getresponse(buffering=True)
             except TypeError: # buffering kw not supported
                 r = h.getresponse()
+            # If the server does not send us a 'Connection: close' header,
+            # HTTPConnection assumes the socket should be left open. Manually
+            # mark the socket to be closed when this response object goes away.
+            if h.sock:
+                h.sock.close()
+                h.sock = None
 
         # Pick apart the HTTPResponse object to get the addinfourl
         # object initialized properly.
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -25,7 +25,7 @@
 
         result = app.call('expr', '2**63')
         typePtr = AsObj(result).typePtr
-        if tkffi.string(typePtr.name) == v"bignum":
+        if tkffi.string(typePtr.name) == b"bignum":
             self.BigNumType = typePtr
 
 
@@ -103,6 +103,8 @@
         return value.internalRep.doubleValue
     if value.typePtr == typeCache.IntType:
         return value.internalRep.longValue
+    if value.typePtr == typeCache.WideIntType:
+        return FromWideIntObj(app, value)
     if value.typePtr == typeCache.BigNumType and tklib.HAVE_LIBTOMMATH:
         return FromBignumObj(app, value)
     if value.typePtr == typeCache.ListType:
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -180,6 +180,7 @@
 typedef int... Tcl_WideInt;
 
 int Tcl_GetWideIntFromObj(Tcl_Interp *interp, Tcl_Obj *obj, Tcl_WideInt *value);
+Tcl_Obj *Tcl_NewWideIntObj(Tcl_WideInt value);
 """)
 
 if HAVE_LIBTOMMATH:
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -135,7 +135,7 @@
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
 is not reliable in PyPy (nor Jython nor IronPython).  It also means that
-weak references may stay alive for a bit longer than expected.  This
+**weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
 suddenly they will really be dead, raising a ``ReferenceError`` on the
@@ -143,6 +143,24 @@
 ``ReferenceError`` at any place that uses them.  (Or, better yet, don't use
 ``weakref.proxy()`` at all; use ``weakref.ref()``.)
 
+Note a detail in the `documentation for weakref callbacks`__:
+
+    If callback is provided and not None, *and the returned weakref
+    object is still alive,* the callback will be called when the object
+    is about to be finalized.
+
+There are cases where, due to CPython's refcount semantics, a weakref
+dies immediately before or after the objects it points to (typically
+with some circular reference).  If it happens to die just after, then
+the callback will be invoked.  In a similar case in PyPy, both the
+object and the weakref will be considered as dead at the same time,
+and the callback will not be invoked.  (Issue `#2030`__)
+
+.. __: https://docs.python.org/2/library/weakref.html
+.. __: https://bitbucket.org/pypy/pypy/issue/2030/
+
+---------------------------------
+
 There are a few extra implications from the difference in the GC.  Most
 notably, if an object has a ``__del__``, the ``__del__`` is never called more
 than once in PyPy; but CPython will call the same ``__del__`` several times
@@ -321,9 +339,8 @@
 Miscellaneous
 -------------
 
-* Hash randomization (``-R``) is ignored in PyPy.  As documented in
-  http://bugs.python.org/issue14621, some of us believe it has no
-  purpose in CPython either.
+* Hash randomization (``-R``) `is ignored in PyPy`_.  In CPython
+  before 3.4 it has `little point`_.
 
 * You can't store non-string keys in type objects.  For example::
 
@@ -338,7 +355,8 @@
   for about 1400 calls.
 
 * since the implementation of dictionary is different, the exact number
-  which ``__hash__`` and ``__eq__`` are called is different. Since CPython
+  of times that ``__hash__`` and ``__eq__`` are called is different. 
+  Since CPython
   does not give any specific guarantees either, don't rely on it.
 
 * assignment to ``__class__`` is limited to the cases where it
@@ -395,3 +413,12 @@
   interactive mode. In a released version, this behaviour is suppressed, but
   setting the environment variable PYPY_IRC_TOPIC will bring it back. Note that
   downstream package providers have been known to totally disable this feature.
+
+* PyPy's readline module was rewritten from scratch: it is not GNU's
+  readline.  It should be mostly compatible, and it adds multiline
+  support (see ``multiline_input()``).  On the other hand,
+  ``parse_and_bind()`` calls are ignored (issue `#2072`_).
+
+.. _`is ignored in PyPy`: http://bugs.python.org/issue14621
+.. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
+.. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,15 +6,9 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
-**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
-on linux, linux64 and windows. We will make it the default on all platforms
-by the time of the next release.
-
-The first thing that you need is to compile PyPy yourself with the option
-``--shared``. We plan to make ``--shared`` the default in the future. Consult
-the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
-or ``pypy.dll`` file or something similar, depending on your platform. Consult
-your platform specification for details.
+**NOTE**: You need a PyPy compiled with the option ``--shared``, i.e.
+with a ``libpypy-c.so`` or ``pypy-c.dll`` file.  This is the default in
+recent versions of PyPy.
 
 The resulting shared library exports very few functions, however they are
 enough to accomplish everything you need, provided you follow a few principles.
@@ -75,10 +69,12 @@
 Note that this API is a lot more minimal than say CPython C API, so at first
 it's obvious to think that you can't do much. However, the trick is to do
 all the logic in Python and expose it via `cffi`_ callbacks. Let's assume
-we're on linux and pypy is installed in ``/opt/pypy`` with the
+we're on linux and pypy is installed in ``/opt/pypy`` (with
+subdirectories like ``lib-python`` and ``lib_pypy``), and with the
 library in ``/opt/pypy/bin/libpypy-c.so``.  (It doesn't need to be
-installed; you can also replace this path with your local checkout.)
-We write a little C program:
+installed; you can also replace these paths with a local extract of the
+installation tarballs, or with your local checkout of pypy.) We write a
+little C program:
 
 .. code-block:: c
 
@@ -92,7 +88,9 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        /* note: in the path /opt/pypy/x, the final x is ignored and
+           replaced with lib-python and lib_pypy. */
+        res = pypy_setup_home("/opt/pypy/x", 1);
         if (res) {
             printf("Error setting pypy home!\n");
             return 1;
@@ -179,7 +177,7 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        res = pypy_setup_home("/opt/pypy/x", 1);
         if (res) {
             fprintf(stderr, "Error setting pypy home!\n");
             return -1;
@@ -220,9 +218,15 @@
 Finding pypy_home
 -----------------
 
-Function pypy_setup_home takes one parameter - the path to libpypy. There's 
-currently no "clean" way (pkg-config comes to mind) how to find this path. You 
-can try the following (GNU-specific) hack (don't forget to link against *dl*):
+The function pypy_setup_home() takes as first parameter the path to a
+file from which it can deduce the location of the standard library.
+More precisely, it tries to remove final components until it finds
+``lib-python`` and ``lib_pypy``.  There is currently no "clean" way
+(pkg-config comes to mind) to find this path.  You can try the following
+(GNU-specific) hack (don't forget to link against *dl*), which assumes
+that the ``libpypy-c.so`` is inside the standard library directory.
+(This must more-or-less be the case anyway, otherwise the ``pypy``
+program itself would not run.)
 
 .. code-block:: c
 
@@ -236,7 +240,7 @@
 
     // caller should free returned pointer to avoid memleaks
     // returns NULL on error
-    char* guess_pypyhome() {
+    char* guess_pypyhome(void) {
         // glibc-only (dladdr is why we #define _GNU_SOURCE)
         Dl_info info;
         void *_rpython_startup_code = dlsym(0,"rpython_startup_code");
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -11,3 +11,14 @@
 .. branch: stdlib-2.7.10
 
 Update stdlib to version 2.7.10
+
+.. branch: issue2062
+
+.. branch: disable-unroll-for-short-loops
+The JIT no longer performs loop unrolling if the loop compiles to too much code.
+
+.. branch: run-create_cffi_imports
+
+Build cffi import libraries as part of translation by monkey-patching an 
+aditional task into translation
+
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -1,6 +1,6 @@
 import py
 
-import os, sys
+import os, sys, subprocess
 
 import pypy
 from pypy.interpreter import gateway
@@ -104,13 +104,16 @@
         from pypy.module.sys.initpath import pypy_find_stdlib
         verbose = rffi.cast(lltype.Signed, verbose)
         if ll_home:
-            home = rffi.charp2str(ll_home)
+            home1 = rffi.charp2str(ll_home)
+            home = os.path.join(home1, 'x') # <- so that 'll_home' can be
+                                            # directly the root directory
         else:
-            home = pypydir
+            home = home1 = pypydir
         w_path = pypy_find_stdlib(space, home)
         if space.is_none(w_path):
             if verbose:
-                debug("Failed to find library based on pypy_find_stdlib")
+                debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
+                      " not found in '%s' or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
         space.call_function(w_pathsetter, w_path)
@@ -301,6 +304,44 @@
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
 
+        # HACKHACKHACK
+        # ugly hack to modify target goal from compile_c to build_cffi_imports
+        # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.translator.driver import taskdef
+        import types
+
+        class Options(object):
+            pass
+
+
+        def mkexename(name):
+            if sys.platform == 'win32':
+                name = name.new(ext='exe')
+            return name
+
+        @taskdef(['compile_c'], "Create cffi bindings for modules")
+        def task_build_cffi_imports(self):
+            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
+            ''' Use cffi to compile cffi interfaces to modules'''
+            exename = mkexename(driver.compute_exe_name())
+            basedir = exename
+            while not basedir.join('include').exists():
+                _basedir = basedir.dirpath()
+                if _basedir == basedir:
+                    raise ValueError('interpreter %s not inside pypy repo', 
+                                     str(exename))
+                basedir = _basedir
+            modules = self.config.objspace.usemodules.getpaths()
+            options = Options()
+            # XXX possibly adapt options using modules
+            failures = create_cffi_import_libraries(exename, options, basedir)
+            # if failures, they were already printed
+            print  >> sys.stderr, str(exename),'successfully built, but errors while building the above modules will be ignored'
+        driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
+        driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, ['compile_c']
+        driver.default_goal = 'build_cffi_imports'
+        # HACKHACKHACK end
+
         return self.get_entry_point(config)
 
     def jitpolicy(self, driver):
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -767,6 +767,7 @@
     # This is important for py3k
     sys.executable = executable
 
+ at hidden_applevel
 def entry_point(executable, argv):
     # note that before calling setup_bootstrap_path, we are limited because we
     # cannot import stdlib modules. In particular, we cannot use unicode
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,6 +1,7 @@
 import sys
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 
 TICK_COUNTER_STEP = 100
diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py
--- a/pypy/interpreter/pytraceback.py
+++ b/pypy/interpreter/pytraceback.py
@@ -65,7 +65,6 @@
 
 
 def check_traceback(space, w_tb, msg):
-    from pypy.interpreter.typedef import PyTraceback
     if w_tb is None or not space.isinstance_w(w_tb, space.gettypeobject(PyTraceback.typedef)):
         raise OperationError(space.w_TypeError, space.wrap(msg))
     return w_tb
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -143,7 +143,7 @@
     @jit.unroll_safe
     def _call(self, funcaddr, args_w):
         space = self.space
-        cif_descr = self.cif_descr
+        cif_descr = self.cif_descr   # 'self' should have been promoted here
         size = cif_descr.exchange_size
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -134,8 +134,7 @@
 
     def convert_to_object(self, cdata):
         unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        s = rffi.wcharpsize2unicode(unichardata, 1)
-        return self.space.wrap(s)
+        return self.space.wrap(unichardata[0])
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -225,9 +225,13 @@
         if (isinstance(w_cdata, cdataobj.W_CDataNewOwning) or
             isinstance(w_cdata, cdataobj.W_CDataPtrToStructOrUnion)):
             if i != 0:
-                space = self.space
-                raise oefmt(space.w_IndexError,
+                raise oefmt(self.space.w_IndexError,
                             "cdata '%s' can only be indexed by 0", self.name)
+        else:
+            if not w_cdata.unsafe_escaping_ptr():
+                raise oefmt(self.space.w_RuntimeError,
+                            "cannot dereference null pointer from cdata '%s'",
+                            self.name)
         return self
 
     def _check_slice_index(self, w_cdata, start, stop):
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -60,12 +60,12 @@
             self.ffi, self.ctx.c_types, getarg(g.c_type_op))
         assert isinstance(rawfunctype, realize_c_type.W_RawFuncType)
         #
-        w_ct, locs = rawfunctype.unwrap_as_nostruct_fnptr(self.ffi)
+        rawfunctype.prepare_nostruct_fnptr(self.ffi)
         #
         ptr = rffi.cast(rffi.CCHARP, g.c_address)
         assert ptr
-        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn, w_ct,
-                                 locs, rawfunctype, fnname, self.libname)
+        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn,
+                                 rawfunctype, fnname, self.libname)
 
     @jit.elidable_promote()
     def _get_attr_elidable(self, attr):
@@ -173,6 +173,10 @@
             if w_value is None:
                 if is_getattr and attr == '__all__':
                     return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
+                if is_getattr and attr == '__dict__':
+                    return self.full_dict_copy()
+                if is_getattr and attr == '__name__':
+                    return self.descr_repr()
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
@@ -212,6 +216,17 @@
                 names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
+    def full_dict_copy(self):
+        space = self.space
+        total = rffi.getintfield(self.ctx, 'c_num_globals')
+        g = self.ctx.c_globals
+        w_result = space.newdict()
+        for i in range(total):
+            w_attr = space.wrap(rffi.charp2str(g[i].c_name))
+            w_value = self._get_attr(w_attr)
+            space.setitem(w_result, w_attr, w_value)
+        return w_result
+
     def address_of_func_or_global_var(self, varname):
         # rebuild a string object from 'varname', to do typechecks and
         # to force a unicode back to a plain string
@@ -224,7 +239,8 @@
         if isinstance(w_value, W_FunctionWrapper):
             # '&func' returns a regular cdata pointer-to-function
             if w_value.directfnptr:
-                return W_CData(space, w_value.directfnptr, w_value.ctype)
+                ctype = w_value.typeof(self.ffi)
+                return W_CData(space, w_value.directfnptr, ctype)
             else:
                 return w_value    # backward compatibility
         #
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib import jit
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -135,8 +136,12 @@
 
 class W_RawFuncType(W_Root):
     """Temporary: represents a C function type (not a function pointer)"""
+
+    _immutable_fields_ = ['nostruct_ctype', 'nostruct_locs', 'nostruct_nargs']
     _ctfuncptr = None
-    _nostruct_ctfuncptr = (None, None)
+    nostruct_ctype = None
+    nostruct_locs = None
+    nostruct_nargs = 0
 
     def __init__(self, opcodes, base_index):
         self.opcodes = opcodes
@@ -168,14 +173,16 @@
         assert self._ctfuncptr is not None
         return self._ctfuncptr
 
-    def unwrap_as_nostruct_fnptr(self, ffi):
-        # tweaked version: instead of returning the ctfuncptr corresponding
-        # exactly to the OP_FUNCTION ... OP_FUNCTION_END opcodes, return
-        # another one in which the struct args are replaced with ptr-to-
-        # struct, and a struct return value is replaced with a hidden first
-        # arg of type ptr-to-struct.  This is how recompiler.py produces
+    @jit.dont_look_inside
+    def prepare_nostruct_fnptr(self, ffi):
+        # tweaked version: instead of returning the ctfuncptr
+        # corresponding exactly to the OP_FUNCTION ... OP_FUNCTION_END
+        # opcodes, this builds in self.nostruct_ctype another one in
+        # which the struct args are replaced with ptr-to- struct, and
+        # a struct return value is replaced with a hidden first arg of
+        # type ptr-to-struct.  This is how recompiler.py produces
         # trampoline functions for PyPy.
-        if self._nostruct_ctfuncptr[0] is None:
+        if self.nostruct_ctype is None:
             fargs, fret, ellipsis = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
             # containing 'A' where a struct argument was detected, and 'R'
@@ -198,8 +205,10 @@
                 locs = None
             else:
                 locs = ''.join(locs)
-            self._nostruct_ctfuncptr = (ctfuncptr, locs)
-        return self._nostruct_ctfuncptr
+            self.nostruct_ctype = ctfuncptr
+            self.nostruct_locs = locs
+            self.nostruct_nargs = len(ctfuncptr.fargs) - (locs is not None and
+                                                          locs[0] == 'R')
 
     def unexpected_fn_type(self, ffi):
         fargs, fret, ellipsis = self._unpack(ffi)
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -362,7 +362,7 @@
 
             case TOK_INTEGER:
                 errno = 0;
-#ifndef MS_WIN32
+#ifndef _MSC_VER
                 if (sizeof(length) > sizeof(unsigned long))
                     length = strtoull(tok->p, &endptr, 0);
                 else
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2099,8 +2099,7 @@
     p = cast(BVoidP, 123456)
     py.test.raises(TypeError, "p[0]")
     p = cast(BVoidP, 0)
-    if 'PY_DOT_PY' in globals(): py.test.skip("NULL crashes early on py.py")
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises((TypeError, RuntimeError), "p[0]")
 
 def test_iter():
     BInt = new_primitive_type("int")
@@ -3333,6 +3332,15 @@
     check(4 | 8,  "CHB", "GTB")
     check(4 | 16, "CHB", "ROB")
 
+def test_dereference_null_ptr():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    p = cast(BIntPtr, 0)
+    py.test.raises(RuntimeError, "p[0]")
+    py.test.raises(RuntimeError, "p[0] = 42")
+    py.test.raises(RuntimeError, "p[42]")
+    py.test.raises(RuntimeError, "p[42] = -1")
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "1.1.2"
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -276,6 +276,15 @@
         """)
         lib.aa = 5
         assert dir(lib) == ['aa', 'ff', 'my_constant']
+        #
+        aaobj = lib.__dict__['aa']
+        assert not isinstance(aaobj, int)    # some internal object instead
+        assert lib.__dict__ == {
+            'ff': lib.ff,
+            'aa': aaobj,
+            'my_constant': -45}
+        lib.__dict__['ff'] = "??"
+        assert lib.ff(10) == 15
 
     def test_verify_opaque_struct(self):
         ffi, lib = self.prepare(
@@ -819,6 +828,22 @@
         assert isinstance(addr, ffi.CData)
         assert ffi.typeof(addr) == ffi.typeof("long(*)(long)")
 
+    def test_address_of_function_with_struct(self):
+        ffi, lib = self.prepare(
+            "struct foo_s { int x; }; long myfunc(struct foo_s);",
+            "test_addressof_function_with_struct", """
+                struct foo_s { int x; };
+                char myfunc(struct foo_s input) { return (char)(input.x + 42); }
+            """)
+        s = ffi.new("struct foo_s *", [5])[0]
+        assert lib.myfunc(s) == 47
+        assert not isinstance(lib.myfunc, ffi.CData)
+        assert ffi.typeof(lib.myfunc) == ffi.typeof("long(*)(struct foo_s)")
+        addr = ffi.addressof(lib, 'myfunc')
+        assert addr(s) == 47
+        assert isinstance(addr, ffi.CData)
+        assert ffi.typeof(addr) == ffi.typeof("long(*)(struct foo_s)")
+
     def test_issue198(self):
         ffi, lib = self.prepare("""
             typedef struct{...;} opaque_t;
@@ -984,5 +1009,6 @@
         assert sys.modules['_CFFI_test_import_from_lib.lib'] is lib
         from _CFFI_test_import_from_lib.lib import MYFOO
         assert MYFOO == 42
-        assert not hasattr(lib, '__dict__')
+        assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
+        assert lib.__name__ == repr(lib)
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -19,12 +19,20 @@
     wrapper is callable, and the arguments it expects and returns
     are directly the struct/union.  Calling ffi.typeof(wrapper)
     also returns the original struct/union signature.
+
+    This class cannot be used for variadic functions.
     """
     _immutable_ = True
     common_doc_str = 'direct call to the C function of the same name'
 
-    def __init__(self, space, fnptr, directfnptr, ctype,
-                 locs, rawfunctype, fnname, modulename):
+    def __init__(self, space, fnptr, directfnptr,
+                 rawfunctype, fnname, modulename):
+        # everything related to the type of the function is accessed
+        # as immutable attributes of the 'rawfunctype' object, which
+        # is a W_RawFuncType.  This gives us an obvious thing to
+        # promote in order to do the call.
+        ctype = rawfunctype.nostruct_ctype
+        locs = rawfunctype.nostruct_locs
         assert isinstance(ctype, W_CTypeFunc)
         assert ctype.cif_descr is not None     # not for '...' functions
         assert locs is None or len(ctype.fargs) == len(locs)
@@ -32,83 +40,86 @@
         self.space = space
         self.fnptr = fnptr
         self.directfnptr = directfnptr
-        self.ctype = ctype
-        self.locs = locs
         self.rawfunctype = rawfunctype
         self.fnname = fnname
         self.modulename = modulename
-        self.nargs_expected = len(ctype.fargs) - (locs is not None and
-                                                  locs[0] == 'R')
 
     def typeof(self, ffi):
         return self.rawfunctype.unwrap_as_fnptr(ffi)
 
-    @jit.unroll_safe
-    def _prepare(self, args_w, start_index):
-        # replaces struct/union arguments with ptr-to-struct/union arguments
+    def descr_call(self, args_w):
         space = self.space
-        locs = self.locs
-        fargs = self.ctype.fargs
-        for i in range(start_index, len(locs)):
-            if locs[i] != 'A':
-                continue
-            w_arg = args_w[i]
-            farg = fargs[i]      # <ptr to struct/union>
-            assert isinstance(farg, W_CTypePtrOrArray)
-            if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
-                # fast way: we are given a W_CData "struct", so just make
-                # a new W_CData "ptr-to-struct" which points to the same
-                # raw memory.  We use unsafe_escaping_ptr(), so we have to
-                # make sure the original 'w_arg' stays alive; the easiest
-                # is to build an instance of W_CDataPtrToStructOrUnion.
-                w_arg = W_CDataPtrToStructOrUnion(
-                    space, w_arg.unsafe_escaping_ptr(), farg, w_arg)
-            else:
-                # slow way: build a new "ptr to struct" W_CData by calling
-                # the equivalent of ffi.new()
-                if space.is_w(w_arg, space.w_None):
-                    continue
-                w_arg = farg.newp(w_arg)
-            args_w[i] = w_arg
-
-    def descr_call(self, args_w):
-        if len(args_w) != self.nargs_expected:
-            space = self.space
-            if self.nargs_expected == 0:
+        rawfunctype = jit.promote(self.rawfunctype)
+        ctype = rawfunctype.nostruct_ctype
+        locs = rawfunctype.nostruct_locs
+        nargs_expected = rawfunctype.nostruct_nargs
+        #
+        if len(args_w) != nargs_expected:
+            if nargs_expected == 0:
                 raise oefmt(space.w_TypeError,
                             "%s() takes no arguments (%d given)",
                             self.fnname, len(args_w))
-            elif self.nargs_expected == 1:
+            elif nargs_expected == 1:
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly one argument (%d given)",
                             self.fnname, len(args_w))
             else:
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly %d arguments (%d given)",
-                            self.fnname, self.nargs_expected, len(args_w))
+                            self.fnname, nargs_expected, len(args_w))
         #
-        if self.locs is not None:
+        if locs is not None:
             # This case is if there are structs as arguments or return values.
             # If the result we want to present to the user is "returns struct",
             # then internally allocate the struct and pass a pointer to it as
             # a first argument.
-            if self.locs[0] == 'R':
-                w_result_cdata = self.ctype.fargs[0].newp(self.space.w_None)
+            if locs[0] == 'R':
+                w_result_cdata = ctype.fargs[0].newp(space.w_None)
                 args_w = [w_result_cdata] + args_w
-                self._prepare(args_w, 1)
-                self.ctype._call(self.fnptr, args_w)    # returns w_None
+                prepare_args(space, rawfunctype, args_w, 1)
+                #
+                ctype._call(self.fnptr, args_w)    # returns w_None
+                #
                 assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
                 return w_result_cdata.structobj
             else:
                 args_w = args_w[:]
-                self._prepare(args_w, 0)
+                prepare_args(space, rawfunctype, args_w, 0)
         #
-        return self.ctype._call(self.fnptr, args_w)
+        return ctype._call(self.fnptr, args_w)
 
     def descr_repr(self, space):
         return space.wrap("<FFIFunctionWrapper for %s()>" % (self.fnname,))
 
 
+ at jit.unroll_safe
+def prepare_args(space, rawfunctype, args_w, start_index):
+    # replaces struct/union arguments with ptr-to-struct/union arguments
+    locs = rawfunctype.nostruct_locs
+    fargs = rawfunctype.nostruct_ctype.fargs
+    for i in range(start_index, len(locs)):
+        if locs[i] != 'A':
+            continue
+        w_arg = args_w[i]
+        farg = fargs[i]      # <ptr to struct/union>
+        assert isinstance(farg, W_CTypePtrOrArray)
+        if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
+            # fast way: we are given a W_CData "struct", so just make
+            # a new W_CData "ptr-to-struct" which points to the same
+            # raw memory.  We use unsafe_escaping_ptr(), so we have to
+            # make sure the original 'w_arg' stays alive; the easiest
+            # is to build an instance of W_CDataPtrToStructOrUnion.
+            w_arg = W_CDataPtrToStructOrUnion(
+                space, w_arg.unsafe_escaping_ptr(), farg, w_arg)
+        else:
+            # slow way: build a new "ptr to struct" W_CData by calling
+            # the equivalent of ffi.new()
+            if space.is_w(w_arg, space.w_None):
+                continue
+            w_arg = farg.newp(w_arg)
+        args_w[i] = w_arg
+
+
 W_FunctionWrapper.typedef = TypeDef(
         'FFIFunctionWrapper',
         __repr__ = interp2app(W_FunctionWrapper.descr_repr),
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -626,6 +626,7 @@
 
     def read_w(self, space, w_size=None):
         self._check_attached(space)
+        self._check_closed(space)
         if not self.w_decoder:
             self._unsupportedoperation(space, "not readable")
 
@@ -667,6 +668,7 @@
 
     def readline_w(self, space, w_limit=None):
         self._check_attached(space)
+        self._check_closed(space)
         self._writeflush(space)
 
         limit = convert_size(space, w_limit)
@@ -762,7 +764,7 @@
 
     def write_w(self, space, w_text):
         self._check_attached(space)
-        # self._check_closed(space)
+        self._check_closed(space)
 
         if not self.w_encoder:
             self._unsupportedoperation(space, "not writable")
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -450,3 +450,56 @@
         with _io.open(filename, 'x') as f:
             assert f.mode == 'x'
         raises(FileExistsError, _io.open, filename, 'x')
+
+
+class AppTestIoAferClose:
+    spaceconfig = dict(usemodules=['_io'])
+
+    def setup_class(cls):
+        tmpfile = udir.join('tmpfile').ensure()
+        cls.w_tmpfile = cls.space.wrap(str(tmpfile))
+
+    def test_io_after_close(self):
+        import _io
+        for kwargs in [
+                {"mode": "w"},
+                {"mode": "wb"},
+                {"mode": "w", "buffering": 1},
+                {"mode": "w", "buffering": 2},
+                {"mode": "wb", "buffering": 0},
+                {"mode": "r"},
+                {"mode": "rb"},
+                {"mode": "r", "buffering": 1},
+                {"mode": "r", "buffering": 2},
+                {"mode": "rb", "buffering": 0},
+                {"mode": "w+"},
+                {"mode": "w+b"},
+                {"mode": "w+", "buffering": 1},
+                {"mode": "w+", "buffering": 2},
+                {"mode": "w+b", "buffering": 0},
+            ]:
+            if "b" not in kwargs["mode"]:
+                kwargs["encoding"] = "ascii"
+            f = _io.open(self.tmpfile, **kwargs)
+            f.close()
+            raises(ValueError, f.flush)
+            raises(ValueError, f.fileno)
+            raises(ValueError, f.isatty)
+            raises(ValueError, f.__iter__)
+            if hasattr(f, "peek"):
+                raises(ValueError, f.peek, 1)
+            raises(ValueError, f.read)
+            if hasattr(f, "read1"):
+                raises(ValueError, f.read1, 1024)
+            if hasattr(f, "readall"):
+                raises(ValueError, f.readall)
+            if hasattr(f, "readinto"):
+                raises(ValueError, f.readinto, bytearray(1024))
+            raises(ValueError, f.readline)
+            raises(ValueError, f.readlines)
+            raises(ValueError, f.seek, 0)
+            raises(ValueError, f.tell)
+            raises(ValueError, f.truncate)
+            raises(ValueError, f.write, b"" if "b" in kwargs['mode'] else u"")
+            raises(ValueError, f.writelines, [])
+            raises(ValueError, next, f)
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -27,8 +27,10 @@
     callback_ptr = global_counter.get(userdata.addarg)
     w_callable = callback_ptr.w_callable
     argtypes = callback_ptr.argtypes
+    must_leave = False
     space = callback_ptr.space
     try:
+        must_leave = space.threadlocals.try_enter_thread(space)
         args_w = [None] * len(argtypes)
         for i in range(len(argtypes)):
             argtype = argtypes[i]
@@ -50,6 +52,8 @@
             resshape = letter2tp(space, callback_ptr.result)
             for i in range(resshape.size):
                 ll_res[i] = '\x00'
+    if must_leave:
+        space.threadlocals.leave_thread(space)
 
 class W_CallbackPtr(W_DataInstance):
 
@@ -75,6 +79,14 @@
         if tracker.DO_TRACING:
             addr = rffi.cast(lltype.Signed, self.ll_callback.ll_closure)
             tracker.trace_allocation(addr, self)
+        #
+        # We must setup the GIL here, in case the callback is invoked in
+        # some other non-Pythonic thread.  This is the same as ctypes on
+        # CPython (but only when creating a callback; on CPython it occurs
+        # as soon as we import _ctypes)
+        if space.config.translation.thread:
+            from pypy.module.thread.os_thread import setup_threads
+            setup_threads(space)
 
     def free(self):
         if tracker.DO_TRACING:
diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py
--- a/pypy/module/_socket/__init__.py
+++ b/pypy/module/_socket/__init__.py
@@ -18,6 +18,10 @@
         from rpython.rlib.rsocket import rsocket_startup
         rsocket_startup()
 
+    def shutdown(self, space):
+        from pypy.module._socket.interp_socket import close_all_sockets
+        close_all_sockets(space)
+
     def buildloaders(cls):
         from rpython.rlib import rsocket
         for name in """
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -2,7 +2,7 @@
 from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
 from rpython.rlib.rarithmetic import intmask
 
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.module._socket.interp_socket import (
     converted_error, W_Socket, addr_as_object, fill_from_object, get_error,
@@ -147,6 +147,19 @@
     newfd = rsocket.dup(fd)
     return space.wrap(newfd)
 
+ at unwrap_spec(fd=int, family=int, type=int, proto=int)
+def fromfd(space, fd, family, type, proto=0):
+    """fromfd(fd, family, type[, proto]) -> socket object
+
+    Create a socket object from the given file descriptor.
+    The remaining arguments are the same as for socket().
+    """
+    try:
+        sock = rsocket.fromfd(fd, family, type, proto)
+    except SocketError, e:
+        raise converted_error(space, e)
+    return space.wrap(W_Socket(space, sock))
+
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
                       type  =rsocket.SOCK_STREAM,
@@ -163,8 +176,8 @@
     except SocketError, e:
         raise converted_error(space, e)
     return space.newtuple([
-        space.wrap(W_Socket(sock1)),
-        space.wrap(W_Socket(sock2))
+        space.wrap(W_Socket(space, sock1)),
+        space.wrap(W_Socket(space, sock2))
     ])
 
 # The following 4 functions refuse all negative numbers, like CPython 2.6.
@@ -250,9 +263,9 @@
         ip = rsocket.inet_ntop(family, packed)
     except SocketError, e:
         raise converted_error(space, e)
-    except ValueError, e:     # XXX the message is lost in RPython
-        raise OperationError(space.w_ValueError,
-                  space.wrap(str(e)))
+    except ValueError:
+        raise oefmt(space.w_ValueError,
+                    "invalid length of packed IP address string")
     return space.wrap(ip)
 
 @unwrap_spec(family=int, type=int, proto=int, flags=int)
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -1,4 +1,5 @@
-from rpython.rlib import rsocket
+import sys
+from rpython.rlib import rsocket, rweaklist
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rsocket import (
     RSocket, AF_INET, SOCK_STREAM, SocketError, SocketErrorWithErrno,
@@ -161,16 +162,14 @@
 
 
 class W_Socket(W_Root):
-
-    # for _dealloc_warn
-    space = None
-
-    def __init__(self, sock):
+    def __init__(self, space, sock):
+        self.space = space
         self.sock = sock
+        register_socket(space, sock)
 
     def descr_new(space, w_subtype, __args__):
         sock = space.allocate_instance(W_Socket, w_subtype)
-        W_Socket.__init__(sock, RSocket.empty_rsocket())
+        W_Socket.__init__(sock, space, RSocket.empty_rsocket())
         return space.wrap(sock)
 
     @unwrap_spec(family=int, type=int, proto=int,
@@ -183,8 +182,7 @@
                                fd=space.c_filedescriptor_w(w_fileno))
             else:
                 sock = RSocket(family, type, proto)
-            W_Socket.__init__(self, sock)
-            self.space = space
+            W_Socket.__init__(self, space, sock)
         except SocketError, e:
             raise converted_error(space, e)
 
@@ -621,6 +619,45 @@
 
 
 # ____________________________________________________________
+# Automatic shutdown()/close()
+
+# On some systems, the C library does not guarantee that when the program
+# finishes, all data sent so far is really sent even if the socket is not
+# explicitly closed.  This behavior has been observed on Windows but not
+# on Linux, so far.
+NEED_EXPLICIT_CLOSE = (sys.platform == 'win32')
+
+class OpenRSockets(rweaklist.RWeakListMixin):
+    pass
+class OpenRSocketsState:
+    def __init__(self, space):
+        self.openrsockets = OpenRSockets()
+        self.openrsockets.initialize()
+
+def getopenrsockets(space):
+    if NEED_EXPLICIT_CLOSE and space.config.translation.rweakref:
+        return space.fromcache(OpenRSocketsState).openrsockets
+    else:
+        return None
+
+def register_socket(space, socket):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        openrsockets.add_handle(socket)
+
+def close_all_sockets(space):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        for sock_wref in openrsockets.get_all_handles():
+            sock = sock_wref()
+            if sock is not None:
+                try:
+                    sock.close()
+                except SocketError:
+                    pass
+
+
+# ____________________________________________________________
 # Error handling
 
 class SocketAPI:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -301,10 +301,16 @@
 
 
 class AppTestSocket:
+    spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct'])
+
     def setup_class(cls):
         cls.space = space
         cls.w_udir = space.wrap(str(udir))
 
+    def teardown_class(cls):
+        if not cls.runappdirect:
+            cls.space.sys.getmodule('_socket').shutdown(cls.space)
+
     def test_module(self):
         import _socket
         assert _socket.socket.__name__ == 'socket'
@@ -602,6 +608,12 @@
         finally:
             os.chdir(oldcwd)
 
+    def test_automatic_shutdown(self):
+        # doesn't really test anything, but at least should not explode
+        # in close_all_sockets()
+        import _socket
+        self.foo = _socket.socket()
+
     def test_subclass_init(self):
         # Socket is not created in __new__, but in __init__.
         import socket
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -143,7 +143,7 @@
     def __init__(self, ctx, protos):
         self.protos = protos
         self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
-        NPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+        NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
         # can be used to create both types of sockets
@@ -158,7 +158,7 @@
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
-        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if npn and npn.protos:
             data_ptr[0] = npn.buf
             len_ptr[0] = rffi.cast(rffi.UINT, len(npn.protos))
@@ -170,7 +170,7 @@
 
     @staticmethod
     def selectNPN_cb(s, out_ptr, outlen_ptr, server, server_len, args):
-        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if npn and npn.protos:
             client = npn.buf
             client_len = len(npn.protos)
@@ -189,7 +189,7 @@
     def __init__(self, ctx, protos):
         self.protos = protos
         self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
-        ALPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+        ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
             if libssl_SSL_CTX_set_alpn_protos(
@@ -204,7 +204,7 @@
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
-        alpn = ALPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        alpn = ALPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if alpn and alpn.protos:
             server = alpn.buf
             server_len = len(alpn.protos)
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -26,7 +26,7 @@
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['vmprof.h', 'trampoline.h'],
-    separate_module_files = [SRC.join('trampoline.asmgcc.s')],
+    separate_module_files = [SRC.join('trampoline.vmprof.s')],
     libraries = ['dl'],
     
     post_include_bits=["""
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.vmprof.s
rename from pypy/module/_vmprof/src/trampoline.asmgcc.s
rename to pypy/module/_vmprof/src/trampoline.vmprof.s
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ b/pypy/module/_vmprof/src/trampoline.vmprof.s
@@ -1,7 +1,6 @@
 // NOTE: you need to use TABs, not spaces!
         
 	.text
-	.p2align 4,,-1
 	.globl	pypy_execute_frame_trampoline
 	.type	pypy_execute_frame_trampoline, @function
 pypy_execute_frame_trampoline:
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -305,7 +305,6 @@
 
 static int remove_sigprof_timer(void) {
     static struct itimerval timer;
-    last_period_usec = 0;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = 0;
     timer.it_value.tv_sec = 0;
@@ -317,11 +316,15 @@
 }
 
 static void atfork_disable_timer(void) {
-    remove_sigprof_timer();
+    if (last_period_usec) {
+        remove_sigprof_timer();
+    }
 }
 
 static void atfork_enable_timer(void) {
-    install_sigprof_timer(last_period_usec);
+    if (last_period_usec) {
+        install_sigprof_timer(last_period_usec);
+    }
 }
 
 static int install_pthread_atfork_hooks(void) {
@@ -412,6 +415,7 @@
     if (remove_sigprof_timer() == -1) {
 		return -1;
 	}
+    last_period_usec = 0;
     if (remove_sigprof_handler() == -1) {
 		return -1;
 	}
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -24,7 +24,7 @@
         }
         """
         module = self.import_module(name='foo', init=init)
-        assert module.py_version == sys.version[:5]
+        assert module.py_version == '%d.%d.%d' % sys.version_info[:3]
         assert module.py_major_version == sys.version_info.major
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -91,7 +91,6 @@
     return w_mod
 
 
-
 class _WIN32Path(object):
     def __init__(self, path):
         self.path = path
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1223,6 +1223,53 @@
         finally:
             sys.path_hooks.pop()
 
+    def test_meta_path_import_error_1(self):
+        # as far as I can tell, the problem is that in CPython, if you
+        # use an import hook that doesn't update sys.modules, then the
+        # import succeeds; but at the same time, you can have the same
+        # result without an import hook (see test_del_from_sys_modules)
+        # and then the import fails.  This looks like even more mess
+        # to replicate, so we ignore it until someone really hits this
+        # case...
+        skip("looks like an inconsistency in CPython")
+
+        class ImportHook(object):
+            def find_module(self, fullname, path=None):
+                assert not fullname.endswith('*')
+                if fullname == 'meta_path_pseudo_module':
+                    return self
+            def load_module(self, fullname):
+                assert fullname == 'meta_path_pseudo_module'
+                # we "forget" to update sys.modules
+                return new.module('meta_path_pseudo_module')
+
+        import sys, new
+        sys.meta_path.append(ImportHook())
+        try:
+            import meta_path_pseudo_module
+        finally:
+            sys.meta_path.pop()
+
+    def test_meta_path_import_star_2(self):
+        class ImportHook(object):
+            def find_module(self, fullname, path=None):
+                if fullname.startswith('meta_path_2_pseudo_module'):
+                    return self
+            def load_module(self, fullname):
+                assert fullname == 'meta_path_2_pseudo_module'
+                m = types.ModuleType('meta_path_2_pseudo_module')
+                m.__path__ = ['/some/random/dir']
+                sys.modules['meta_path_2_pseudo_module'] = m
+                return m
+
+        import sys, types
+        sys.meta_path.append(ImportHook())
+        try:
+            exec("from meta_path_2_pseudo_module import *", {})
+        finally:
+            sys.meta_path.pop()
+
+
 class AppTestPyPyExtension(object):
     spaceconfig = dict(usemodules=['imp', 'zipimport', '__pypy__'])
 
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -368,7 +368,7 @@
             else:
                 partials.append(v)
     if special_sum != 0.0:
-        if rfloat.isnan(special_sum):
+        if rfloat.isnan(inf_sum):
             raise OperationError(space.w_ValueError, space.wrap("-inf + inf"))
         return space.wrap(special_sum)
     hi = 0.0
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -1,5 +1,6 @@
 from __future__ import with_statement
 
+import py
 from pypy.interpreter.function import Function
 from pypy.interpreter.gateway import BuiltinCode
 from pypy.module.math.test import test_direct
@@ -113,6 +114,10 @@
             ([2.**n - 2.**(n+50) + 2.**(n+52) for n in range(-1074, 972, 2)] +
              [-2.**1022],
              float.fromhex('0x1.5555555555555p+970')),
+            # infinity and nans
+            ([float("inf")], float("inf")),
+            ([float("-inf")], float("-inf")),
+            ([float("nan")], float("nan")),
             ]
 
         for i, (vals, expected) in enumerate(test_values):
@@ -124,7 +129,8 @@
             except ValueError:
                 py.test.fail("test %d failed: got ValueError, expected %r "
                           "for math.fsum(%.100r)" % (i, expected, vals))
-            assert actual == expected
+            assert actual == expected or (
+                math.isnan(actual) and math.isnan(expected))
 
     def test_factorial(self):
         import math, sys
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -266,6 +266,15 @@
     """)
     lib.aa = 5
     assert dir(lib) == ['aa', 'ff', 'my_constant']
+    #
+    aaobj = lib.__dict__['aa']
+    assert not isinstance(aaobj, int)    # some internal object instead
+    assert lib.__dict__ == {
+        'ff': lib.ff,
+        'aa': aaobj,
+        'my_constant': -45}
+    lib.__dict__['ff'] = "??"
+    assert lib.ff(10) == 15
 
 def test_verify_opaque_struct():
     ffi = FFI()
@@ -1053,5 +1062,5 @@
     assert sys.modules['_CFFI_test_import_from_lib.lib'] is lib
     from _CFFI_test_import_from_lib.lib import MYFOO
     assert MYFOO == 42
-    assert not hasattr(lib, '__dict__')
+    assert hasattr(lib, '__dict__')
     assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -167,19 +167,26 @@
         return iter(self.unerase(w_dict.dstorage)[1])
 
     def getiteritems(self, w_dict):
-        keys = self.unerase(w_dict.dstorage)[0]
-        return iter(range(len(keys)))
+        return Zip(*self.unerase(w_dict.dstorage))
 
     wrapkey = _wrapkey
 
 
-def next_item(self):
-    strategy = self.strategy
-    assert isinstance(strategy, KwargsDictStrategy)
-    for i in self.iterator:
-        keys, values_w = strategy.unerase(self.w_dict.dstorage)
-        return _wrapkey(self.space, keys[i]), values_w[i]
-    else:
-        return None, None
+class Zip(object):
+    def __init__(self, list1, list2):
+        assert len(list1) == len(list2)
+        self.list1 = list1
+        self.list2 = list2
+        self.i = 0
 
-create_iterator_classes(KwargsDictStrategy, override_next_item=next_item)
+    def __iter__(self):
+        return self
+
+    def next(self):
+        i = self.i
+        if i >= len(self.list1):
+            raise StopIteration
+        self.i = i + 1
+        return (self.list1[i], self.list2[i])
+
+create_iterator_classes(KwargsDictStrategy)
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -193,9 +193,9 @@
 
     def switch_to_object_strategy(self):
         list_w = self.getitems()
-        self.strategy = self.space.fromcache(ObjectListStrategy)
-        # XXX this is quite indirect
-        self.init_from_list_w(list_w)
+        object_strategy = self.space.fromcache(ObjectListStrategy)
+        self.strategy = object_strategy
+        object_strategy.init_from_list_w(self, list_w)
 
     def _temporarily_as_objects(self):
         if self.strategy is self.space.fromcache(ObjectListStrategy):
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -97,6 +97,8 @@
         #
         raises(TypeError, dictproxy, 3)
         raises(TypeError, dictproxy, [3])
+        #
+        {}.update(proxy)
 
 class AppTestUserObjectMethodCache(AppTestUserObject):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -160,6 +160,14 @@
         assert a == 3
         assert "KwargsDictStrategy" in self.get_strategy(d)
 
+    def test_iteritems_bug(self):
+        def f(**args):
+            return args
+
+        d = f(a=2, b=3, c=4)
+        for key, value in d.items():
+            None in d
+
     def test_unicode(self):
         """
         def f(**kwargs):
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/build_cffi_imports.py
@@ -0,0 +1,77 @@
+import sys, shutil
+from rpython.tool.runsubprocess import run_subprocess
+
+class MissingDependenciesError(Exception):
+    pass
+
+
+cffi_build_scripts = {
+    "sqlite3": "_sqlite3_build.py",
+    "audioop": "_audioop_build.py",
+    "tk": "_tkinter/tklib_build.py",
+    "curses": "_curses_build.py" if sys.platform != "win32" else None,
+    "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
+    "_gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
+    "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
+    "lzma": "_lzma_build.py",
+    "_decimal": "_decimal_build.py",
+    "xx": None,    # for testing: 'None' should be completely ignored
+    }
+
+def create_cffi_import_libraries(pypy_c, options, basedir):
+    shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
+                  ignore_errors=True)
+    failures = []
+    for key, module in sorted(cffi_build_scripts.items()):
+        if module is None or getattr(options, 'no_' + key, False):
+            continue
+        if module.endswith('.py'):
+            args = [module]
+            cwd = str(basedir.join('lib_pypy'))
+        else:
+            args = ['-c', 'import ' + module]
+            cwd = None
+        print >> sys.stderr, '*', ' '.join(args)
+        try:
+            status, stdout, stderr = run_subprocess(str(pypy_c), args, cwd=cwd)
+            if status != 0:
+                print >> sys.stderr, stdout, stderr
+                failures.append((key, module))
+        except:
+            import traceback;traceback.print_exc()
+            failures.append((key, module))
+    return failures
+
+if __name__ == '__main__':
+    import py, os
+    if '__pypy__' not in sys.builtin_module_names:
+        print 'Call with a pypy interpreter'
+        sys.exit(-1)
+
+    class Options(object):
+        pass
+
+    exename = py.path.local(sys.executable) 
+    basedir = exename
+    while not basedir.join('include').exists():
+        _basedir = basedir.dirpath()
+        if _basedir == basedir:
+            raise ValueError('interpreter %s not inside pypy repo', 
+                                 str(exename))
+        basedir = _basedir
+    options = Options()
+    print >> sys.stderr, "There should be no failures here"
+    failures = create_cffi_import_libraries(exename, options, basedir)
+    if len(failures) > 0:
+        print 'failed to build', [f[1] for f in failures]
+        assert False
+
+    # monkey patch a failure, just to test
+    print >> sys.stderr, 'This line should be followed by a traceback'
+    for k in cffi_build_scripts:
+        setattr(options, 'no_' + k, True)
+    must_fail = '_missing_build_script.py'
+    assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
+    cffi_build_scripts['should_fail'] = must_fail
+    failures = create_cffi_import_libraries(exename, options, basedir)
+    assert len(failures) == 1
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -29,6 +29,9 @@
 # XXX: don't hardcode the version
 POSIX_EXE = 'pypy3.3'
 
+from pypy.tool.build_cffi_imports import (create_cffi_import_libraries, 
+        MissingDependenciesError, cffi_build_scripts)
+
 def ignore_patterns(*patterns):
     """Function that can be used as copytree() ignore parameter.
 
@@ -44,50 +47,12 @@
 class PyPyCNotFound(Exception):
     pass
 
-class MissingDependenciesError(Exception):
-    pass
-
 def fix_permissions(dirname):
     if sys.platform != 'win32':
         os.system("chmod -R a+rX %s" % dirname)
         os.system("chmod -R g-w %s" % dirname)
 
 
-cffi_build_scripts = {
-    "sqlite3": "_sqlite3_build.py",
-    "audioop": "_audioop_build.py",
-    "tk": "_tkinter/tklib_build.py",
-    "curses": "_curses_build.py" if sys.platform != "win32" else None,
-    "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
-    "_gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
-    "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
-    "lzma": "_lzma_build.py",
-    "_decimal": "_decimal_build.py",
-    "xx": None,    # for testing: 'None' should be completely ignored
-    }
-
-def create_cffi_import_libraries(pypy_c, options, basedir):
-    shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
-                  ignore_errors=True)
-    for key, module in sorted(cffi_build_scripts.items()):
-        if module is None or getattr(options, 'no_' + key):
-            continue
-        if module.endswith('.py'):
-            args = [str(pypy_c), module]
-            cwd = str(basedir.join('lib_pypy'))
-        else:
-            args = [str(pypy_c), '-c', 'import ' + module]
-            cwd = None
-        print >> sys.stderr, '*', ' '.join(args)
-        try:
-            subprocess.check_call(args, cwd=cwd)
-        except subprocess.CalledProcessError:
-            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
-You can either install development headers package,
-add the --without-{0} option to skip packaging this
-binary CFFI extension, or say --without-cffi.""".format(key)
-            raise MissingDependenciesError(module)
-
 def pypy_runs(pypy_c, quiet=False):
     kwds = {}
     if quiet:
@@ -119,9 +84,13 @@
     if not _fake and not pypy_runs(pypy_c):
         raise OSError("Running %r failed!" % (str(pypy_c),))
     if not options.no_cffi:
-        try:
-            create_cffi_import_libraries(pypy_c, options, basedir)
-        except MissingDependenciesError:
+        failures = create_cffi_import_libraries(pypy_c, options, basedir)
+        for key, module in failures:
+            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
+                You can either install development headers package,
+                add the --without-{0} option to skip packaging this
+                binary CFFI extension, or say --without-cffi.""".format(key)
+        if len(failures) > 0:
             return 1, None
 
     if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -1207,7 +1207,8 @@
     def nomoreblocks(self, ctx):
         w_exc = self.w_exc
         if w_exc.w_type == const(ImportError):
-            msg = 'import statement always raises %s' % self
+            msg = 'ImportError is raised in RPython: %s' % (
+                getattr(w_exc.w_value, 'value', '<not a constant message>'),)
             raise ImportError(msg)
         link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock)
         ctx.recorder.crnt_block.closeblock(link)
diff --git a/rpython/flowspace/test/cant_import.py b/rpython/flowspace/test/cant_import.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/cant_import.py
@@ -0,0 +1,1 @@
+raise ImportError("some explanation here")
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -816,6 +816,12 @@
             from rpython import this_does_not_exist
         py.test.raises(ImportError, 'self.codetest(f)')
 
+    def test_importerror_3(self):
+        def f():
+            import rpython.flowspace.test.cant_import
+        e = py.test.raises(ImportError, 'self.codetest(f)')
+        assert "some explanation here" in str(e.value)
+
     def test_relative_import(self):
         def f():
             from ..objspace import build_flow
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -73,8 +73,6 @@
                 self.emit_pending_zeros()
             elif op.can_malloc():
                 self.emitting_an_operation_that_can_collect()
-            elif op.getopnum() == rop.DEBUG_MERGE_POINT:
-                continue # ignore debug_merge_points
             elif op.getopnum() == rop.LABEL:
                 self.emitting_an_operation_that_can_collect()
                 self.known_lengths.clear()
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -60,6 +60,26 @@
             if not value.is_unescaped:
                 del d[value]
 
+
+class FieldUpdater(object):
+    def __init__(self, heapcache, value, cache, fieldvalue):
+        self.heapcache = heapcache
+        self.value = value
+        self.cache = cache
+        if fieldvalue is not None:
+            self.currfieldbox = fieldvalue.box
+        else:
+            self.currfieldbox = None
+
+    def getfield_now_known(self, fieldbox):
+        fieldvalue = self.heapcache.getvalue(fieldbox)
+        self.cache.read_now_known(self.value, fieldvalue)
+
+    def setfield(self, fieldbox):
+        fieldvalue = self.heapcache.getvalue(fieldbox)
+        self.cache.do_write_with_aliasing(self.value, fieldvalue)
+
+
 class HeapCache(object):
     def __init__(self):
         self.reset()
@@ -98,9 +118,9 @@
         self.heap_cache = {}
         self.heap_array_cache = {}
 
-    def getvalue(self, box):
+    def getvalue(self, box, create=True):
         value = self.values.get(box, None)
-        if not value:
+        if not value and create:
             value = self.values[box] = HeapCacheValue(box)
         return value
 
@@ -111,25 +131,26 @@
         self.mark_escaped(opnum, descr, argboxes)
         self.clear_caches(opnum, descr, argboxes)
 
+    def _escape_from_write(self, box, fieldbox):
+        value = self.getvalue(box, create=False)
+        fieldvalue = self.getvalue(fieldbox, create=False)
+        if (value is not None and value.is_unescaped and
+                fieldvalue is not None and fieldvalue.is_unescaped):
+            if value.dependencies is None:
+                value.dependencies = []
+            value.dependencies.append(fieldvalue)
+        elif fieldvalue is not None:
+            self._escape(fieldvalue)
+
     def mark_escaped(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
-            value, fieldvalue = self.getvalues(argboxes)
-            if value.is_unescaped and fieldvalue.is_unescaped:
-                if value.dependencies is None:
-                    value.dependencies = []
-                value.dependencies.append(fieldvalue)
-            else:
-                self._escape(fieldvalue)
+            box, fieldbox = argboxes
+            self._escape_from_write(box, fieldbox)
         elif opnum == rop.SETARRAYITEM_GC:
             assert len(argboxes) == 3
-            value, indexvalue, fieldvalue = self.getvalues(argboxes)
-            if value.is_unescaped and fieldvalue.is_unescaped:
-                if value.dependencies is None:
-                    value.dependencies = []
-                value.dependencies.append(fieldvalue)
-            else:
-                self._escape(fieldvalue)
+            box, indexbox, fieldbox = argboxes
+            self._escape_from_write(box, fieldbox)
         elif (opnum == rop.CALL and
               descr.get_extra_info().oopspecindex == descr.get_extra_info().OS_ARRAYCOPY and
               isinstance(argboxes[3], ConstInt) and
@@ -153,7 +174,7 @@
                 self._escape_box(box)
 
     def _escape_box(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if not value:
             return
         self._escape(value)
@@ -261,7 +282,7 @@
         self.reset_keep_likely_virtuals()
 
     def is_class_known(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.known_class
         return False
@@ -270,7 +291,7 @@
         self.getvalue(box).known_class = True
 
     def is_nonstandard_virtualizable(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.nonstandard_virtualizable
         return False
@@ -279,13 +300,13 @@
         self.getvalue(box).nonstandard_virtualizable = True
 
     def is_unescaped(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.is_unescaped
         return False
 
     def is_likely_virtual(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             return value.likely_virtual
         return False
@@ -301,7 +322,7 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value:
             cache = self.heap_cache.get(descr, None)
             if cache:
@@ -310,26 +331,28 @@
                     return tovalue.box
         return None
 
-    def getfield_now_known(self, box, descr, fieldbox):
+    def get_field_updater(self, box, descr):
         value = self.getvalue(box)
-        fieldvalue = self.getvalue(fieldbox)
         cache = self.heap_cache.get(descr, None)
         if cache is None:
             cache = self.heap_cache[descr] = CacheEntry()
-        cache.read_now_known(value, fieldvalue)
+            fieldvalue = None
+        else:
+            fieldvalue = cache.read(value)
+        return FieldUpdater(self, value, cache, fieldvalue)
+
+    def getfield_now_known(self, box, descr, fieldbox):
+        upd = self.get_field_updater(box, descr)
+        upd.getfield_now_known(fieldbox)
 
     def setfield(self, box, fieldbox, descr):
-        cache = self.heap_cache.get(descr, None)
-        if cache is None:
-            cache = self.heap_cache[descr] = CacheEntry()
-        value = self.getvalue(box)
-        fieldvalue = self.getvalue(fieldbox)
-        cache.do_write_with_aliasing(value, fieldvalue)
+        upd = self.get_field_updater(box, descr)
+        upd.setfield(fieldbox)
 
     def getarrayitem(self, box, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
             return None
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value is None:
             return None
         index = indexbox.getint()
@@ -373,7 +396,7 @@
             indexcache.do_write_with_aliasing(value, fieldvalue)
 
     def arraylen(self, box):
-        value = self.values.get(box, None)
+        value = self.getvalue(box, create=False)
         if value and value.length:
             return value.length.box
         return None
@@ -383,7 +406,7 @@
         value.length = self.getvalue(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        value = self.values.get(oldbox, None)
+        value = self.getvalue(oldbox, create=False)
         if value is None:
             return
         value.box = newbox
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -187,7 +187,12 @@
         [i0]
         jump(i0)
         """
-        self.optimize_loop(ops, expected)
+        short = """
+        [i2]
+        p3 = cast_int_to_ptr(i2)
+        jump(i2)
+        """
+        self.optimize_loop(ops, expected, expected_short=short)
 
     def test_reverse_of_cast_2(self):
         ops = """
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -649,16 +649,16 @@
 
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
-        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
-        if tobox is not None:
+        upd = self.metainterp.heapcache.get_field_updater(box, fielddescr)
+        if upd.currfieldbox is not None:
             # sanity check: see whether the current struct value
             # corresponds to what the cache thinks the value is
             resbox = executor.execute(self.metainterp.cpu, self.metainterp,
                                       rop.GETFIELD_GC, fielddescr, box)
-            assert resbox.constbox().same_constant(tobox.constbox())
-            return tobox
+            assert resbox.constbox().same_constant(upd.currfieldbox.constbox())
+            return upd.currfieldbox
         resbox = self.execute_with_descr(opnum, fielddescr, box)
-        self.metainterp.heapcache.getfield_now_known(box, fielddescr, resbox)
+        upd.getfield_now_known(resbox)
         return resbox
 
     @arguments("box", "descr", "orgpc")
@@ -679,10 +679,11 @@
 
     @arguments("box", "box", "descr")
     def _opimpl_setfield_gc_any(self, box, valuebox, fielddescr):
-        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
-        if tobox is valuebox:
+        upd = self.metainterp.heapcache.get_field_updater(box, fielddescr)
+        if upd.currfieldbox is valuebox:
             return
-        self.metainterp.execute_setfield_gc(fielddescr, box, valuebox)
+        self.metainterp.execute_and_record(rop.SETFIELD_GC, fielddescr, box, valuebox)
+        upd.setfield(valuebox)
         # The following logic is disabled because buggy.  It is supposed
         # to be: not(we're writing null into a freshly allocated object)
         # but the bug is that is_unescaped() can be True even after the
@@ -1922,9 +1923,10 @@
         resbox = executor.execute(self.cpu, self, opnum, descr, *argboxes)
         if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST:
             return self._record_helper_pure(opnum, resbox, descr, *argboxes)
-        else:
-            return self._record_helper_nonpure_varargs(opnum, resbox, descr,
-                                                       list(argboxes))
+        if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
+            return self._record_helper_ovf(opnum, resbox, descr, *argboxes)
+        return self._record_helper_nonpure_varargs(opnum, resbox, descr,
+                                                   list(argboxes))
 
     @specialize.arg(1)
     def execute_and_record_varargs(self, opnum, argboxes, descr=None):
@@ -1951,6 +1953,12 @@
             resbox = resbox.nonconstbox()    # ensure it is a Box
             return self._record_helper_nonpure_varargs(opnum, resbox, descr, list(argboxes))
 
+    def _record_helper_ovf(self, opnum, resbox, descr, *argboxes):
+        if (self.last_exc_value_box is None and
+                self._all_constants(*argboxes)):
+            return resbox.constbox()
+        return self._record_helper_nonpure_varargs(opnum, resbox, descr, list(argboxes))
+
     def _record_helper_pure_varargs(self, opnum, resbox, descr, argboxes):
         canfold = self._all_constants_varargs(argboxes)
         if canfold:
@@ -1962,10 +1970,6 @@
 
     def _record_helper_nonpure_varargs(self, opnum, resbox, descr, argboxes):
         assert resbox is None or isinstance(resbox, Box)
-        if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST and
-            self.last_exc_value_box is None and
-            self._all_constants_varargs(argboxes)):
-            return resbox.constbox()
         # record the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum, Counters.RECORDED_OPS)
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -52,21 +52,22 @@
             return (op.opname in LL_OPERATIONS and
                     LL_OPERATIONS[op.opname].canmallocgc)
 
-def find_initializing_stores(collect_analyzer, graph):
-    from rpython.flowspace.model import mkentrymap
-    entrymap = mkentrymap(graph)
-    # a bit of a hackish analysis: if a block contains a malloc and check that
-    # the result is not zero, then the block following the True link will
-    # usually initialize the newly allocated object
-    result = set()
-    def find_in_block(block, mallocvars):
+def propagate_no_write_barrier_needed(result, block, mallocvars,
+                                      collect_analyzer, entrymap,
+                                      startindex=0):
+    # We definitely know that no write barrier is needed in the 'block'
+    # for any of the variables in 'mallocvars'.  Propagate this information
+    # forward.  Note that "definitely know" implies that we just did either
+    # a fixed-size malloc (variable-size might require card marking), or
+    # that we just did a full write barrier (not just for card marking).
+    if 1:       # keep indentation
         for i, op in enumerate(block.operations):
+            if i < startindex:
+                continue
             if op.opname in ("cast_pointer", "same_as"):
                 if op.args[0] in mallocvars:
                     mallocvars[op.result] = True
             elif op.opname in ("setfield", "setarrayitem", "setinteriorfield"):
-                # note that 'mallocvars' only tracks fixed-size mallocs,
-                # so no risk that they use card marking
                 TYPE = op.args[-1].concretetype
                 if (op.args[0] in mallocvars and
                     isinstance(TYPE, lltype.Ptr) and
@@ -83,7 +84,15 @@
                 if var in mallocvars:
                     newmallocvars[exit.target.inputargs[i]] = True
             if newmallocvars:
-                find_in_block(exit.target, newmallocvars)
+                propagate_no_write_barrier_needed(result, exit.target,
+                                                  newmallocvars,
+                                                  collect_analyzer, entrymap)
+
+def find_initializing_stores(collect_analyzer, graph, entrymap):
+    # a bit of a hackish analysis: if a block contains a malloc and check that
+    # the result is not zero, then the block following the True link will
+    # usually initialize the newly allocated object
+    result = set()
     mallocnum = 0
     blockset = set(graph.iterblocks())
     while blockset:
@@ -113,7 +122,8 @@
         target = exit.target
         mallocvars = {target.inputargs[index]: True}
         mallocnum += 1
-        find_in_block(target, mallocvars)
+        propagate_no_write_barrier_needed(result, target, mallocvars,
+                                          collect_analyzer, entrymap)
     #if result:
     #    print "found %s initializing stores in %s" % (len(result), graph.name)
     return result
@@ -698,8 +708,11 @@
                                 " %s" % func)
 
         if self.write_barrier_ptr:
+            from rpython.flowspace.model import mkentrymap
+            self._entrymap = mkentrymap(graph)
             self.clean_sets = (
-                find_initializing_stores(self.collect_analyzer, graph))
+                find_initializing_stores(self.collect_analyzer, graph,
+                                         self._entrymap))
             if self.gcdata.gc.can_optimize_clean_setarrayitems():
                 self.clean_sets = self.clean_sets.union(
                     find_clean_setarrayitems(self.collect_analyzer, graph))
@@ -1269,6 +1282,17 @@
                 hop.genop("direct_call", [self.write_barrier_ptr,
                                           self.c_const_gc,
                                           v_structaddr])
+                # we just did a full write barrier here, so we can use
+                # this helper to propagate this knowledge forward and
+                # avoid to repeat the write barrier.
+                if self.curr_block is not None:   # for tests
+                    assert self.curr_block.operations[hop.index] is hop.spaceop
+                    propagate_no_write_barrier_needed(self.clean_sets,
+                                                      self.curr_block,

From noreply at buildbot.pypy.org  Sat Jul  4 18:18:48 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 18:18:48 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #115: document the assignments
 where we store a list or a dict.
Message-ID: <20150704161848.F0D471C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2202:2763b22591f2
Date: 2015-07-04 18:19 +0200
http://bitbucket.org/cffi/cffi/changeset/2763b22591f2/

Log:	Issue #115: document the assignments where we store a list or a
	dict.

diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -200,6 +200,25 @@
     p = ffi.new("foo_t *", [5, 3])         # length 3 with 0 in the array
     p = ffi.new("foo_t *", {'y': 3})       # length 3 with 0 everywhere
 
+Finally, note that any Python object used as initializer can also be
+used directly without ``ffi.new()`` in assignments to array items or
+struct fields.  In fact, ``p = ffi.new("T*", initializer)`` is
+equivalent to ``p = ffi.new("T*"); p[0] = initializer``.  Examples:
+
+.. code-block:: python
+
+    # if 'p' is a <cdata 'int[5][5]'>
+    p[2] = [10, 20]             # writes to p[2][0] and p[2][1]
+
+    # if 'p' is a <cdata 'foo_t *'>, and foo_t has fields x, y and z
+    p[0] = {'x': 10, 'z': 20}   # writes to p.x and p.z; p.y unmodified
+
+    # if, on the other hand, foo_t has a field 'char a[5]':
+    p.a = "abc"                 # writes 'a', 'b', 'c' and '\0'; p.a[4] unmodified
+
+In function calls, when passing arguments, these rules can be used too;
+see `Function calls`_.
+
 
 Python 3 support
 ----------------
@@ -297,8 +316,8 @@
 You can also pass unicode strings as ``wchar_t *`` arguments.  Note that
 in general, there is no difference between C argument declarations that
 use ``type *`` or ``type[]``.  For example, ``int *`` is fully
-equivalent to ``int[]`` or ``int[5]``.  So you can pass an ``int *`` as
-a list of integers:
+equivalent to ``int[]`` (or even ``int[5]``; the 5 is ignored).  So you
+can pass an ``int *`` as a list of integers:
 
 .. code-block:: python
 
@@ -306,6 +325,10 @@
 
     lib.do_something_with_array([1, 2, 3, 4, 5])
 
+See `Reference: conversions`_ for a similar way to pass ``struct foo_s
+*`` arguments---but in general, it is clearer to simply pass
+``ffi.new('struct foo_s *', initializer)``.
+
 CFFI supports passing and returning structs to functions and callbacks.
 Example:
 
@@ -862,6 +885,8 @@
    function with a ``char *`` argument to which you pass a Python
    string will not actually modify the array of characters passed in,
    and so passes directly a pointer inside the Python string object.
+   (PyPy might in the future do the same, but it is harder because a
+   string object can move in memory when the GC runs.)
 
 `(**)` C function calls are done with the GIL released.
 

From noreply at buildbot.pypy.org  Sat Jul  4 18:56:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 18:56:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a FAQ entry about sandboxing
Message-ID: <20150704165620.5F5931C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78427:a3cfaaba2f49
Date: 2015-07-04 18:56 +0200
http://bitbucket.org/pypy/pypy/changeset/a3cfaaba2f49/

Log:	Add a FAQ entry about sandboxing

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -70,6 +70,20 @@
 .. _`use virtualenv (as documented here)`: getting-started.html#installing-using-virtualenv
 
 
+Module xyz does not work in the sandboxed PyPy?
+-----------------------------------------------
+
+You cannot import *any* extension module in a `sandboxed PyPy`_,
+sorry.  Even the built-in modules available are very limited.
+Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
+it is only a proof of concept.  It seriously requires someone working
+on it.  Before this occurs, it can only be used it for "pure Python"
+examples: programs that import mostly nothing (or only pure Python
+modules, recursively).
+
+.. _`sandboxed PyPy`: sandbox.html
+
+
 .. _`See below.`:
 
 Do CPython Extension modules work with PyPy?

From noreply at buildbot.pypy.org  Sat Jul  4 19:25:54 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat,  4 Jul 2015 19:25:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test after changes in 36a1899115e0
Message-ID: <20150704172554.A801F1C0170@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78428:75d199ee1dc7
Date: 2015-07-04 19:23 +0200
http://bitbucket.org/pypy/pypy/changeset/75d199ee1dc7/

Log:	fix test after changes in 36a1899115e0

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -14,42 +14,45 @@
         assert len(log.loops) == 1
         loop = log._filter(log.loops[0])
         assert loop.match("""
-            guard_class(p0, #, descr=...)
-            p4 = getfield_gc_pure(p0, descr=<FieldP pypy.module.micronumpy.iterators.ArrayIter.inst_array \d+>)
-            i5 = getfield_gc(p2, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset \d+>)
+            guard_class(p1, #, descr=...)
+            p4 = getfield_gc_pure(p1, descr=<FieldP pypy.module.micronumpy.iterators.ArrayIter.inst_array \d+>)
+            i5 = getfield_gc(p0, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset \d+>)
             p6 = getfield_gc_pure(p4, descr=<FieldP pypy.module.micronumpy.concrete.BaseConcreteArray.inst_dtype \d+>)
             p7 = getfield_gc_pure(p6, descr=<FieldP pypy.module.micronumpy.descriptor.W_Dtype.inst_itemtype \d+>)
             guard_class(p7, ConstClass(Float64), descr=...)
             i9 = getfield_gc_pure(p4, descr=<FieldU pypy.module.micronumpy.concrete.BaseConcreteArray.inst_storage \d+>)
-            f10 = raw_load(i9, i5, descr=<ArrayF \d+>)
-            i11 = getfield_gc_pure(p7, descr=<FieldU pypy.module.micronumpy.types.BaseType.inst_native \d+>)
-            guard_true(i11, descr=...)
+            i10 = getfield_gc_pure(p6, descr=<FieldU pypy.module.micronumpy.descriptor.W_Dtype.inst_byteorder \d+>)
+            i12 = int_eq(i10, 61)
+            i14 = int_eq(i10, 60)
+            i15 = int_or(i12, i14)
+            f16 = raw_load(i9, i5, descr=<ArrayF \d+>)
+            guard_true(i15, descr=...)
             guard_not_invalidated(descr=...)
-            i12 = cast_float_to_int(f10)
-            i14 = int_and(i12, 255)
-            guard_true(i14, descr=...)
-            i15 = getfield_gc_pure(p1, descr=<FieldU pypy.module.micronumpy.boxes.W_BoolBox.inst_value \d+>)
-            i16 = int_is_true(i15)
-            guard_false(i16, descr=...)
-            i20 = getfield_gc(p2, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index \d+>)
-            i21 = getfield_gc_pure(p0, descr=<FieldU pypy.module.micronumpy.iterators.ArrayIter.inst_track_index \d+>)
-            guard_true(i21, descr=...)
-            i23 = int_add(i20, 1)
-            p24 = getfield_gc_pure(p2, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices \d+>)
-            i25 = getfield_gc_pure(p0, descr=<FieldS pypy.module.micronumpy.iterators.ArrayIter.inst_contiguous \d+>)
-            i26 = int_is_true(i25)
-            guard_true(i26, descr=...)
-            i27 = getfield_gc_pure(p6, descr=<FieldS pypy.module.micronumpy.descriptor.W_Dtype.inst_elsize \d+>)
-            i28 = int_add(i5, i27)
-            i29 = getfield_gc_pure(p0, descr=<FieldS pypy.module.micronumpy.iterators.ArrayIter.inst_size \d+>)
-            i30 = int_ge(i23, i29)
-            guard_false(i30, descr=...)
-            p32 = new_with_vtable(#)
+            i17 = cast_float_to_int(f16)
+            i19 = int_and(i17, 255)
+            guard_true(i19, descr=...)
+            i20 = getfield_gc_pure(p2, descr=<FieldU pypy.module.micronumpy.boxes.W_BoolBox.inst_value \d+>)
+            i21 = int_is_true(i20)
+            guard_false(i21, descr=...)
+            i22 = getfield_gc(p0, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index \d+>)
+            i23 = getfield_gc_pure(p1, descr=<FieldU pypy.module.micronumpy.iterators.ArrayIter.inst_track_index \d+>)
+            guard_true(i23, descr=...)
+            i25 = int_add(i22, 1)
+            p26 = getfield_gc_pure(p0, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices \d+>)
+            i27 = getfield_gc_pure(p1, descr=<FieldS pypy.module.micronumpy.iterators.ArrayIter.inst_contiguous \d+>)
+            i28 = int_is_true(i27)
+            guard_true(i28, descr=...)
+            i29 = getfield_gc_pure(p6, descr=<FieldS pypy.module.micronumpy.descriptor.W_Dtype.inst_elsize \d+>)
+            i30 = int_add(i5, i29)
+            i31 = getfield_gc_pure(p1, descr=<FieldS pypy.module.micronumpy.iterators.ArrayIter.inst_size \d+>)
+            i32 = int_ge(i25, i31)
+            guard_false(i32, descr=...)
+            p34 = new_with_vtable(#)
             {{{
-            setfield_gc(p32, i23, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index \d+>)
-            setfield_gc(p32, p24, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices \d+>)
-            setfield_gc(p32, i28, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset \d+>)
-            setfield_gc(p32, p0, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_iterator \d+>)
+            setfield_gc(p34, p1, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_iterator \d+>)
+            setfield_gc(p34, i25, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index \d+>)
+            setfield_gc(p34, p26, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices \d+>)
+            setfield_gc(p34, i30, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset \d+>)
             }}}
             jump(..., descr=...)
         """)

From noreply at buildbot.pypy.org  Sat Jul  4 19:39:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 19:39:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: add comment
Message-ID: <20150704173904.CEB641C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78429:318e8acc19df
Date: 2015-07-04 19:39 +0200
http://bitbucket.org/pypy/pypy/changeset/318e8acc19df/

Log:	add comment

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1333,7 +1333,7 @@
         loc1, = arglocs
         assert isinstance(resloc, RegLoc)
         assert isinstance(loc1, RegLoc)
-        self.mc.MOVD32_xr(resloc.value, loc1.value)
+        self.mc.MOVD32_xr(resloc.value, loc1.value)   # zero-extending
 
     def genop_llong_eq(self, op, arglocs, resloc):
         loc1, loc2, locxtmp = arglocs

From noreply at buildbot.pypy.org  Sat Jul  4 19:56:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 19:56:59 +0200 (CEST)
Subject: [pypy-commit] cffi default: Use the logic in cgc.c to implement
 ffi.gc() also for the pure Python
Message-ID: <20150704175659.D33351C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2203:e12558a3ce6b
Date: 2015-07-04 19:57 +0200
http://bitbucket.org/cffi/cffi/changeset/e12558a3ce6b/

Log:	Use the logic in cgc.c to implement ffi.gc() also for the pure
	Python in-line version of FFI

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5512,6 +5512,9 @@
                            (PyObject *)&CTypeDescr_Type);
 }
 
+/* forward: implemented in ffi_obj.c */
+static PyObject *b_gcp(PyObject *self, PyObject *args);
+
 /************************************************************/
 
 static char _testfunc0(char a, char b)
@@ -5817,6 +5820,7 @@
     {"newp_handle", b_newp_handle, METH_VARARGS},
     {"from_handle", b_from_handle, METH_O},
     {"from_buffer", b_from_buffer, METH_VARARGS},
+    {"gcp", b_gcp, METH_VARARGS},
 #ifdef MS_WIN32
     {"getwinerror", (PyCFunction)b_getwinerror, METH_VARARGS | METH_KEYWORDS},
 #endif
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -680,6 +680,19 @@
     return gc_weakrefs_build(self, cd, destructor);
 }
 
+static PyObject *b_gcp(PyObject *self, PyObject *args)
+{
+    /* for in-line mode */
+    static FFIObject *ffi1 = NULL;
+
+    if (ffi1 == NULL) {
+        ffi1 = ffi_internal_new(&FFI_Type, NULL);
+        if (ffi1 == NULL)
+            return NULL;
+    }
+    return ffi_gc(ffi1, args, NULL);
+}
+
 PyDoc_STRVAR(ffi_callback_doc,
 "Return a callback object or a decorator making such a callback object.\n"
 "'cdecl' must name a C function pointer type.  The callback invokes the\n"
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -327,6 +327,13 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
+        try:
+            gcp = self._backend.gcp
+        except AttributeError:
+            pass
+        else:
+            return gcp(cdata, destructor)
+        #
         with self._lock:
             try:
                 gc_weakrefs = self.gc_weakrefs
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1503,16 +1503,19 @@
 
     def test_gc_finite_list(self):
         ffi = FFI(backend=self.Backend())
+        public = not hasattr(ffi._backend, 'gcp')
         p = ffi.new("int *", 123)
         keepalive = []
         for i in range(10):
             keepalive.append(ffi.gc(p, lambda p: None))
-            assert len(ffi.gc_weakrefs.data) == i + 1  #should be a private attr
+            if public:
+                assert len(ffi.gc_weakrefs.data) == i + 1
         del keepalive[:]
         import gc; gc.collect(); gc.collect()
         for i in range(10):
             keepalive.append(ffi.gc(p, lambda p: None))
-        assert len(ffi.gc_weakrefs.data) == 10
+        if public:
+            assert len(ffi.gc_weakrefs.data) == 10
 
     def test_CData_CType(self):
         ffi = FFI(backend=self.Backend())

From noreply at buildbot.pypy.org  Sat Jul  4 20:09:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 20:09:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix in the annotator! Still!
Message-ID: <20150704180904.7E0851C0822@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78430:705d7abb7534
Date: 2015-07-04 20:09 +0200
http://bitbucket.org/pypy/pypy/changeset/705d7abb7534/

Log:	Test and fix in the annotator! Still!

diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -156,7 +156,10 @@
     if s_type.is_constant():
         typ = s_type.const
         if issubclass(typ, rpython.rlib.rarithmetic.base_int):
-            r.const = issubclass(s_obj.knowntype, typ)
+            try:
+                r.const = issubclass(s_obj.knowntype, typ)
+            except TypeError:    # s_obj.knowntype is not a Python type at all
+                r.const = False
         else:
             if typ == long:
                 getbookkeeper().warning("isinstance(., long) is not RPython")
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -1135,7 +1135,7 @@
 
         assert famCinit.calltables == {(1, (), False): [{mdescCinit.funcdesc: gfCinit}] }
 
-    def test_isinstance_usigned(self):
+    def test_isinstance_unsigned_1(self):
         def f(x):
             return isinstance(x, r_uint)
         def g():
@@ -1145,6 +1145,18 @@
         s = a.build_types(g, [])
         assert s.const == True
 
+    def test_isinstance_unsigned_2(self):
+        class Foo:
+            pass
+        def f(x):
+            return isinstance(x, r_uint)
+        def g():
+            v = Foo()
+            return f(v)
+        a = self.RPythonAnnotator()
+        s = a.build_types(g, [])
+        assert s.const == False
+
     def test_isinstance_base_int(self):
         def f(x):
             return isinstance(x, base_int)

From noreply at buildbot.pypy.org  Sat Jul  4 20:38:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 20:38:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: Import and copy cffi/e12558a3ce6b
Message-ID: <20150704183852.236B91C1048@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78431:9a4cb1384683
Date: 2015-07-04 20:39 +0200
http://bitbucket.org/pypy/pypy/changeset/9a4cb1384683/

Log:	Import and copy cffi/e12558a3ce6b

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -327,6 +327,13 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
+        try:
+            gcp = self._backend.gcp
+        except AttributeError:
+            pass
+        else:
+            return gcp(cdata, destructor)
+        #
         with self._lock:
             try:
                 gc_weakrefs = self.gc_weakrefs
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -37,6 +37,7 @@
         'from_handle': 'handle.from_handle',
         '_get_types': 'func._get_types',
         'from_buffer': 'func.from_buffer',
+        'gcp': 'func.gcp',
 
         'string': 'func.string',
         'buffer': 'cbuffer.buffer',
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -539,13 +539,18 @@
 
 
 @jit.dont_look_inside
-def W_FFIObject___new__(space, w_subtype, __args__):
-    r = space.allocate_instance(W_FFIObject, w_subtype)
+def make_plain_ffi_object(space, w_ffitype=None):
+    if w_ffitype is None:
+        w_ffitype = space.gettypefor(W_FFIObject)
+    r = space.allocate_instance(W_FFIObject, w_ffitype)
     # get in 'src_ctx' a NULL which translation doesn't consider to be constant
     src_ctx = rffi.cast(parse_c_type.PCTX, 0)
     r.__init__(space, src_ctx)
     return space.wrap(r)
 
+def W_FFIObject___new__(space, w_subtype, __args__):
+    return make_plain_ffi_object(space, w_subtype)
+
 def make_CData(space):
     return space.gettypefor(W_CData)
 
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -105,3 +105,18 @@
                     "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
+
+# ____________________________________________________________
+
+class ConstantFFI:
+    ffi1 = None
+    def _cleanup_(self):
+        self.ffi1 = None
+constant_ffi = ConstantFFI()
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData)
+def gcp(space, w_cdata, w_destructor):
+    if constant_ffi.ffi1 is None:
+        from pypy.module._cffi_backend import ffi_obj
+        constant_ffi.ffi1 = ffi_obj.make_plain_ffi_object(space)
+    return constant_ffi.ffi1.descr_gc(w_cdata, w_destructor)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -4,6 +4,8 @@
     spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
 
     def teardown_method(self, meth):
+        from pypy.module._cffi_backend.func import constant_ffi
+        constant_ffi._cleanup_()
         _clean_cache(self.space)
 
     def test_ffi_new(self):
@@ -222,9 +224,10 @@
             assert p1[0] == 123
             seen.append(1)
         ffi.gc(p, destructor=destructor)    # instantly forgotten
+        _cffi1_backend.gcp(p, destructor=destructor)
         for i in range(5):
             if seen:
                 break
             import gc
             gc.collect()
-        assert seen == [1]
+        assert seen == [1, 1]
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1504,16 +1504,19 @@
 
     def test_gc_finite_list(self):
         ffi = FFI(backend=self.Backend())
+        public = not hasattr(ffi._backend, 'gcp')
         p = ffi.new("int *", 123)
         keepalive = []
         for i in range(10):
             keepalive.append(ffi.gc(p, lambda p: None))
-            assert len(ffi.gc_weakrefs.data) == i + 1  #should be a private attr
+            if public:
+                assert len(ffi.gc_weakrefs.data) == i + 1
         del keepalive[:]
         import gc; gc.collect(); gc.collect()
         for i in range(10):
             keepalive.append(ffi.gc(p, lambda p: None))
-        assert len(ffi.gc_weakrefs.data) == 10
+        if public:
+            assert len(ffi.gc_weakrefs.data) == 10
 
     def test_CData_CType(self):
         ffi = FFI(backend=self.Backend())

From noreply at buildbot.pypy.org  Sat Jul  4 21:51:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 21:51:56 +0200 (CEST)
Subject: [pypy-commit] cffi default: New argument "onerror" on ffi.callback()
Message-ID: <20150704195156.235291C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2204:2b30856ad741
Date: 2015-07-04 21:52 +0200
http://bitbucket.org/cffi/cffi/changeset/2b30856ad741/

Log:	New argument "onerror" on ffi.callback()

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -4725,9 +4725,11 @@
 #endif
     f = PySys_GetObject("stderr");
     if (f != NULL) {
-        PyFile_WriteString("From cffi callback ", f);
-        PyFile_WriteObject(obj, f, 0);
-        PyFile_WriteString(":\n", f);
+        if (obj != NULL) {
+            PyFile_WriteString("From cffi callback ", f);
+            PyFile_WriteObject(obj, f, 0);
+            PyFile_WriteString(":\n", f);
+        }
         if (extra_error_line != NULL)
             PyFile_WriteString(extra_error_line, f);
         PyErr_Display(t, v, tb);
@@ -4752,6 +4754,7 @@
     PyObject *py_args = NULL;
     PyObject *py_res = NULL;
     PyObject *py_rawerr;
+    PyObject *onerror_cb;
     Py_ssize_t i, n;
     char *extra_error_line = NULL;
 
@@ -4789,7 +4792,35 @@
     return;
 
  error:
+    onerror_cb = PyTuple_GET_ITEM(cb_args, 3);
+    if (onerror_cb != Py_None) {
+        PyObject *exc1, *val1, *tb1, *res1, *exc2, *val2, *tb2;
+        PyErr_Fetch(&exc1, &val1, &tb1);
+        PyErr_NormalizeException(&exc1, &val1, &tb1);
+        res1 = PyObject_CallFunctionObjArgs(onerror_cb,
+                                            exc1 ? exc1 : Py_None,
+                                            val1 ? val1 : Py_None,
+                                            tb1  ? tb1  : Py_None,
+                                            NULL);
+        Py_XDECREF(res1);
+
+        if (!PyErr_Occurred()) {
+            Py_XDECREF(exc1);
+            Py_XDECREF(val1);
+            Py_XDECREF(tb1);
+            goto no_more_exception;
+        }
+        /* double exception! print a double-traceback... */
+        PyErr_Fetch(&exc2, &val2, &tb2);
+        PyErr_Restore(exc1, val1, tb1);
+        _my_PyErr_WriteUnraisable(py_ob, extra_error_line);
+        PyErr_Restore(exc2, val2, tb2);
+        extra_error_line = ("\nDuring the call to 'onerror', "
+                            "another exception occurred:\n\n");
+        py_ob = NULL;
+    }
     _my_PyErr_WriteUnraisable(py_ob, extra_error_line);
+ no_more_exception:
     if (SIGNATURE(1)->ct_size > 0) {
         py_rawerr = PyTuple_GET_ITEM(cb_args, 2);
         memcpy(result, PyBytes_AS_STRING(py_rawerr),
@@ -4805,14 +4836,14 @@
 {
     CTypeDescrObject *ct, *ctresult;
     CDataObject *cd;
-    PyObject *ob, *error_ob = Py_None;
+    PyObject *ob, *error_ob = Py_None, *onerror_ob = Py_None;
     PyObject *py_rawerr, *infotuple = NULL;
     cif_description_t *cif_descr;
     ffi_closure *closure;
     Py_ssize_t size;
 
-    if (!PyArg_ParseTuple(args, "O!O|O:callback", &CTypeDescr_Type, &ct, &ob,
-                          &error_ob))
+    if (!PyArg_ParseTuple(args, "O!O|OO:callback", &CTypeDescr_Type, &ct, &ob,
+                          &error_ob, &onerror_ob))
         return NULL;
 
     if (!(ct->ct_flags & CT_FUNCTIONPTR)) {
@@ -4826,6 +4857,12 @@
                      Py_TYPE(ob)->tp_name);
         return NULL;
     }
+    if (onerror_ob != Py_None && !PyCallable_Check(onerror_ob)) {
+        PyErr_Format(PyExc_TypeError,
+                     "expected a callable object for 'onerror', not %.200s",
+                     Py_TYPE(onerror_ob)->tp_name);
+        return NULL;
+    }
 
     ctresult = (CTypeDescrObject *)PyTuple_GET_ITEM(ct->ct_stuff, 1);
     size = ctresult->ct_size;
@@ -4842,7 +4879,7 @@
             return NULL;
         }
     }
-    infotuple = Py_BuildValue("OOO", ct, ob, py_rawerr);
+    infotuple = Py_BuildValue("OOOO", ct, ob, py_rawerr, onerror_ob);
     Py_DECREF(py_rawerr);
     if (infotuple == NULL)
         return NULL;
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -714,11 +714,13 @@
 static PyObject *ffi_callback(FFIObject *self, PyObject *args, PyObject *kwds)
 {
     PyObject *c_decl, *python_callable = Py_None, *error = Py_None;
-    PyObject *res;
-    static char *keywords[] = {"cdecl", "python_callable", "error", NULL};
+    PyObject *res, *onerror = Py_None;
+    static char *keywords[] = {"cdecl", "python_callable", "error",
+                               "onerror", NULL};
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO", keywords,
-                                     &c_decl, &python_callable, &error))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OOO", keywords,
+                                     &c_decl, &python_callable, &error,
+                                     &onerror))
         return NULL;
 
     c_decl = (PyObject *)_ffi_type(self, c_decl, ACCEPT_STRING | ACCEPT_CTYPE |
@@ -726,7 +728,7 @@
     if (c_decl == NULL)
         return NULL;
 
-    args = Py_BuildValue("(OOO)", c_decl, python_callable, error);
+    args = Py_BuildValue("(OOOO)", c_decl, python_callable, error, onerror);
     if (args == NULL)
         return NULL;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1181,6 +1181,12 @@
     BShort = new_primitive_type("short")
     BFunc = new_function_type((BShort,), BShort, False)
     f = callback(BFunc, Zcb1, -42)
+    #
+    seen = []
+    def oops(*args):
+        seen.append(args)
+    ff = callback(BFunc, Zcb1, -42, oops)
+    #
     orig_stderr = sys.stderr
     orig_getline = linecache.getline
     try:
@@ -1206,6 +1212,30 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        assert len(seen) == 0
+        assert ff(bigvalue) == -42
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        seen = "not a list"    # this makes the oops() function crash
+        assert ff(bigvalue) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+Traceback (most recent call last):
+  File "$", line $, in oops
+    seen.append(args)
+AttributeError: 'str' object has no attribute 'append'
+""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -286,7 +286,7 @@
         """
         return self._backend.from_buffer(self.BCharA, python_buffer)
 
-    def callback(self, cdecl, python_callable=None, error=None):
+    def callback(self, cdecl, python_callable=None, error=None, onerror=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
         The callback invokes the specified 'python_callable' (which may
@@ -298,7 +298,8 @@
             if not callable(python_callable):
                 raise TypeError("the 'python_callable' argument "
                                 "is not callable")
-            return self._backend.callback(cdecl, python_callable, error)
+            return self._backend.callback(cdecl, python_callable,
+                                          error, onerror)
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl, consider_function_as_funcptr=True)
         if python_callable is None:
diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
--- a/cffi/backend_ctypes.py
+++ b/cffi/backend_ctypes.py
@@ -989,7 +989,8 @@
     def cast(self, BType, source):
         return BType._cast_from(source)
 
-    def callback(self, BType, source, error):
+    def callback(self, BType, source, error, onerror):
+        assert onerror is None   # XXX not implemented
         return BType(source, error)
 
     typeof = type
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -38,6 +38,22 @@
         assert ffi.from_handle(ffi.cast("char *", p)) is o
         py.test.raises(RuntimeError, ffi.from_handle, ffi.NULL)
 
+    def test_callback_onerror(self):
+        ffi = FFI(backend=self.Backend())
+        seen = []
+        def oops(*args):
+            seen.append(args)
+        def cb(n):
+            raise LookupError
+        a = ffi.callback("int(*)(int)", cb, error=42, onerror=oops)
+        res = a(2)
+        assert res == 42
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is LookupError
+        assert isinstance(val, LookupError)
+        assert tb.tb_frame.f_code.co_name == 'cb'
+
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -104,6 +104,35 @@
     assert deco(lambda x: x + "")(10) == -66
     assert deco(lambda x: x + 42)(10) == 52
 
+def test_ffi_callback_onerror():
+    ffi = _cffi1_backend.FFI()
+    seen = []
+    def oops(*args):
+        seen.append(args)
+
+    @ffi.callback("int(int)", onerror=oops)
+    def fn1(x):
+        return x + ""
+    assert fn1(10) == 0
+
+    @ffi.callback("int(int)", onerror=oops, error=-66)
+    def fn2(x):
+        return x + ""
+    assert fn2(10) == -66
+
+    assert len(seen) == 2
+    exc, val, tb = seen[0]
+    assert exc is TypeError
+    assert isinstance(val, TypeError)
+    assert tb.tb_frame.f_code.co_name == "fn1"
+    exc, val, tb = seen[1]
+    assert exc is TypeError
+    assert isinstance(val, TypeError)
+    assert tb.tb_frame.f_code.co_name == "fn2"
+    #
+    py.test.raises(TypeError, ffi.callback, "int(int)",
+                   lambda x: x, onerror=42)   # <- not callable
+
 def test_ffi_getctype():
     ffi = _cffi1_backend.FFI()
     assert ffi.getctype("int") == "int"

From noreply at buildbot.pypy.org  Sat Jul  4 22:43:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 22:43:18 +0200 (CEST)
Subject: [pypy-commit] cffi default: 'onerror' can also return a custom
	value to return
Message-ID: <20150704204318.118D21C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2205:bdbb14a1b774
Date: 2015-07-04 22:44 +0200
http://bitbucket.org/cffi/cffi/changeset/bdbb14a1b774/

Log:	'onerror' can also return a custom value to return

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -4802,13 +4802,18 @@
                                             val1 ? val1 : Py_None,
                                             tb1  ? tb1  : Py_None,
                                             NULL);
-        Py_XDECREF(res1);
-
-        if (!PyErr_Occurred()) {
-            Py_XDECREF(exc1);
-            Py_XDECREF(val1);
-            Py_XDECREF(tb1);
-            goto no_more_exception;
+        if (res1 != NULL) {
+            if (res1 != Py_None)
+                convert_from_object_fficallback(result, SIGNATURE(1), res1);
+            Py_DECREF(res1);
+            if (!PyErr_Occurred()) {
+                Py_XDECREF(exc1);
+                Py_XDECREF(val1);
+                Py_XDECREF(tb1);
+                if (res1 != Py_None)
+                    goto done;
+                goto no_more_exception;
+            }
         }
         /* double exception! print a double-traceback... */
         PyErr_Fetch(&exc2, &val2, &tb2);
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1183,8 +1183,10 @@
     f = callback(BFunc, Zcb1, -42)
     #
     seen = []
+    oops_result = None
     def oops(*args):
         seen.append(args)
+        return oops_result
     ff = callback(BFunc, Zcb1, -42, oops)
     #
     orig_stderr = sys.stderr
@@ -1222,6 +1224,35 @@
         assert exc is OverflowError
         assert str(val) == "integer 60000 does not fit 'short'"
         #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = 81
+        assert ff(bigvalue) == 81
+        oops_result = None
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = "xy"     # not None and not an int!
+        assert ff(bigvalue) == -42
+        oops_result = None
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+TypeError: an integer is required
+""")
+        #
+        sys.stderr = cStringIO.StringIO()
         seen = "not a list"    # this makes the oops() function crash
         assert ff(bigvalue) == -42
         assert matches(sys.stderr.getvalue(), """\

From noreply at buildbot.pypy.org  Sat Jul  4 23:02:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 23:02:35 +0200 (CEST)
Subject: [pypy-commit] cffi default: Show a semi-hackish way to get at the
 value of the arguments when the
Message-ID: <20150704210235.737651C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2206:5d494c19cb68
Date: 2015-07-04 23:03 +0200
http://bitbucket.org/cffi/cffi/changeset/5d494c19cb68/

Log:	Show a semi-hackish way to get at the value of the arguments when
	the crash occurred.

diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -43,16 +43,19 @@
         seen = []
         def oops(*args):
             seen.append(args)
+        def otherfunc():
+            raise LookupError
         def cb(n):
-            raise LookupError
+            otherfunc()
         a = ffi.callback("int(*)(int)", cb, error=42, onerror=oops)
-        res = a(2)
+        res = a(234)
         assert res == 42
         assert len(seen) == 1
         exc, val, tb = seen[0]
         assert exc is LookupError
         assert isinstance(val, LookupError)
         assert tb.tb_frame.f_code.co_name == 'cb'
+        assert tb.tb_frame.f_locals['n'] == 234
 
 
 class TestBitfield:

From noreply at buildbot.pypy.org  Sat Jul  4 23:30:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 23:30:28 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-callback-onerror: hg merge default
Message-ID: <20150704213028.88B141C11C6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-callback-onerror
Changeset: r78432:088ea8f70900
Date: 2015-07-04 23:19 +0200
http://bitbucket.org/pypy/pypy/changeset/088ea8f70900/

Log:	hg merge default

diff too long, truncating to 2000 out of 4366 lines

diff --git a/lib-python/2.7/test/test_urllib2.py b/lib-python/2.7/test/test_urllib2.py
--- a/lib-python/2.7/test/test_urllib2.py
+++ b/lib-python/2.7/test/test_urllib2.py
@@ -291,6 +291,7 @@
         self.req_headers = []
         self.data = None
         self.raise_on_endheaders = False
+        self.sock = None
         self._tunnel_headers = {}
 
     def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
diff --git a/lib-python/2.7/urllib2.py b/lib-python/2.7/urllib2.py
--- a/lib-python/2.7/urllib2.py
+++ b/lib-python/2.7/urllib2.py
@@ -1200,6 +1200,12 @@
                 r = h.getresponse(buffering=True)
             except TypeError: # buffering kw not supported
                 r = h.getresponse()
+            # If the server does not send us a 'Connection: close' header,
+            # HTTPConnection assumes the socket should be left open. Manually
+            # mark the socket to be closed when this response object goes away.
+            if h.sock:
+                h.sock.close()
+                h.sock = None
 
         # Pick apart the HTTPResponse object to get the addinfourl
         # object initialized properly.
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -108,6 +108,8 @@
         return value.internalRep.doubleValue
     if value.typePtr == typeCache.IntType:
         return value.internalRep.longValue
+    if value.typePtr == typeCache.WideIntType:
+        return FromWideIntObj(app, value)
     if value.typePtr == typeCache.BigNumType and tklib.HAVE_LIBTOMMATH:
         return FromBignumObj(app, value)
     if value.typePtr == typeCache.ListType:
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -179,6 +179,7 @@
 typedef int... Tcl_WideInt;
 
 int Tcl_GetWideIntFromObj(Tcl_Interp *interp, Tcl_Obj *obj, Tcl_WideInt *value);
+Tcl_Obj *Tcl_NewWideIntObj(Tcl_WideInt value);
 """)
 
 if HAVE_LIBTOMMATH:
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.1.2
+Version: 1.2.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.2"
-__version_info__ = (1, 1, 2)
+__version__ = "1.2.0"
+__version_info__ = (1, 2, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -328,6 +328,13 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
+        try:
+            gcp = self._backend.gcp
+        except AttributeError:
+            pass
+        else:
+            return gcp(cdata, destructor)
+        #
         with self._lock:
             try:
                 gc_weakrefs = self.gc_weakrefs
@@ -429,6 +436,8 @@
             raise TypeError("ffi.include() expects an argument that is also of"
                             " type cffi.FFI, not %r" % (
                                 type(ffi_to_include).__name__,))
+        if ffi_to_include is self:
+            raise ValueError("self.include(self)")
         with ffi_to_include._lock:
             with self._lock:
                 self._parser.include(ffi_to_include._parser)
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -633,6 +633,8 @@
 
     def include(self, other):
         for name, tp in other._declarations.items():
+            if name.startswith('anonymous $enum_$'):
+                continue   # fix for test_anonymous_enum_include
             kind = name.split(' ', 1)[0]
             if kind in ('struct', 'union', 'enum', 'anonymous'):
                 self._declare(name, tp, included=True)
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -35,9 +35,6 @@
     def is_integer_type(self):
         return False
 
-    def sizeof_enabled(self):
-        return False
-
     def get_cached_btype(self, ffi, finishlist, can_delay=False):
         try:
             BType = ffi._cached_btypes[self]
@@ -80,8 +77,7 @@
 
 
 class BasePrimitiveType(BaseType):
-    def sizeof_enabled(self):
-        return True
+    pass
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -205,9 +201,6 @@
 class FunctionPtrType(BaseFunctionType):
     _base_pattern = '(*&)(%s)'
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         result = self.result.get_cached_btype(ffi, finishlist)
         args = []
@@ -233,9 +226,6 @@
             extra = self._base_pattern
         self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra)
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True)
         return global_cache(self, ffi, 'new_pointer_type', BItem)
@@ -276,9 +266,6 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def sizeof_enabled(self):
-        return self.item.sizeof_enabled() and self.length is not None
-
     def resolve_length(self, newlength):
         return ArrayType(self.item, newlength)
 
@@ -433,9 +420,6 @@
             from . import ffiplatform
             raise ffiplatform.VerificationMissing(self._get_c_name())
 
-    def sizeof_enabled(self):
-        return self.fldtypes is not None
-
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
@@ -464,9 +448,6 @@
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
-    def sizeof_enabled(self):
-        return True     # not strictly true, but external enums are obscure
-
     def force_the_name(self, forcename):
         StructOrUnionOrEnum.force_the_name(self, forcename)
         if self.forcename is None:
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -26,6 +26,7 @@
 #define _CFFI_OP_GLOBAL_VAR     33
 #define _CFFI_OP_DLOPEN_FUNC    35
 #define _CFFI_OP_DLOPEN_CONST   37
+#define _CFFI_OP_GLOBAL_VAR_F   39
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -981,10 +981,6 @@
         if not self.target_is_python and tp.is_integer_type():
             type_op = CffiOp(OP_CONSTANT_INT, -1)
         else:
-            if not tp.sizeof_enabled():
-                raise ffiplatform.VerificationError(
-                    "constant '%s' is of type '%s', whose size is not known"
-                    % (name, tp._get_c_name()))
             if self.target_is_python:
                 const_kind = OP_DLOPEN_CONST
             else:
@@ -1069,18 +1065,36 @@
         self._do_collect_type(self._global_type(tp, name))
 
     def _generate_cpy_variable_decl(self, tp, name):
-        pass
+        prnt = self._prnt
+        tp = self._global_type(tp, name)
+        if isinstance(tp, model.ArrayType) and tp.length is None:
+            tp = tp.item
+            ampersand = ''
+        else:
+            ampersand = '&'
+        # This code assumes that casts from "tp *" to "void *" is a
+        # no-op, i.e. a function that returns a "tp *" can be called
+        # as if it returned a "void *".  This should be generally true
+        # on any modern machine.  The only exception to that rule (on
+        # uncommon architectures, and as far as I can tell) might be
+        # if 'tp' were a function type, but that is not possible here.
+        # (If 'tp' is a function _pointer_ type, then casts from "fn_t
+        # **" to "void *" are again no-ops, as far as I can tell.)
+        prnt('static ' + tp.get_c_name('*_cffi_var_%s(void)' % (name,)))
+        prnt('{')
+        prnt('  return %s(%s);' % (ampersand, name))
+        prnt('}')
+        prnt()
 
     def _generate_cpy_variable_ctx(self, tp, name):
         tp = self._global_type(tp, name)
         type_index = self._typesdict[tp]
-        type_op = CffiOp(OP_GLOBAL_VAR, type_index)
-        if tp.sizeof_enabled():
-            size = "sizeof(%s)" % (name,)
+        if self.target_is_python:
+            op = OP_GLOBAL_VAR
         else:
-            size = 0
+            op = OP_GLOBAL_VAR_F
         self._lsts["global"].append(
-            GlobalExpr(name, '&%s' % name, type_op, size))
+            GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index)))
 
     # ----------
     # emitting the opcodes for individual types
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -135,7 +135,7 @@
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
 is not reliable in PyPy (nor Jython nor IronPython).  It also means that
-weak references may stay alive for a bit longer than expected.  This
+**weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
 suddenly they will really be dead, raising a ``ReferenceError`` on the
@@ -143,6 +143,24 @@
 ``ReferenceError`` at any place that uses them.  (Or, better yet, don't use
 ``weakref.proxy()`` at all; use ``weakref.ref()``.)
 
+Note a detail in the `documentation for weakref callbacks`__:
+
+    If callback is provided and not None, *and the returned weakref
+    object is still alive,* the callback will be called when the object
+    is about to be finalized.
+
+There are cases where, due to CPython's refcount semantics, a weakref
+dies immediately before or after the objects it points to (typically
+with some circular reference).  If it happens to die just after, then
+the callback will be invoked.  In a similar case in PyPy, both the
+object and the weakref will be considered as dead at the same time,
+and the callback will not be invoked.  (Issue `#2030`__)
+
+.. __: https://docs.python.org/2/library/weakref.html
+.. __: https://bitbucket.org/pypy/pypy/issue/2030/
+
+---------------------------------
+
 There are a few extra implications from the difference in the GC.  Most
 notably, if an object has a ``__del__``, the ``__del__`` is never called more
 than once in PyPy; but CPython will call the same ``__del__`` several times
@@ -321,9 +339,8 @@
 Miscellaneous
 -------------
 
-* Hash randomization (``-R``) is ignored in PyPy.  As documented in
-  http://bugs.python.org/issue14621, some of us believe it has no
-  purpose in CPython either.
+* Hash randomization (``-R``) `is ignored in PyPy`_.  In CPython
+  before 3.4 it has `little point`_.
 
 * You can't store non-string keys in type objects.  For example::
 
@@ -338,7 +355,8 @@
   for about 1400 calls.
 
 * since the implementation of dictionary is different, the exact number
-  which ``__hash__`` and ``__eq__`` are called is different. Since CPython
+  of times that ``__hash__`` and ``__eq__`` are called is different. 
+  Since CPython
   does not give any specific guarantees either, don't rely on it.
 
 * assignment to ``__class__`` is limited to the cases where it
@@ -395,3 +413,12 @@
   interactive mode. In a released version, this behaviour is suppressed, but
   setting the environment variable PYPY_IRC_TOPIC will bring it back. Note that
   downstream package providers have been known to totally disable this feature.
+
+* PyPy's readline module was rewritten from scratch: it is not GNU's
+  readline.  It should be mostly compatible, and it adds multiline
+  support (see ``multiline_input()``).  On the other hand,
+  ``parse_and_bind()`` calls are ignored (issue `#2072`_).
+
+.. _`is ignored in PyPy`: http://bugs.python.org/issue14621
+.. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
+.. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,15 +6,9 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
-**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
-on linux, linux64 and windows. We will make it the default on all platforms
-by the time of the next release.
-
-The first thing that you need is to compile PyPy yourself with the option
-``--shared``. We plan to make ``--shared`` the default in the future. Consult
-the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
-or ``pypy.dll`` file or something similar, depending on your platform. Consult
-your platform specification for details.
+**NOTE**: You need a PyPy compiled with the option ``--shared``, i.e.
+with a ``libpypy-c.so`` or ``pypy-c.dll`` file.  This is the default in
+recent versions of PyPy.
 
 The resulting shared library exports very few functions, however they are
 enough to accomplish everything you need, provided you follow a few principles.
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -70,6 +70,20 @@
 .. _`use virtualenv (as documented here)`: getting-started.html#installing-using-virtualenv
 
 
+Module xyz does not work in the sandboxed PyPy?
+-----------------------------------------------
+
+You cannot import *any* extension module in a `sandboxed PyPy`_,
+sorry.  Even the built-in modules available are very limited.
+Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
+it is only a proof of concept.  It seriously requires someone working
+on it.  Before this occurs, it can only be used it for "pure Python"
+examples: programs that import mostly nothing (or only pure Python
+modules, recursively).
+
+.. _`sandboxed PyPy`: sandbox.html
+
+
 .. _`See below.`:
 
 Do CPython Extension modules work with PyPy?
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -11,3 +11,14 @@
 .. branch: stdlib-2.7.10
 
 Update stdlib to version 2.7.10
+
+.. branch: issue2062
+
+.. branch: disable-unroll-for-short-loops
+The JIT no longer performs loop unrolling if the loop compiles to too much code.
+
+.. branch: run-create_cffi_imports
+
+Build cffi import libraries as part of translation by monkey-patching an 
+aditional task into translation
+
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -1,6 +1,6 @@
 import py
 
-import os, sys
+import os, sys, subprocess
 
 import pypy
 from pypy.interpreter import gateway
@@ -298,6 +298,44 @@
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
 
+        # HACKHACKHACK
+        # ugly hack to modify target goal from compile_c to build_cffi_imports
+        # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.translator.driver import taskdef
+        import types
+
+        class Options(object):
+            pass
+
+
+        def mkexename(name):
+            if sys.platform == 'win32':
+                name = name.new(ext='exe')
+            return name
+
+        @taskdef(['compile_c'], "Create cffi bindings for modules")
+        def task_build_cffi_imports(self):
+            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
+            ''' Use cffi to compile cffi interfaces to modules'''
+            exename = mkexename(driver.compute_exe_name())
+            basedir = exename
+            while not basedir.join('include').exists():
+                _basedir = basedir.dirpath()
+                if _basedir == basedir:
+                    raise ValueError('interpreter %s not inside pypy repo', 
+                                     str(exename))
+                basedir = _basedir
+            modules = self.config.objspace.usemodules.getpaths()
+            options = Options()
+            # XXX possibly adapt options using modules
+            failures = create_cffi_import_libraries(exename, options, basedir)
+            # if failures, they were already printed
+            print  >> sys.stderr, str(exename),'successfully built, but errors while building the above modules will be ignored'
+        driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
+        driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, ['compile_c']
+        driver.default_goal = 'build_cffi_imports'
+        # HACKHACKHACK end
+
         return self.get_entry_point(config)
 
     def jitpolicy(self, driver):
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -40,6 +40,11 @@
 PYPYLOG: If set to a non-empty value, enable logging.
 """
 
+try:
+    from __pypy__ import get_hidden_tb, hidden_applevel
+except ImportError:
+    get_hidden_tb = lambda: sys.exc_info()[2]
+    hidden_applevel = lambda f: f
 import sys
 
 DEBUG = False       # dump exceptions before calling the except hook
@@ -63,6 +68,7 @@
             exitcode = 1
     raise SystemExit(exitcode)
 
+ at hidden_applevel
 def run_toplevel(f, *fargs, **fkwds):
     """Calls f() and handles all OperationErrors.
     Intended use is to run the main program or one interactive statement.
@@ -87,13 +93,13 @@
 
     except SystemExit as e:
         handle_sys_exit(e)
-    except:
-        display_exception()
+    except BaseException as e:
+        display_exception(e)
         return False
     return True   # success
 
-def display_exception():
-    etype, evalue, etraceback = sys.exc_info()
+def display_exception(e):
+    etype, evalue, etraceback = type(e), e, get_hidden_tb()
     try:
         # extra debugging info in case the code below goes very wrong
         if DEBUG and hasattr(sys, 'stderr'):
@@ -119,11 +125,11 @@
         hook(etype, evalue, etraceback)
         return # done
 
-    except:
+    except BaseException as e:
         try:
             stderr = sys.stderr
             print >> stderr, 'Error calling sys.excepthook:'
-            originalexcepthook(*sys.exc_info())
+            originalexcepthook(type(e), e, e.__traceback__)
             print >> stderr
             print >> stderr, 'Original exception was:'
         except:
@@ -509,6 +515,7 @@
 
     return options
 
+ at hidden_applevel
 def run_command_line(interactive,
                      inspect,
                      run_command,
@@ -597,6 +604,7 @@
             # Put '' on sys.path
             sys.path.insert(0, '')
 
+            @hidden_applevel
             def run_it():
                 exec run_command in mainmodule.__dict__
             success = run_toplevel(run_it)
@@ -634,6 +642,7 @@
                         print >> sys.stderr, "Could not open PYTHONSTARTUP"
                         print >> sys.stderr, "IOError:", e
                     else:
+                        @hidden_applevel
                         def run_it():
                             co_python_startup = compile(startup,
                                                         python_startup,
@@ -650,6 +659,7 @@
                 inspect = True
             else:
                 # If not interactive, just read and execute stdin normally.
+                @hidden_applevel
                 def run_it():
                     co_stdin = compile(sys.stdin.read(), '<stdin>', 'exec',
                                        PyCF_ACCEPT_NULL_BYTES)
@@ -689,7 +699,7 @@
     except SystemExit as e:
         status = e.code
         if inspect_requested():
-            display_exception()
+            display_exception(e)
     else:
         status = not success
 
@@ -743,6 +753,7 @@
     # This is important for py3k
     sys.executable = executable
 
+ at hidden_applevel
 def entry_point(executable, argv):
     # note that before calling setup_bootstrap_path, we are limited because we
     # cannot import stdlib modules. In particular, we cannot use unicode
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,6 +1,7 @@
 import sys
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 
 TICK_COUNTER_STEP = 100
@@ -214,13 +215,21 @@
             self._trace(frame, 'exception', None, operationerr)
         #operationerr.print_detailed_traceback(self.space)
 
-    def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
+    @specialize.arg(1)
+    def sys_exc_info(self, for_hidden=False):
         """Implements sys.exc_info().
-        Return an OperationError instance or None."""
+        Return an OperationError instance or None.
+
+        Ignores exceptions within hidden frames unless for_hidden=True
+        is specified.
+
+        # NOTE: the result is not the wrapped sys.exc_info() !!!
+
+        """
         frame = self.gettopframe()
         while frame:
             if frame.last_exception is not None:
-                if (not frame.hide() or
+                if ((for_hidden or not frame.hide()) or
                         frame.last_exception is
                             get_cleared_operation_error(self.space)):
                     return frame.last_exception
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -15,7 +15,10 @@
         self.running = False
 
     def descr__repr__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = '<finished>'
+        else:
+            code_name = self.pycode.co_name
         addrstring = self.getaddrstring(space)
         return space.wrap("<generator object %s at 0x%s>" %
                           (code_name, addrstring))
@@ -45,6 +48,8 @@
         w_framestate, w_running = args_w
         if space.is_w(w_framestate, space.w_None):
             self.frame = None
+            self.space = space
+            self.pycode = None
         else:
             frame = instantiate(space.FrameClass)   # XXX fish
             frame.descr__setstate__(space, w_framestate)
@@ -62,9 +67,10 @@
 
     def send_ex(self, w_arg, operr=None):
         pycode = self.pycode
-        if jit.we_are_jitted() and should_not_inline(pycode):
-            generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
-                                                  operr=operr, pycode=pycode)
+        if pycode is not None:
+            if jit.we_are_jitted() and should_not_inline(pycode):
+                generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
+                                                    operr=operr, pycode=pycode)
         return self._send_ex(w_arg, operr)
 
     def _send_ex(self, w_arg, operr):
@@ -158,7 +164,10 @@
         return self.pycode
 
     def descr__name__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = '<finished>'
+        else:
+            code_name = self.pycode.co_name
         return space.wrap(code_name)
 
     # Results can be either an RPython list of W_Root, or it can be an
diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py
--- a/pypy/interpreter/pytraceback.py
+++ b/pypy/interpreter/pytraceback.py
@@ -60,7 +60,6 @@
 
 
 def check_traceback(space, w_tb, msg):
-    from pypy.interpreter.typedef import PyTraceback
     if w_tb is None or not space.isinstance_w(w_tb, space.gettypeobject(PyTraceback.typedef)):
         raise OperationError(space.w_TypeError, space.wrap(msg))
     return w_tb
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -491,6 +491,22 @@
         assert pack.mod is result
 
 
+    def test_pickle_generator_crash(self):
+        import pickle
+
+        def f():
+            yield 0
+
+        x = f()
+        x.next()
+        try:
+            x.next()
+        except StopIteration:
+            y = pickle.loads(pickle.dumps(x))
+        assert 'finished' in y.__name__
+        assert 'finished' in repr(y)
+        assert y.gi_code is None
+
 class AppTestGeneratorCloning:
 
     def setup_class(cls):
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -71,6 +71,8 @@
         'debug_print_once'          : 'interp_debug.debug_print_once',
         'debug_flush'               : 'interp_debug.debug_flush',
         'builtinify'                : 'interp_magic.builtinify',
+        'hidden_applevel'           : 'interp_magic.hidden_applevel',
+        'get_hidden_tb'             : 'interp_magic.get_hidden_tb',
         'lookup_special'            : 'interp_magic.lookup_special',
         'do_what_I_mean'            : 'interp_magic.do_what_I_mean',
         'validate_fd'               : 'interp_magic.validate_fd',
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -59,6 +59,20 @@
     bltn = BuiltinFunction(func)
     return space.wrap(bltn)
 
+def hidden_applevel(space, w_func):
+    """Decorator that hides a function's frame from app-level"""
+    from pypy.interpreter.function import Function
+    func = space.interp_w(Function, w_func)
+    func.getcode().hidden_applevel = True
+    return w_func
+
+def get_hidden_tb(space):
+    """Return the traceback of the current exception being handled by a
+    frame hidden from applevel.
+    """
+    operr = space.getexecutioncontext().sys_exc_info(for_hidden=True)
+    return space.w_None if operr is None else space.wrap(operr.get_traceback())
+
 @unwrap_spec(meth=str)
 def lookup_special(space, w_obj, meth):
     """Lookup up a special method on an object."""
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -27,6 +27,52 @@
         assert A.a is not A.__dict__['a']
         assert A.b is A.__dict__['b']
 
+    def test_hidden_applevel(self):
+        import __pypy__
+        import sys
+
+        @__pypy__.hidden_applevel
+        def sneak(): (lambda: 1/0)()
+        try:
+            sneak()
+        except ZeroDivisionError as e:
+            tb = sys.exc_info()[2]
+            assert tb.tb_frame == sys._getframe()
+            assert tb.tb_next.tb_frame.f_code.co_name == '<lambda>'
+        else:
+            assert False, 'Expected ZeroDivisionError'
+
+    def test_hidden_applevel_frames(self):
+        import __pypy__
+        import sys
+
+        @__pypy__.hidden_applevel
+        def test_hidden():
+            assert sys._getframe().f_code.co_name != 'test_hidden'
+            def e(): 1/0
+            try: e()
+            except ZeroDivisionError as e:
+                assert sys.exc_info() == (None, None, None)
+            else: assert False
+            return 2
+        assert test_hidden() == 2
+
+    def test_get_hidden_tb(self):
+        import __pypy__
+        import sys
+
+        @__pypy__.hidden_applevel
+        def test_hidden_with_tb():
+            def not_hidden(): 1/0
+            try: not_hidden()
+            except ZeroDivisionError as e:
+                assert sys.exc_info() == (None, None, None)
+                tb = __pypy__.get_hidden_tb()
+                assert tb.tb_frame.f_code.co_name == 'not_hidden'
+                return True
+            else: return False
+        assert test_hidden_with_tb()
+
     def test_lookup_special(self):
         from __pypy__ import lookup_special
         class X(object):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.2"
+VERSION = "1.2.0"
 
 
 class Module(MixedModule):
@@ -37,6 +37,7 @@
         'from_handle': 'handle.from_handle',
         '_get_types': 'func._get_types',
         'from_buffer': 'func.from_buffer',
+        'gcp': 'func.gcp',
 
         'string': 'func.string',
         'buffer': 'cbuffer.buffer',
diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -36,7 +36,10 @@
                         self.libname)
         try:
             cdata = dlsym(self.libhandle, name)
+            found = bool(cdata)
         except KeyError:
+            found = False
+        if not found:
             raise oefmt(self.ffi.w_FFIError,
                         "symbol '%s' not found in library '%s'",
                         name, self.libname)
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/pypy/module/_cffi_backend/cglob.py b/pypy/module/_cffi_backend/cglob.py
--- a/pypy/module/_cffi_backend/cglob.py
+++ b/pypy/module/_cffi_backend/cglob.py
@@ -2,23 +2,38 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend import newtype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+FNPTR = rffi.CCallback([], rffi.VOIDP)
 
 
 class W_GlobSupport(W_Root):
-    def __init__(self, space, w_ctype, ptr):
+    _immutable_fields_ = ['w_ctype', 'ptr', 'fetch_addr']
+
+    def __init__(self, space, w_ctype, ptr=lltype.nullptr(rffi.CCHARP.TO),
+                 fetch_addr=lltype.nullptr(rffi.VOIDP.TO)):
         self.space = space
         self.w_ctype = w_ctype
         self.ptr = ptr
+        self.fetch_addr = rffi.cast(FNPTR, fetch_addr)
+
+    def fetch_global_var_addr(self):
+        if self.ptr:
+            return self.ptr
+        result = self.fetch_addr()
+        return rffi.cast(rffi.CCHARP, result)
 
     def read_global_var(self):
-        return self.w_ctype.convert_to_object(self.ptr)
+        return self.w_ctype.convert_to_object(self.fetch_global_var_addr())
 
     def write_global_var(self, w_newvalue):
-        self.w_ctype.convert_from_object(self.ptr, w_newvalue)
+        self.w_ctype.convert_from_object(self.fetch_global_var_addr(),
+                                         w_newvalue)
 
     def address(self):
         w_ctypeptr = newtype.new_pointer_type(self.space, self.w_ctype)
-        return W_CData(self.space, self.ptr, w_ctypeptr)
+        return W_CData(self.space, self.fetch_global_var_addr(), w_ctypeptr)
 
 W_GlobSupport.typedef = TypeDef("FFIGlobSupport")
 W_GlobSupport.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -143,7 +143,7 @@
     @jit.unroll_safe
     def _call(self, funcaddr, args_w):
         space = self.space
-        cif_descr = self.cif_descr
+        cif_descr = self.cif_descr   # 'self' should have been promoted here
         size = cif_descr.exchange_size
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -134,8 +134,7 @@
 
     def convert_to_object(self, cdata):
         unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        s = rffi.wcharpsize2unicode(unichardata, 1)
-        return self.space.wrap(s)
+        return self.space.wrap(unichardata[0])
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -223,9 +223,13 @@
         if (isinstance(w_cdata, cdataobj.W_CDataNewOwning) or
             isinstance(w_cdata, cdataobj.W_CDataPtrToStructOrUnion)):
             if i != 0:
-                space = self.space
-                raise oefmt(space.w_IndexError,
+                raise oefmt(self.space.w_IndexError,
                             "cdata '%s' can only be indexed by 0", self.name)
+        else:
+            if not w_cdata.unsafe_escaping_ptr():
+                raise oefmt(self.space.w_RuntimeError,
+                            "cannot dereference null pointer from cdata '%s'",
+                            self.name)
         return self
 
     def _check_slice_index(self, w_cdata, start, stop):
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -542,13 +542,18 @@
 
 
 @jit.dont_look_inside
-def W_FFIObject___new__(space, w_subtype, __args__):
-    r = space.allocate_instance(W_FFIObject, w_subtype)
+def make_plain_ffi_object(space, w_ffitype=None):
+    if w_ffitype is None:
+        w_ffitype = space.gettypefor(W_FFIObject)
+    r = space.allocate_instance(W_FFIObject, w_ffitype)
     # get in 'src_ctx' a NULL which translation doesn't consider to be constant
     src_ctx = rffi.cast(parse_c_type.PCTX, 0)
     r.__init__(space, src_ctx)
     return space.wrap(r)
 
+def W_FFIObject___new__(space, w_subtype, __args__):
+    return make_plain_ffi_object(space, w_subtype)
+
 def make_CData(space):
     return space.gettypefor(W_CData)
 
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -105,3 +105,18 @@
                     "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
+
+# ____________________________________________________________
+
+class ConstantFFI:
+    ffi1 = None
+    def _cleanup_(self):
+        self.ffi1 = None
+constant_ffi = ConstantFFI()
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData)
+def gcp(space, w_cdata, w_destructor):
+    if constant_ffi.ffi1 is None:
+        from pypy.module._cffi_backend import ffi_obj
+        constant_ffi.ffi1 = ffi_obj.make_plain_ffi_object(space)
+    return constant_ffi.ffi1.descr_gc(w_cdata, w_destructor)
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -60,12 +60,12 @@
             self.ffi, self.ctx.c_types, getarg(g.c_type_op))
         assert isinstance(rawfunctype, realize_c_type.W_RawFuncType)
         #
-        w_ct, locs = rawfunctype.unwrap_as_nostruct_fnptr(self.ffi)
+        rawfunctype.prepare_nostruct_fnptr(self.ffi)
         #
         ptr = rffi.cast(rffi.CCHARP, g.c_address)
         assert ptr
-        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn, w_ct,
-                                 locs, rawfunctype, fnname, self.libname)
+        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn,
+                                 rawfunctype, fnname, self.libname)
 
     @jit.elidable_promote()
     def _get_attr_elidable(self, attr):
@@ -102,6 +102,8 @@
                 #
             elif op == cffi_opcode.OP_GLOBAL_VAR:
                 # A global variable of the exact type specified here
+                # (nowadays, only used by the ABI mode or backend
+                # compatibility; see OP_GLOBAL_F for the API mode
                 w_ct = realize_c_type.realize_c_type(
                     self.ffi, self.ctx.c_types, getarg(g.c_type_op))
                 g_size = rffi.cast(lltype.Signed, g.c_size_or_direct_fn)
@@ -113,7 +115,13 @@
                 ptr = rffi.cast(rffi.CCHARP, g.c_address)
                 if not ptr:   # for dlopen() style
                     ptr = self.cdlopen_fetch(attr)
-                w_result = cglob.W_GlobSupport(space, w_ct, ptr)
+                w_result = cglob.W_GlobSupport(space, w_ct, ptr=ptr)
+                #
+            elif op == cffi_opcode.OP_GLOBAL_VAR_F:
+                w_ct = realize_c_type.realize_c_type(
+                    self.ffi, self.ctx.c_types, getarg(g.c_type_op))
+                w_result = cglob.W_GlobSupport(space, w_ct,
+                                               fetch_addr=g.c_address)
                 #
             elif (op == cffi_opcode.OP_CONSTANT_INT or
                   op == cffi_opcode.OP_ENUM):
@@ -131,6 +139,9 @@
                     realize_c_type.FUNCPTR_FETCH_CHARP,
                     g.c_address)
                 if w_ct.size <= 0:
+                    raise oefmt(self.ffi.w_FFIError,
+                                "constant '%s' is of type '%s', "
+                                "whose size is not known", attr, w_ct.name)
                     raise oefmt(space.w_SystemError,
                                 "constant has no known size")
                 if not fetch_funcptr:   # for dlopen() style
@@ -172,7 +183,11 @@
             w_value = self._build_attr(attr)
             if w_value is None:
                 if is_getattr and attr == '__all__':
-                    return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
+                    return self.dir1(ignore_global_vars=True)
+                if is_getattr and attr == '__dict__':
+                    return self.full_dict_copy()
+                if is_getattr and attr == '__name__':
+                    return self.descr_repr()
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
@@ -202,16 +217,31 @@
     def descr_dir(self):
         return self.dir1()
 
-    def dir1(self, ignore_type=-1):
+    def dir1(self, ignore_global_vars=False):
         space = self.space
         total = rffi.getintfield(self.ctx, 'c_num_globals')
         g = self.ctx.c_globals
         names_w = []
         for i in range(total):
-            if getop(g[i].c_type_op) != ignore_type:
-                names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
+            if ignore_global_vars:
+                op = getop(g[i].c_type_op)
+                if (op == cffi_opcode.OP_GLOBAL_VAR or
+                    op == cffi_opcode.OP_GLOBAL_VAR_F):
+                    continue
+            names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
+    def full_dict_copy(self):
+        space = self.space
+        total = rffi.getintfield(self.ctx, 'c_num_globals')
+        g = self.ctx.c_globals
+        w_result = space.newdict()
+        for i in range(total):
+            w_attr = space.wrap(rffi.charp2str(g[i].c_name))
+            w_value = self._get_attr(w_attr)
+            space.setitem(w_result, w_attr, w_value)
+        return w_result
+
     def address_of_func_or_global_var(self, varname):
         # rebuild a string object from 'varname', to do typechecks and
         # to force a unicode back to a plain string
@@ -224,7 +254,8 @@
         if isinstance(w_value, W_FunctionWrapper):
             # '&func' returns a regular cdata pointer-to-function
             if w_value.directfnptr:
-                return W_CData(space, w_value.directfnptr, w_value.ctype)
+                ctype = w_value.typeof(self.ffi)
+                return W_CData(space, w_value.directfnptr, ctype)
             else:
                 return w_value    # backward compatibility
         #
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib import jit
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -135,8 +136,12 @@
 
 class W_RawFuncType(W_Root):
     """Temporary: represents a C function type (not a function pointer)"""
+
+    _immutable_fields_ = ['nostruct_ctype', 'nostruct_locs', 'nostruct_nargs']
     _ctfuncptr = None
-    _nostruct_ctfuncptr = (None, None)
+    nostruct_ctype = None
+    nostruct_locs = None
+    nostruct_nargs = 0
 
     def __init__(self, opcodes, base_index):
         self.opcodes = opcodes
@@ -168,14 +173,16 @@
         assert self._ctfuncptr is not None
         return self._ctfuncptr
 
-    def unwrap_as_nostruct_fnptr(self, ffi):
-        # tweaked version: instead of returning the ctfuncptr corresponding
-        # exactly to the OP_FUNCTION ... OP_FUNCTION_END opcodes, return
-        # another one in which the struct args are replaced with ptr-to-
-        # struct, and a struct return value is replaced with a hidden first
-        # arg of type ptr-to-struct.  This is how recompiler.py produces
+    @jit.dont_look_inside
+    def prepare_nostruct_fnptr(self, ffi):
+        # tweaked version: instead of returning the ctfuncptr
+        # corresponding exactly to the OP_FUNCTION ... OP_FUNCTION_END
+        # opcodes, this builds in self.nostruct_ctype another one in
+        # which the struct args are replaced with ptr-to- struct, and
+        # a struct return value is replaced with a hidden first arg of
+        # type ptr-to-struct.  This is how recompiler.py produces
         # trampoline functions for PyPy.
-        if self._nostruct_ctfuncptr[0] is None:
+        if self.nostruct_ctype is None:
             fargs, fret, ellipsis = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
             # containing 'A' where a struct argument was detected, and 'R'
@@ -198,8 +205,10 @@
                 locs = None
             else:
                 locs = ''.join(locs)
-            self._nostruct_ctfuncptr = (ctfuncptr, locs)
-        return self._nostruct_ctfuncptr
+            self.nostruct_ctype = ctfuncptr
+            self.nostruct_locs = locs
+            self.nostruct_nargs = len(ctfuncptr.fargs) - (locs is not None and
+                                                          locs[0] == 'R')
 
     def unexpected_fn_type(self, ffi):
         fargs, fret, ellipsis = self._unpack(ffi)
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -362,7 +362,7 @@
 
             case TOK_INTEGER:
                 errno = 0;
-#ifndef MS_WIN32
+#ifndef _MSC_VER
                 if (sizeof(length) > sizeof(unsigned long))
                     length = strtoull(tok->p, &endptr, 0);
                 else
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2099,8 +2099,7 @@
     p = cast(BVoidP, 123456)
     py.test.raises(TypeError, "p[0]")
     p = cast(BVoidP, 0)
-    if 'PY_DOT_PY' in globals(): py.test.skip("NULL crashes early on py.py")
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises((TypeError, RuntimeError), "p[0]")
 
 def test_iter():
     BInt = new_primitive_type("int")
@@ -3333,6 +3332,15 @@
     check(4 | 8,  "CHB", "GTB")
     check(4 | 16, "CHB", "ROB")
 
+def test_dereference_null_ptr():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    p = cast(BIntPtr, 0)
+    py.test.raises(RuntimeError, "p[0]")
+    py.test.raises(RuntimeError, "p[0] = 42")
+    py.test.raises(RuntimeError, "p[42]")
+    py.test.raises(RuntimeError, "p[42] = -1")
+
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.2"
+    assert __version__ == "1.2.0"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -4,6 +4,8 @@
     spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
 
     def teardown_method(self, meth):
+        from pypy.module._cffi_backend.func import constant_ffi
+        constant_ffi._cleanup_()
         _clean_cache(self.space)
 
     def test_ffi_new(self):
@@ -234,9 +236,10 @@
             assert p1[0] == 123
             seen.append(1)
         ffi.gc(p, destructor=destructor)    # instantly forgotten
+        _cffi1_backend.gcp(p, destructor=destructor)
         for i in range(5):
             if seen:
                 break
             import gc
             gc.collect()
-        assert seen == [1]
+        assert seen == [1, 1]
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -16,8 +16,8 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 0, 4):
-        py.test.skip("system cffi module needs to be at least 1.0.4")
+    if cffi.__version_info__ < (1, 2, 0):
+        py.test.skip("system cffi module needs to be at least 1.2.0")
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -276,6 +276,15 @@
         """)
         lib.aa = 5
         assert dir(lib) == ['aa', 'ff', 'my_constant']
+        #
+        aaobj = lib.__dict__['aa']
+        assert not isinstance(aaobj, int)    # some internal object instead
+        assert lib.__dict__ == {
+            'ff': lib.ff,
+            'aa': aaobj,
+            'my_constant': -45}
+        lib.__dict__['ff'] = "??"
+        assert lib.ff(10) == 15
 
     def test_verify_opaque_struct(self):
         ffi, lib = self.prepare(
@@ -491,28 +500,33 @@
             "int foo(int x) { return x + 32; }")
         assert lib.foo(10) == 42
 
-    def test_bad_size_of_global_1(self):
-        ffi, lib = self.prepare(
-            "short glob;",
-            "test_bad_size_of_global_1",
-            "long glob;")
-        raises(ffi.error, getattr, lib, "glob")
-
-    def test_bad_size_of_global_2(self):
-        ffi, lib = self.prepare(
-            "int glob[10];",
-            "test_bad_size_of_global_2",
-            "int glob[9];")
-        e = raises(ffi.error, getattr, lib, "glob")
-        assert str(e.value) == ("global variable 'glob' should be 40 bytes "
-                                "according to the cdef, but is actually 36")
-
-    def test_unspecified_size_of_global(self):
+    def test_unspecified_size_of_global_1(self):
         ffi, lib = self.prepare(
             "int glob[];",
-            "test_unspecified_size_of_global",
+            "test_unspecified_size_of_global_1",
             "int glob[10];")
-        lib.glob    # does not crash
+        assert ffi.typeof(lib.glob) == ffi.typeof("int *")
+
+    def test_unspecified_size_of_global_2(self):
+        ffi, lib = self.prepare(
+            "int glob[][5];",
+            "test_unspecified_size_of_global_2",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+    def test_unspecified_size_of_global_3(self):
+        ffi, lib = self.prepare(
+            "int glob[][...];",
+            "test_unspecified_size_of_global_3",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+    def test_unspecified_size_of_global_4(self):
+        ffi, lib = self.prepare(
+            "int glob[...][...];",
+            "test_unspecified_size_of_global_4",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
     def test_include_1(self):
         ffi1, lib1 = self.prepare(
@@ -819,6 +833,22 @@
         assert isinstance(addr, ffi.CData)
         assert ffi.typeof(addr) == ffi.typeof("long(*)(long)")
 
+    def test_address_of_function_with_struct(self):
+        ffi, lib = self.prepare(
+            "struct foo_s { int x; }; long myfunc(struct foo_s);",
+            "test_addressof_function_with_struct", """
+                struct foo_s { int x; };
+                char myfunc(struct foo_s input) { return (char)(input.x + 42); }
+            """)
+        s = ffi.new("struct foo_s *", [5])[0]
+        assert lib.myfunc(s) == 47
+        assert not isinstance(lib.myfunc, ffi.CData)
+        assert ffi.typeof(lib.myfunc) == ffi.typeof("long(*)(struct foo_s)")
+        addr = ffi.addressof(lib, 'myfunc')
+        assert addr(s) == 47
+        assert isinstance(addr, ffi.CData)
+        assert ffi.typeof(addr) == ffi.typeof("long(*)(struct foo_s)")
+
     def test_issue198(self):
         ffi, lib = self.prepare("""
             typedef struct{...;} opaque_t;
@@ -844,11 +874,22 @@
             """)
         assert lib.almost_forty_two == 42.25
 
+    def test_constant_of_unknown_size(self):
+        ffi, lib = self.prepare(
+            "typedef ... opaque_t;"
+            "const opaque_t CONSTANT;",
+            'test_constant_of_unknown_size',
+            "typedef int opaque_t;"
+            "const int CONSTANT = 42;")
+        e = raises(ffi.error, getattr, lib, 'CONSTANT')
+        assert str(e.value) == ("constant 'CONSTANT' is of "
+                                "type 'opaque_t', whose size is not known")
+
     def test_variable_of_unknown_size(self):
         ffi, lib = self.prepare("""
             typedef ... opaque_t;
             opaque_t globvar;
-        """, 'test_constant_of_unknown_size', """
+        """, 'test_variable_of_unknown_size', """
             typedef char opaque_t[6];
             opaque_t globvar = "hello";
         """)
@@ -984,5 +1025,35 @@
         assert sys.modules['_CFFI_test_import_from_lib.lib'] is lib
         from _CFFI_test_import_from_lib.lib import MYFOO
         assert MYFOO == 42
-        assert not hasattr(lib, '__dict__')
+        assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
+        assert lib.__name__ == repr(lib)
+
+    def test_macro_var_callback(self):
+        ffi, lib = self.prepare(
+            "int my_value; int *(*get_my_value)(void);",
+            'test_macro_var_callback',
+            "int *(*get_my_value)(void);\n"
+            "#define my_value (*get_my_value())")
+        #
+        values = ffi.new("int[50]")
+        def it():
+            for i in range(50):
+                yield i
+        it = it()
+        #
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            return values + it.next()
+        lib.get_my_value = get_my_value
+        #
+        values[0] = 41
+        assert lib.my_value == 41            # [0]
+        p = ffi.addressof(lib, 'my_value')   # [1]
+        assert p == values + 1
+        assert p[-1] == 41
+        assert p[+1] == 0
+        lib.my_value = 42                    # [2]
+        assert values[2] == 42
+        assert p[-1] == 41
+        assert p[+1] == 42
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -19,12 +19,20 @@
     wrapper is callable, and the arguments it expects and returns
     are directly the struct/union.  Calling ffi.typeof(wrapper)
     also returns the original struct/union signature.
+
+    This class cannot be used for variadic functions.
     """
     _immutable_ = True
     common_doc_str = 'direct call to the C function of the same name'
 
-    def __init__(self, space, fnptr, directfnptr, ctype,
-                 locs, rawfunctype, fnname, modulename):
+    def __init__(self, space, fnptr, directfnptr,
+                 rawfunctype, fnname, modulename):
+        # everything related to the type of the function is accessed
+        # as immutable attributes of the 'rawfunctype' object, which
+        # is a W_RawFuncType.  This gives us an obvious thing to
+        # promote in order to do the call.
+        ctype = rawfunctype.nostruct_ctype
+        locs = rawfunctype.nostruct_locs
         assert isinstance(ctype, W_CTypeFunc)
         assert ctype.cif_descr is not None     # not for '...' functions
         assert locs is None or len(ctype.fargs) == len(locs)
@@ -32,83 +40,86 @@
         self.space = space
         self.fnptr = fnptr
         self.directfnptr = directfnptr
-        self.ctype = ctype
-        self.locs = locs
         self.rawfunctype = rawfunctype
         self.fnname = fnname
         self.modulename = modulename
-        self.nargs_expected = len(ctype.fargs) - (locs is not None and
-                                                  locs[0] == 'R')
 
     def typeof(self, ffi):
         return self.rawfunctype.unwrap_as_fnptr(ffi)
 
-    @jit.unroll_safe
-    def _prepare(self, args_w, start_index):
-        # replaces struct/union arguments with ptr-to-struct/union arguments
+    def descr_call(self, args_w):
         space = self.space
-        locs = self.locs
-        fargs = self.ctype.fargs
-        for i in range(start_index, len(locs)):
-            if locs[i] != 'A':
-                continue
-            w_arg = args_w[i]
-            farg = fargs[i]      # <ptr to struct/union>
-            assert isinstance(farg, W_CTypePtrOrArray)
-            if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
-                # fast way: we are given a W_CData "struct", so just make
-                # a new W_CData "ptr-to-struct" which points to the same
-                # raw memory.  We use unsafe_escaping_ptr(), so we have to
-                # make sure the original 'w_arg' stays alive; the easiest
-                # is to build an instance of W_CDataPtrToStructOrUnion.
-                w_arg = W_CDataPtrToStructOrUnion(
-                    space, w_arg.unsafe_escaping_ptr(), farg, w_arg)
-            else:
-                # slow way: build a new "ptr to struct" W_CData by calling
-                # the equivalent of ffi.new()
-                if space.is_w(w_arg, space.w_None):
-                    continue
-                w_arg = farg.newp(w_arg)
-            args_w[i] = w_arg
-
-    def descr_call(self, args_w):
-        if len(args_w) != self.nargs_expected:
-            space = self.space
-            if self.nargs_expected == 0:
+        rawfunctype = jit.promote(self.rawfunctype)
+        ctype = rawfunctype.nostruct_ctype
+        locs = rawfunctype.nostruct_locs
+        nargs_expected = rawfunctype.nostruct_nargs
+        #
+        if len(args_w) != nargs_expected:
+            if nargs_expected == 0:
                 raise oefmt(space.w_TypeError,
                             "%s() takes no arguments (%d given)",
                             self.fnname, len(args_w))
-            elif self.nargs_expected == 1:
+            elif nargs_expected == 1:
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly one argument (%d given)",
                             self.fnname, len(args_w))
             else:
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly %d arguments (%d given)",
-                            self.fnname, self.nargs_expected, len(args_w))
+                            self.fnname, nargs_expected, len(args_w))
         #
-        if self.locs is not None:
+        if locs is not None:
             # This case is if there are structs as arguments or return values.
             # If the result we want to present to the user is "returns struct",
             # then internally allocate the struct and pass a pointer to it as
             # a first argument.
-            if self.locs[0] == 'R':
-                w_result_cdata = self.ctype.fargs[0].newp(self.space.w_None)
+            if locs[0] == 'R':
+                w_result_cdata = ctype.fargs[0].newp(space.w_None)
                 args_w = [w_result_cdata] + args_w
-                self._prepare(args_w, 1)
-                self.ctype._call(self.fnptr, args_w)    # returns w_None
+                prepare_args(space, rawfunctype, args_w, 1)
+                #
+                ctype._call(self.fnptr, args_w)    # returns w_None
+                #
                 assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
                 return w_result_cdata.structobj
             else:
                 args_w = args_w[:]
-                self._prepare(args_w, 0)
+                prepare_args(space, rawfunctype, args_w, 0)
         #
-        return self.ctype._call(self.fnptr, args_w)
+        return ctype._call(self.fnptr, args_w)
 
     def descr_repr(self, space):
         return space.wrap("<FFIFunctionWrapper for %s()>" % (self.fnname,))
 
 
+ at jit.unroll_safe
+def prepare_args(space, rawfunctype, args_w, start_index):
+    # replaces struct/union arguments with ptr-to-struct/union arguments
+    locs = rawfunctype.nostruct_locs
+    fargs = rawfunctype.nostruct_ctype.fargs
+    for i in range(start_index, len(locs)):
+        if locs[i] != 'A':
+            continue
+        w_arg = args_w[i]
+        farg = fargs[i]      # <ptr to struct/union>
+        assert isinstance(farg, W_CTypePtrOrArray)
+        if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
+            # fast way: we are given a W_CData "struct", so just make
+            # a new W_CData "ptr-to-struct" which points to the same
+            # raw memory.  We use unsafe_escaping_ptr(), so we have to
+            # make sure the original 'w_arg' stays alive; the easiest
+            # is to build an instance of W_CDataPtrToStructOrUnion.
+            w_arg = W_CDataPtrToStructOrUnion(
+                space, w_arg.unsafe_escaping_ptr(), farg, w_arg)
+        else:
+            # slow way: build a new "ptr to struct" W_CData by calling
+            # the equivalent of ffi.new()
+            if space.is_w(w_arg, space.w_None):
+                continue
+            w_arg = farg.newp(w_arg)
+        args_w[i] = w_arg
+
+
 W_FunctionWrapper.typedef = TypeDef(
         'FFIFunctionWrapper',
         __repr__ = interp2app(W_FunctionWrapper.descr_repr),
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -613,7 +613,7 @@
 # ____________________________________________________________
 
 def wrap_list_of_str(space, lst):
-    return space.newlist([space.wrap(s) for s in lst])
+    return space.newlist_bytes(lst)
 
 class FileState:
     def __init__(self, space):
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -600,6 +600,7 @@
 
     def read_w(self, space, w_size=None):
         self._check_attached(space)
+        self._check_closed(space)
         if not self.w_decoder:
             raise OperationError(space.w_IOError, space.wrap("not readable"))
 
@@ -641,6 +642,7 @@
 
     def readline_w(self, space, w_limit=None):
         self._check_attached(space)
+        self._check_closed(space)
         self._writeflush(space)
 
         limit = convert_size(space, w_limit)
@@ -736,7 +738,7 @@
 
     def write_w(self, space, w_text):
         self._check_attached(space)
-        # self._check_closed(space)
+        self._check_closed(space)
 
         if not self.w_encoder:
             raise OperationError(space.w_IOError, space.wrap("not writable"))
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -391,3 +391,57 @@
             f.seek(1, 0)
             f.read(buffer_size * 2)
             assert f.tell() == 1 + buffer_size * 2
+
+
+class AppTestIoAferClose:
+    spaceconfig = dict(usemodules=['_io'])
+
+    def setup_class(cls):
+        tmpfile = udir.join('tmpfile').ensure()
+        cls.w_tmpfile = cls.space.wrap(str(tmpfile))
+
+    def test_io_after_close(self):
+        import _io
+        for kwargs in [
+                {"mode": "w"},
+                {"mode": "wb"},
+                {"mode": "w", "buffering": 1},
+                {"mode": "w", "buffering": 2},
+                {"mode": "wb", "buffering": 0},
+                {"mode": "r"},
+                {"mode": "rb"},
+                {"mode": "r", "buffering": 1},
+                {"mode": "r", "buffering": 2},
+                {"mode": "rb", "buffering": 0},
+                {"mode": "w+"},
+                {"mode": "w+b"},
+                {"mode": "w+", "buffering": 1},
+                {"mode": "w+", "buffering": 2},
+                {"mode": "w+b", "buffering": 0},
+            ]:
+            print kwargs
+            if "b" not in kwargs["mode"]:
+                kwargs["encoding"] = "ascii"
+            f = _io.open(self.tmpfile, **kwargs)
+            f.close()
+            raises(ValueError, f.flush)
+            raises(ValueError, f.fileno)
+            raises(ValueError, f.isatty)
+            raises(ValueError, f.__iter__)
+            if hasattr(f, "peek"):
+                raises(ValueError, f.peek, 1)
+            raises(ValueError, f.read)
+            if hasattr(f, "read1"):
+                raises(ValueError, f.read1, 1024)
+            if hasattr(f, "readall"):
+                raises(ValueError, f.readall)
+            if hasattr(f, "readinto"):
+                raises(ValueError, f.readinto, bytearray(1024))
+            raises(ValueError, f.readline)
+            raises(ValueError, f.readlines)
+            raises(ValueError, f.seek, 0)
+            raises(ValueError, f.tell)
+            raises(ValueError, f.truncate)
+            raises(ValueError, f.write, b"" if "b" in kwargs['mode'] else u"")
+            raises(ValueError, f.writelines, [])
+            raises(ValueError, next, f)
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -27,8 +27,10 @@
     callback_ptr = global_counter.get(userdata.addarg)
     w_callable = callback_ptr.w_callable
     argtypes = callback_ptr.argtypes
+    must_leave = False
     space = callback_ptr.space
     try:
+        must_leave = space.threadlocals.try_enter_thread(space)
         args_w = [None] * len(argtypes)
         for i in range(len(argtypes)):
             argtype = argtypes[i]
@@ -50,6 +52,8 @@
             resshape = letter2tp(space, callback_ptr.result)
             for i in range(resshape.size):
                 ll_res[i] = '\x00'
+    if must_leave:
+        space.threadlocals.leave_thread(space)
 
 class W_CallbackPtr(W_DataInstance):
 
@@ -75,6 +79,14 @@
         if tracker.DO_TRACING:
             addr = rffi.cast(lltype.Signed, self.ll_callback.ll_closure)
             tracker.trace_allocation(addr, self)
+        #
+        # We must setup the GIL here, in case the callback is invoked in
+        # some other non-Pythonic thread.  This is the same as ctypes on
+        # CPython (but only when creating a callback; on CPython it occurs
+        # as soon as we import _ctypes)
+        if space.config.translation.thread:
+            from pypy.module.thread.os_thread import setup_threads
+            setup_threads(space)
 
     def free(self):
         if tracker.DO_TRACING:
diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py
--- a/pypy/module/_socket/__init__.py
+++ b/pypy/module/_socket/__init__.py
@@ -18,6 +18,10 @@
         from rpython.rlib.rsocket import rsocket_startup
         rsocket_startup()
 
+    def shutdown(self, space):
+        from pypy.module._socket.interp_socket import close_all_sockets
+        close_all_sockets(space)
+
     def buildloaders(cls):
         from rpython.rlib import rsocket
         for name in """
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -142,7 +142,7 @@
         sock = rsocket.fromfd(fd, family, type, proto)
     except SocketError, e:
         raise converted_error(space, e)
-    return space.wrap(W_Socket(sock))
+    return space.wrap(W_Socket(space, sock))
 
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
@@ -160,8 +160,8 @@
     except SocketError, e:
         raise converted_error(space, e)
     return space.newtuple([
-        space.wrap(W_Socket(sock1)),
-        space.wrap(W_Socket(sock2))
+        space.wrap(W_Socket(space, sock1)),
+        space.wrap(W_Socket(space, sock2))
     ])
 
 # The following 4 functions refuse all negative numbers, like CPython 2.6.
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -1,4 +1,5 @@
-from rpython.rlib import rsocket
+import sys
+from rpython.rlib import rsocket, rweaklist
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rsocket import (
     RSocket, AF_INET, SOCK_STREAM, SocketError, SocketErrorWithErrno,
@@ -153,8 +154,9 @@
 
 
 class W_Socket(W_Root):
-    def __init__(self, sock):
+    def __init__(self, space, sock):
         self.sock = sock
+        register_socket(space, sock)
 
     def get_type_w(self, space):
         return space.wrap(self.sock.type)
@@ -183,7 +185,7 @@
             fd, addr = self.sock.accept()
             sock = rsocket.make_socket(
                 fd, self.sock.family, self.sock.type, self.sock.proto)
-            return space.newtuple([space.wrap(W_Socket(sock)),
+            return space.newtuple([space.wrap(W_Socket(space, sock)),
                                    addr_as_object(addr, sock.fd, space)])
         except SocketError as e:
             raise converted_error(space, e)
@@ -248,7 +250,7 @@
     def dup_w(self, space):
         try:
             sock = self.sock.dup()
-            return W_Socket(sock)
+            return W_Socket(space, sock)
         except SocketError as e:
             raise converted_error(space, e)
 
@@ -592,10 +594,50 @@
         sock = RSocket(family, type, proto)
     except SocketError as e:
         raise converted_error(space, e)
-    W_Socket.__init__(self, sock)
+    W_Socket.__init__(self, space, sock)
     return space.wrap(self)
 descr_socket_new = interp2app(newsocket)
 
+
+# ____________________________________________________________
+# Automatic shutdown()/close()
+
+# On some systems, the C library does not guarantee that when the program
+# finishes, all data sent so far is really sent even if the socket is not
+# explicitly closed.  This behavior has been observed on Windows but not
+# on Linux, so far.
+NEED_EXPLICIT_CLOSE = (sys.platform == 'win32')
+
+class OpenRSockets(rweaklist.RWeakListMixin):
+    pass
+class OpenRSocketsState:
+    def __init__(self, space):
+        self.openrsockets = OpenRSockets()
+        self.openrsockets.initialize()
+
+def getopenrsockets(space):
+    if NEED_EXPLICIT_CLOSE and space.config.translation.rweakref:
+        return space.fromcache(OpenRSocketsState).openrsockets
+    else:
+        return None
+
+def register_socket(space, socket):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        openrsockets.add_handle(socket)
+
+def close_all_sockets(space):
+    openrsockets = getopenrsockets(space)
+    if openrsockets is not None:
+        for sock_wref in openrsockets.get_all_handles():
+            sock = sock_wref()
+            if sock is not None:
+                try:
+                    sock.close()
+                except SocketError:
+                    pass
+
+
 # ____________________________________________________________
 # Error handling
 
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -309,10 +309,16 @@
 
 
 class AppTestSocket:
+    spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct'])
+
     def setup_class(cls):
         cls.space = space
         cls.w_udir = space.wrap(str(udir))
 
+    def teardown_class(cls):
+        if not cls.runappdirect:
+            cls.space.sys.getmodule('_socket').shutdown(cls.space)
+
     def test_module(self):
         import _socket
         assert _socket.socket.__name__ == 'socket'
@@ -614,6 +620,12 @@
         finally:
             os.chdir(oldcwd)
 
+    def test_automatic_shutdown(self):
+        # doesn't really test anything, but at least should not explode
+        # in close_all_sockets()
+        import _socket
+        self.foo = _socket.socket()
+
 
 class AppTestPacket:
     def setup_class(cls):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -136,7 +136,7 @@
     def __init__(self, ctx, protos):
         self.protos = protos
         self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
-        NPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+        NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
         # can be used to create both types of sockets
@@ -151,7 +151,7 @@
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
-        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if npn and npn.protos:
             data_ptr[0] = npn.buf
             len_ptr[0] = rffi.cast(rffi.UINT, len(npn.protos))
@@ -163,7 +163,7 @@
 
     @staticmethod
     def selectNPN_cb(s, out_ptr, outlen_ptr, server, server_len, args):
-        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if npn and npn.protos:
             client = npn.buf
             client_len = len(npn.protos)
@@ -182,7 +182,7 @@
     def __init__(self, ctx, protos):
         self.protos = protos
         self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
-        ALPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+        ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
             if libssl_SSL_CTX_set_alpn_protos(
@@ -197,7 +197,7 @@
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
-        alpn = ALPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        alpn = ALPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
         if alpn and alpn.protos:
             server = alpn.buf
             server_len = len(alpn.protos)
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -26,7 +26,7 @@
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['vmprof.h', 'trampoline.h'],
-    separate_module_files = [SRC.join('trampoline.asmgcc.s')],
+    separate_module_files = [SRC.join('trampoline.vmprof.s')],
     libraries = ['dl'],
     
     post_include_bits=["""
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.vmprof.s
rename from pypy/module/_vmprof/src/trampoline.asmgcc.s
rename to pypy/module/_vmprof/src/trampoline.vmprof.s
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ b/pypy/module/_vmprof/src/trampoline.vmprof.s
@@ -1,7 +1,6 @@
 // NOTE: you need to use TABs, not spaces!
         
 	.text
-	.p2align 4,,-1
 	.globl	pypy_execute_frame_trampoline
 	.type	pypy_execute_frame_trampoline, @function
 pypy_execute_frame_trampoline:
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -305,7 +305,6 @@
 
 static int remove_sigprof_timer(void) {
     static struct itimerval timer;
-    last_period_usec = 0;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = 0;
     timer.it_value.tv_sec = 0;
@@ -317,11 +316,15 @@
 }
 
 static void atfork_disable_timer(void) {
-    remove_sigprof_timer();
+    if (last_period_usec) {
+        remove_sigprof_timer();
+    }
 }
 
 static void atfork_enable_timer(void) {
-    install_sigprof_timer(last_period_usec);
+    if (last_period_usec) {
+        install_sigprof_timer(last_period_usec);
+    }
 }
 
 static int install_pthread_atfork_hooks(void) {
@@ -412,6 +415,7 @@
     if (remove_sigprof_timer() == -1) {
 		return -1;
 	}
+    last_period_usec = 0;
     if (remove_sigprof_handler() == -1) {
 		return -1;
 	}
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -38,4 +38,4 @@
     'C') or Fortran-style (fortran is 'F') contiguous or either one
     (fortran is 'A').  Return 0 otherwise."""
     # PyPy only supports contiguous Py_buffers for now.
-    return space.wrap(1)
+    return 1
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -16,7 +16,7 @@
         }
         """
         module = self.import_module(name='foo', init=init)
-        assert module.py_version == sys.version[:5]
+        assert module.py_version == '%d.%d.%d' % sys.version_info[:3]
         assert module.py_major_version == sys.version_info.major
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -349,6 +349,11 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
+                else:
+                    fromlist_w = []
+                    # "from x import *" with x already imported and no x.__all__
+                    # always succeeds without doing more imports.  It will
+                    # just copy everything from x.__dict__ as it is now.
             for w_name in fromlist_w:
                 if try_getattr(space, w_mod, w_name) is None:
                     return None
@@ -389,6 +394,8 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
+                else:
+                    fromlist_w = []
             for w_name in fromlist_w:
                 if try_getattr(space, w_mod, w_name) is None:
                     load_part(space, w_path, prefix, space.str0_w(w_name),
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -66,6 +66,14 @@
              b          = "insubpackage = 1",
              )
     setuppkg("pkg.pkg2", a='', b='')
+    setuppkg("pkg.withall",
+             __init__  = "__all__ = ['foobar']",
+             foobar    = "found = 123")
+    setuppkg("pkg.withoutall",
+             __init__  = "",
+             foobar    = "found = 123")
+    setuppkg("pkg.bogusall",
+             __init__  = "__all__ = 42")
     setuppkg("pkg_r", inpkg = "import x.y")
     setuppkg("pkg_r.x")
     setuppkg("x", y='')
@@ -677,6 +685,32 @@
         import imp
         raises(ValueError, imp.load_module, "", "", "", [1, 2, 3, 4])
 
+    def test_import_star_finds_submodules_with___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withall import *" in d
+            assert d["foobar"].found == 123

From noreply at buildbot.pypy.org  Sat Jul  4 23:30:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 23:30:29 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-callback-onerror: Implement 'onerror' in
 exactly the same way as cffi on CPython
Message-ID: <20150704213029.B630D1C11C6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-callback-onerror
Changeset: r78433:77365fd8018a
Date: 2015-07-04 23:30 +0200
http://bitbucket.org/pypy/pypy/changeset/77365fd8018a/

Log:	Implement 'onerror' in exactly the same way as cffi on CPython

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -252,7 +252,8 @@
                                w_t, w_v, w_tb],
                 """(where, objrepr, extra_line, t, v, tb):
                     import sys, traceback
-                    sys.stderr.write('From %s%s:\\n' % (where, objrepr))
+                    if where or objrepr:
+                        sys.stderr.write('From %s%s:\\n' % (where, objrepr))
                     if extra_line:
                         sys.stderr.write(extra_line)
                     traceback.print_exception(t, v, tb)
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -93,14 +93,14 @@
         return ctype
 
     @jit.unroll_safe
-    def prepare_invoke_args(self, ll_args):
+    def invoke(self, ll_args):
         space = self.space
         ctype = self.getfunctype()
         args_w = []
         for i, farg in enumerate(ctype.fargs):
             ll_arg = rffi.cast(rffi.CCHARP, ll_args[i])
             args_w.append(farg.convert_to_object(ll_arg))
-        return space.newtuple(args_w)
+        return space.call(self.w_callable, space.newtuple(args_w))
 
     def convert_result(self, ll_res, w_res):
         fresult = self.getfunctype().ctitem
@@ -168,6 +168,29 @@
 STDERR = 2
 
 
+ at jit.dont_look_inside
+def _handle_applevel_exception(space, callback, e, ll_res, extra_line):
+    callback.write_error_return_value(ll_res)
+    if callback.w_onerror is None:
+        callback.print_error(e, extra_line)
+    else:
+        try:
+            e.normalize_exception(space)
+            w_t = e.w_type
+            w_v = e.get_w_value(space)
+            w_tb = space.wrap(e.get_traceback())
+            w_res = space.call_function(callback.w_onerror,
+                                        w_t, w_v, w_tb)
+            if not space.is_none(w_res):
+                callback.convert_result(ll_res, w_res)
+        except OperationError, e2:
+            # double exception! print a double-traceback...
+            callback.print_error(e, extra_line)    # original traceback
+            e2.write_unraisable(space, '', with_traceback=True,
+                                extra_line="\nDuring the call to 'onerror', "
+                                           "another exception occurred:\n\n")
+
+
 @jit.jit_callback("CFFI")
 def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
     """ Callback specification.
@@ -185,7 +208,7 @@
         try:
             os.write(STDERR, "SystemError: invoking a callback "
                              "that was already freed\n")
-        except OSError:
+        except:
             pass
         # In this case, we don't even know how big ll_res is.  Let's assume
         # it is just a 'ffi_arg', and store 0 there.
@@ -198,24 +221,11 @@
         must_leave = space.threadlocals.try_enter_thread(space)
         extra_line = ''
         try:
-            w_args = callback.prepare_invoke_args(ll_args)
-            try:
-                w_res = space.call(callback.w_callable, w_args)
-                extra_line = "Trying to convert the result back to C:\n"
-                callback.convert_result(ll_res, w_res)
-            except OperationError, e:
-                # got an app-level exception
-                if callback.w_onerror is None:
-                    raise
-                e.normalize_exception(space)
-                w_t = e.w_type
-                w_v = e.get_w_value(space)
-                w_tb = space.wrap(e.get_traceback())
-                space.call_function(callback.w_onerror, w_t, w_v, w_tb, w_args)
-                callback.write_error_return_value(ll_res)
+            w_res = callback.invoke(ll_args)
+            extra_line = "Trying to convert the result back to C:\n"
+            callback.convert_result(ll_res, w_res)
         except OperationError, e:
-            callback.print_error(e, extra_line)
-            callback.write_error_return_value(ll_res)
+            _handle_applevel_exception(space, callback, e, ll_res, extra_line)
         #
     except Exception, e:
         # oups! last-level attempt to recover.
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1170,6 +1170,14 @@
     BShort = new_primitive_type("short")
     BFunc = new_function_type((BShort,), BShort, False)
     f = callback(BFunc, Zcb1, -42)
+    #
+    seen = []
+    oops_result = None
+    def oops(*args):
+        seen.append(args)
+        return oops_result
+    ff = callback(BFunc, Zcb1, -42, oops)
+    #
     orig_stderr = sys.stderr
     orig_getline = linecache.getline
     try:
@@ -1195,6 +1203,59 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        assert len(seen) == 0
+        assert ff(bigvalue) == -42
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = 81
+        assert ff(bigvalue) == 81
+        oops_result = None
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = "xy"     # not None and not an int!
+        assert ff(bigvalue) == -42
+        oops_result = None
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+TypeError: $integer$
+""")
+        #
+        sys.stderr = cStringIO.StringIO()
+        seen = "not a list"    # this makes the oops() function crash
+        assert ff(bigvalue) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+Traceback (most recent call last):
+  File "$", line $, in oops
+    $
+AttributeError: 'str' object has no attribute 'append'
+""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -120,13 +120,13 @@
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
         seen = []
-        def myerror(exc, val, tb, args):
-            seen.append((exc, args))
+        def myerror(exc, val, tb):
+            seen.append(exc)
         cb = ffi.callback("int(int)", lambda x: x + "", onerror=myerror)
         assert cb(10) == 0
         cb = ffi.callback("int(int)", lambda x:int(1E100), -66, onerror=myerror)
-        assert cb(12) == -66
-        assert seen == [(TypeError, (10,)), (OverflowError, (12,))]
+        assert cb(10) == -66
+        assert seen == [TypeError, OverflowError]
 
     def test_ffi_callback_decorator(self):
         import _cffi_backend as _cffi1_backend
@@ -136,6 +136,37 @@
         assert deco(lambda x: x + "")(10) == -66
         assert deco(lambda x: x + 42)(10) == 52
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def oops(*args):
+            seen.append(args)
+
+        @ffi.callback("int(int)", onerror=oops)
+        def fn1(x):
+            return x + ""
+        assert fn1(10) == 0
+
+        @ffi.callback("int(int)", onerror=oops, error=-66)
+        def fn2(x):
+            return x + ""
+        assert fn2(10) == -66
+
+        assert len(seen) == 2
+        exc, val, tb = seen[0]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn1"
+        exc, val, tb = seen[1]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn2"
+        del seen[:]
+        #
+        raises(TypeError, ffi.callback, "int(int)",
+               lambda x: x, onerror=42)   # <- not callable
+
     def test_ffi_getctype(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()

From noreply at buildbot.pypy.org  Sat Jul  4 23:31:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 23:31:56 +0200 (CEST)
Subject: [pypy-commit] cffi default: Generalize the test for pypy
Message-ID: <20150704213156.71FC01C11C6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2207:5882a34c8d29
Date: 2015-07-04 23:32 +0200
http://bitbucket.org/cffi/cffi/changeset/5882a34c8d29/

Log:	Generalize the test for pypy

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1249,7 +1249,7 @@
 
 During the call to 'onerror', another exception occurred:
 
-TypeError: an integer is required
+TypeError: $integer$
 """)
         #
         sys.stderr = cStringIO.StringIO()
@@ -1264,7 +1264,7 @@
 
 Traceback (most recent call last):
   File "$", line $, in oops
-    seen.append(args)
+    $
 AttributeError: 'str' object has no attribute 'append'
 """)
     finally:

From noreply at buildbot.pypy.org  Sat Jul  4 23:35:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 23:35:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Translation fix
Message-ID: <20150704213547.598491C11C6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78434:f87b995d0d7a
Date: 2015-07-04 23:35 +0200
http://bitbucket.org/pypy/pypy/changeset/f87b995d0d7a/

Log:	Translation fix

diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -546,10 +546,10 @@
     # get in 'src_ctx' a NULL which translation doesn't consider to be constant
     src_ctx = rffi.cast(parse_c_type.PCTX, 0)
     r.__init__(space, src_ctx)
-    return space.wrap(r)
+    return r
 
 def W_FFIObject___new__(space, w_subtype, __args__):
-    return make_plain_ffi_object(space, w_subtype)
+    return space.wrap(make_plain_ffi_object(space, w_subtype))
 
 def make_CData(space):
     return space.gettypefor(W_CData)

From noreply at buildbot.pypy.org  Sat Jul  4 23:47:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 23:47:27 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-callback-onerror: Close branch,
	ready to merge
Message-ID: <20150704214727.BDDAD1C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-callback-onerror
Changeset: r78435:e4268e240c4b
Date: 2015-07-04 23:36 +0200
http://bitbucket.org/pypy/pypy/changeset/e4268e240c4b/

Log:	Close branch, ready to merge


From noreply at buildbot.pypy.org  Sat Jul  4 23:47:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 23:47:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: hg merge cffi-callback-onerror
Message-ID: <20150704214728.F41B51C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78436:2e6be5bbe1dc
Date: 2015-07-04 23:38 +0200
http://bitbucket.org/pypy/pypy/changeset/2e6be5bbe1dc/

Log:	hg merge cffi-callback-onerror

	update to cffi 'onerror' argument on callbacks (2b30856ad741,
	bdbb14a1b774)

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -286,7 +286,7 @@
         """
         return self._backend.from_buffer(self.BCharA, python_buffer)
 
-    def callback(self, cdecl, python_callable=None, error=None):
+    def callback(self, cdecl, python_callable=None, error=None, onerror=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
         The callback invokes the specified 'python_callable' (which may
@@ -298,7 +298,8 @@
             if not callable(python_callable):
                 raise TypeError("the 'python_callable' argument "
                                 "is not callable")
-            return self._backend.callback(cdecl, python_callable, error)
+            return self._backend.callback(cdecl, python_callable, error,
+                                          onerror)
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl, consider_function_as_funcptr=True)
         if python_callable is None:
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -252,7 +252,8 @@
                                w_t, w_v, w_tb],
                 """(where, objrepr, extra_line, t, v, tb):
                     import sys, traceback
-                    sys.stderr.write('From %s%s:\\n' % (where, objrepr))
+                    if where or objrepr:
+                        sys.stderr.write('From %s%s:\\n' % (where, objrepr))
                     if extra_line:
                         sys.stderr.write(extra_line)
                     traceback.print_exception(t, v, tb)
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -22,8 +22,9 @@
 class W_CDataCallback(W_CData):
     #_immutable_fields_ = ...
     ll_error = lltype.nullptr(rffi.CCHARP.TO)
+    w_onerror = None
 
-    def __init__(self, space, ctype, w_callable, w_error):
+    def __init__(self, space, ctype, w_callable, w_error, w_onerror):
         raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc())
         W_CData.__init__(self, space, raw_closure, ctype)
         #
@@ -31,6 +32,12 @@
             raise oefmt(space.w_TypeError,
                         "expected a callable object, not %T", w_callable)
         self.w_callable = w_callable
+        if not space.is_none(w_onerror):
+            if not space.is_true(space.callable(w_onerror)):
+                raise oefmt(space.w_TypeError,
+                            "expected a callable object for 'onerror', not %T",
+                            w_onerror)
+            self.w_onerror = w_onerror
         #
         fresult = self.getfunctype().ctitem
         size = fresult.size
@@ -161,6 +168,29 @@
 STDERR = 2
 
 
+ at jit.dont_look_inside
+def _handle_applevel_exception(space, callback, e, ll_res, extra_line):
+    callback.write_error_return_value(ll_res)
+    if callback.w_onerror is None:
+        callback.print_error(e, extra_line)
+    else:
+        try:
+            e.normalize_exception(space)
+            w_t = e.w_type
+            w_v = e.get_w_value(space)
+            w_tb = space.wrap(e.get_traceback())
+            w_res = space.call_function(callback.w_onerror,
+                                        w_t, w_v, w_tb)
+            if not space.is_none(w_res):
+                callback.convert_result(ll_res, w_res)
+        except OperationError, e2:
+            # double exception! print a double-traceback...
+            callback.print_error(e, extra_line)    # original traceback
+            e2.write_unraisable(space, '', with_traceback=True,
+                                extra_line="\nDuring the call to 'onerror', "
+                                           "another exception occurred:\n\n")
+
+
 @jit.jit_callback("CFFI")
 def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
     """ Callback specification.
@@ -178,7 +208,7 @@
         try:
             os.write(STDERR, "SystemError: invoking a callback "
                              "that was already freed\n")
-        except OSError:
+        except:
             pass
         # In this case, we don't even know how big ll_res is.  Let's assume
         # it is just a 'ffi_arg', and store 0 there.
@@ -195,9 +225,7 @@
             extra_line = "Trying to convert the result back to C:\n"
             callback.convert_result(ll_res, w_res)
         except OperationError, e:
-            # got an app-level exception
-            callback.print_error(e, extra_line)
-            callback.write_error_return_value(ll_res)
+            _handle_applevel_exception(space, callback, e, ll_res, extra_line)
         #
     except Exception, e:
         # oups! last-level attempt to recover.
@@ -205,7 +233,7 @@
             os.write(STDERR, "SystemError: callback raised ")
             os.write(STDERR, str(e))
             os.write(STDERR, "\n")
-        except OSError:
+        except:
             pass
         callback.write_error_return_value(ll_res)
     if must_leave:
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -276,8 +276,9 @@
 
 
     @unwrap_spec(w_python_callable=WrappedDefault(None),
-                 w_error=WrappedDefault(None))
-    def descr_callback(self, w_cdecl, w_python_callable, w_error):
+                 w_error=WrappedDefault(None),
+                 w_onerror=WrappedDefault(None))
+    def descr_callback(self, w_cdecl, w_python_callable, w_error, w_onerror):
         """\
 Return a callback object or a decorator making such a callback object.
 'cdecl' must name a C function pointer type.  The callback invokes the
@@ -290,14 +291,16 @@
         space = self.space
         if not space.is_none(w_python_callable):
             return ccallback.W_CDataCallback(space, w_ctype,
-                                             w_python_callable, w_error)
+                                             w_python_callable, w_error,
+                                             w_onerror)
         else:
             # decorator mode: returns a single-argument function
-            return space.appexec([w_ctype, w_error],
-            """(ctype, error):
+            return space.appexec([w_ctype, w_error, w_onerror],
+            """(ctype, error, onerror):
                 import _cffi_backend
                 return lambda python_callable: (
-                    _cffi_backend.callback(ctype, python_callable, error))""")
+                    _cffi_backend.callback(ctype, python_callable,
+                                           error, onerror))""")
 
 
     def descr_cast(self, w_arg, w_ob):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -18,9 +18,9 @@
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
-def callback(space, w_ctype, w_callable, w_error=None):
+def callback(space, w_ctype, w_callable, w_error=None, w_onerror=None):
     from pypy.module._cffi_backend.ccallback import W_CDataCallback
-    return W_CDataCallback(space, w_ctype, w_callable, w_error)
+    return W_CDataCallback(space, w_ctype, w_callable, w_error, w_onerror)
 
 # ____________________________________________________________
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1170,6 +1170,14 @@
     BShort = new_primitive_type("short")
     BFunc = new_function_type((BShort,), BShort, False)
     f = callback(BFunc, Zcb1, -42)
+    #
+    seen = []
+    oops_result = None
+    def oops(*args):
+        seen.append(args)
+        return oops_result
+    ff = callback(BFunc, Zcb1, -42, oops)
+    #
     orig_stderr = sys.stderr
     orig_getline = linecache.getline
     try:
@@ -1195,6 +1203,59 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        assert len(seen) == 0
+        assert ff(bigvalue) == -42
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = 81
+        assert ff(bigvalue) == 81
+        oops_result = None
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = "xy"     # not None and not an int!
+        assert ff(bigvalue) == -42
+        oops_result = None
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+TypeError: $integer$
+""")
+        #
+        sys.stderr = cStringIO.StringIO()
+        seen = "not a list"    # this makes the oops() function crash
+        assert ff(bigvalue) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+Traceback (most recent call last):
+  File "$", line $, in oops
+    $
+AttributeError: 'str' object has no attribute 'append'
+""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -116,6 +116,18 @@
         assert ffi.callback("int(int)", lambda x: x + "", -66)(10) == -66
         assert ffi.callback("int(int)", lambda x: x + "", error=-66)(10) == -66
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myerror(exc, val, tb):
+            seen.append(exc)
+        cb = ffi.callback("int(int)", lambda x: x + "", onerror=myerror)
+        assert cb(10) == 0
+        cb = ffi.callback("int(int)", lambda x:int(1E100), -66, onerror=myerror)
+        assert cb(10) == -66
+        assert seen == [TypeError, OverflowError]
+
     def test_ffi_callback_decorator(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
@@ -124,6 +136,37 @@
         assert deco(lambda x: x + "")(10) == -66
         assert deco(lambda x: x + 42)(10) == 52
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def oops(*args):
+            seen.append(args)
+
+        @ffi.callback("int(int)", onerror=oops)
+        def fn1(x):
+            return x + ""
+        assert fn1(10) == 0
+
+        @ffi.callback("int(int)", onerror=oops, error=-66)
+        def fn2(x):
+            return x + ""
+        assert fn2(10) == -66
+
+        assert len(seen) == 2
+        exc, val, tb = seen[0]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn1"
+        exc, val, tb = seen[1]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn2"
+        del seen[:]
+        #
+        raises(TypeError, ffi.callback, "int(int)",
+               lambda x: x, onerror=42)   # <- not callable
+
     def test_ffi_getctype(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()

From noreply at buildbot.pypy.org  Sat Jul  4 23:47:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  4 Jul 2015 23:47:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: Document branches
Message-ID: <20150704214730.18CDD1C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78437:ba44600bac9b
Date: 2015-07-04 23:47 +0200
http://bitbucket.org/pypy/pypy/changeset/ba44600bac9b/

Log:	Document branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -20,5 +20,15 @@
 .. branch: run-create_cffi_imports
 
 Build cffi import libraries as part of translation by monkey-patching an 
-aditional task into translation
+additional task into translation
 
+.. branch: int-float-list-strategy
+
+Use a compact strategy for Python lists that mix integers and floats,
+at least if the integers fit inside 32 bits.  These lists are now
+stored as an array of floats, like lists that contain only floats; the
+difference is that integers are stored as tagged NaNs.  (This should
+have no visible effect!  After ``lst = [42, 42.5]``, the value of
+``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
+
+.. branch: cffi-callback-onerror

From noreply at buildbot.pypy.org  Sun Jul  5 02:35:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  5 Jul 2015 02:35:08 +0200 (CEST)
Subject: [pypy-commit] cffi default: Minor clean up
Message-ID: <20150705003508.C2DEA1C1050@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2208:d1604ab4ecc6
Date: 2015-07-05 02:35 +0200
http://bitbucket.org/cffi/cffi/changeset/d1604ab4ecc6/

Log:	Minor clean up

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -4792,8 +4792,16 @@
     return;
 
  error:
+    if (SIGNATURE(1)->ct_size > 0) {
+        py_rawerr = PyTuple_GET_ITEM(cb_args, 2);
+        memcpy(result, PyBytes_AS_STRING(py_rawerr),
+                       PyBytes_GET_SIZE(py_rawerr));
+    }
     onerror_cb = PyTuple_GET_ITEM(cb_args, 3);
-    if (onerror_cb != Py_None) {
+    if (onerror_cb == Py_None) {
+        _my_PyErr_WriteUnraisable(py_ob, extra_error_line);
+    }
+    else {
         PyObject *exc1, *val1, *tb1, *res1, *exc2, *val2, *tb2;
         PyErr_Fetch(&exc1, &val1, &tb1);
         PyErr_NormalizeException(&exc1, &val1, &tb1);
@@ -4806,30 +4814,22 @@
             if (res1 != Py_None)
                 convert_from_object_fficallback(result, SIGNATURE(1), res1);
             Py_DECREF(res1);
-            if (!PyErr_Occurred()) {
-                Py_XDECREF(exc1);
-                Py_XDECREF(val1);
-                Py_XDECREF(tb1);
-                if (res1 != Py_None)
-                    goto done;
-                goto no_more_exception;
-            }
         }
-        /* double exception! print a double-traceback... */
-        PyErr_Fetch(&exc2, &val2, &tb2);
-        PyErr_Restore(exc1, val1, tb1);
-        _my_PyErr_WriteUnraisable(py_ob, extra_error_line);
-        PyErr_Restore(exc2, val2, tb2);
-        extra_error_line = ("\nDuring the call to 'onerror', "
-                            "another exception occurred:\n\n");
-        py_ob = NULL;
-    }
-    _my_PyErr_WriteUnraisable(py_ob, extra_error_line);
- no_more_exception:
-    if (SIGNATURE(1)->ct_size > 0) {
-        py_rawerr = PyTuple_GET_ITEM(cb_args, 2);
-        memcpy(result, PyBytes_AS_STRING(py_rawerr),
-                       PyBytes_GET_SIZE(py_rawerr));
+        if (!PyErr_Occurred()) {
+            Py_XDECREF(exc1);
+            Py_XDECREF(val1);
+            Py_XDECREF(tb1);
+        }
+        else {
+            /* double exception! print a double-traceback... */
+            PyErr_Fetch(&exc2, &val2, &tb2);
+            PyErr_Restore(exc1, val1, tb1);
+            _my_PyErr_WriteUnraisable(py_ob, extra_error_line);
+            PyErr_Restore(exc2, val2, tb2);
+            extra_error_line = ("\nDuring the call to 'onerror', "
+                                "another exception occurred:\n\n");
+            _my_PyErr_WriteUnraisable(NULL, extra_error_line);
+        }
     }
     goto done;
     }

From noreply at buildbot.pypy.org  Sun Jul  5 09:32:41 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  5 Jul 2015 09:32:41 +0200 (CEST)
Subject: [pypy-commit] cffi default: Document 'onerror'.
Message-ID: <20150705073241.4FFA61C103D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2209:08e9e358f971
Date: 2015-07-05 09:33 +0200
http://bitbucket.org/cffi/cffi/changeset/08e9e358f971/

Log:	Document 'onerror'.

diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -506,7 +506,7 @@
 
     @ffi.callback("int(int, int)", error=-1)
 
-In all cases the exception is printed to stderr, so this should be
+The exception is still printed to stderr, so this should be
 used only as a last-resort solution.
 
 Deprecated: you can also use ``ffi.callback()`` not as a decorator but
@@ -514,6 +514,30 @@
 discouraged: using this a style, we are more likely to forget the
 callback object too early, when it is still in use.
 
+.. versionadded:: 1.2
+
+   If you want to be sure to catch all exceptions, use
+   ``ffi.callback(..., onerror=func)``.  If an exception occurs and
+   ``onerror`` is specified, then ``onerror(exception, exc_value,
+   traceback)`` is called.  This is useful in some situations where
+   you cannot simply write ``try: except:`` in the main callback
+   function, because it might not catch exceptions raised by signal
+   handlers: if a signal occurs while in C, it will be called after
+   entering the main callback function but before executing the
+   ``try:``.
+
+   If ``onerror`` returns normally, then it is assumed that it handled
+   the exception on its own and nothing is printed to stderr.  If
+   ``onerror`` raises, then both tracebacks are printed.  Finally,
+   ``onerror`` can itself provide the result value of the callback in
+   C, but doesn't have to: if it simply returns None---or if
+   ``onerror`` itself fails---then the value of ``error`` will be
+   used, if any.
+
+   Note the following hack: in ``onerror``, you can access some of the
+   original callback arguments by attempting to read
+   ``traceback.tb_frame.f_locals['argname']``.
+
 
 FFI Interface
 -------------
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -41,6 +41,14 @@
   only for NULL: if you dereference random or dead pointers you might
   still get segfaults.
 
+* Issue #152: callbacks: added an argument ``ffi.callback(...,
+  onerror=...)``.  If the main callback function raises an exception
+  and ``onerror`` is provided, then ``onerror(exception, exc_value,
+  traceback)`` is called.  This is similar to writing a ``try:
+  except:`` in the main callback function, but in some cases (e.g. a
+  signal) an exception can occur at the very start of the callback
+  function---before it had time to enter the ``try: except:`` block.
+
 
 1.1.2
 =====

From noreply at buildbot.pypy.org  Sun Jul  5 09:58:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  5 Jul 2015 09:58:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Trying to fix this call,
 which should be done with the GIL released
Message-ID: <20150705075853.51AFC1C14DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78438:cb8906cadec3
Date: 2015-07-05 09:00 +0100
http://bitbucket.org/pypy/pypy/changeset/cb8906cadec3/

Log:	Trying to fix this call, which should be done with the GIL released

diff --git a/pypy/module/_cffi_backend/cglob.py b/pypy/module/_cffi_backend/cglob.py
--- a/pypy/module/_cffi_backend/cglob.py
+++ b/pypy/module/_cffi_backend/cglob.py
@@ -4,8 +4,7 @@
 from pypy.module._cffi_backend import newtype
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem import lltype, rffi
-
-FNPTR = rffi.CCallback([], rffi.VOIDP)
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 
 class W_GlobSupport(W_Root):
@@ -16,12 +15,20 @@
         self.space = space
         self.w_ctype = w_ctype
         self.ptr = ptr
-        self.fetch_addr = rffi.cast(FNPTR, fetch_addr)
+        self.fetch_addr = fetch_addr
 
     def fetch_global_var_addr(self):
         if self.ptr:
             return self.ptr
-        result = self.fetch_addr()
+        if not we_are_translated():
+            FNPTR = rffi.CCallback([], rffi.VOIDP)
+            fetch_addr = rffi.cast(FNPTR, self.fetch_addr)
+            result = fetch_addr()
+        else:
+            # careful in translated versions: we need to call fetch_addr,
+            # but in a GIL-releasing way.  The easiest is to invoke a
+            # llexternal() helper.
+            result = pypy__cffi_fetch_var(self.fetch_addr)
         return rffi.cast(rffi.CCHARP, result)
 
     def read_global_var(self):
@@ -37,3 +44,14 @@
 
 W_GlobSupport.typedef = TypeDef("FFIGlobSupport")
 W_GlobSupport.typedef.acceptable_as_base_class = False
+
+
+eci = ExternalCompilationInfo(post_include_bits=["""
+static void *pypy__cffi_fetch_var(void *fetch_addr) {
+    return ((void*(*)(void))fetch_addr)();
+}
+"""])
+
+pypy__cffi_fetch_var = rffi.llexternal(
+    "pypy__cffi_fetch_var", [rffi.VOIDP], rffi.VOIDP,
+    compilation_info=eci)

From noreply at buildbot.pypy.org  Sun Jul  5 11:39:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  5 Jul 2015 11:39:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Import cffi/08e9e358f971
Message-ID: <20150705093920.4D46C1C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78439:d791ed0bc8a9
Date: 2015-07-05 09:36 +0200
http://bitbucket.org/pypy/pypy/changeset/d791ed0bc8a9/

Log:	Import cffi/08e9e358f971

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -298,8 +298,8 @@
             if not callable(python_callable):
                 raise TypeError("the 'python_callable' argument "
                                 "is not callable")
-            return self._backend.callback(cdecl, python_callable, error,
-                                          onerror)
+            return self._backend.callback(cdecl, python_callable,
+                                          error, onerror)
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl, consider_function_as_funcptr=True)
         if python_callable is None:
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -989,7 +989,8 @@
     def cast(self, BType, source):
         return BType._cast_from(source)
 
-    def callback(self, BType, source, error):
+    def callback(self, BType, source, error, onerror):
+        assert onerror is None   # XXX not implemented
         return BType(source, error)
 
     typeof = type
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -39,6 +39,25 @@
         assert ffi.from_handle(ffi.cast("char *", p)) is o
         py.test.raises(RuntimeError, ffi.from_handle, ffi.NULL)
 
+    def test_callback_onerror(self):
+        ffi = FFI(backend=self.Backend())
+        seen = []
+        def oops(*args):
+            seen.append(args)
+        def otherfunc():
+            raise LookupError
+        def cb(n):
+            otherfunc()
+        a = ffi.callback("int(*)(int)", cb, error=42, onerror=oops)
+        res = a(234)
+        assert res == 42
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is LookupError
+        assert isinstance(val, LookupError)
+        assert tb.tb_frame.f_code.co_name == 'cb'
+        assert tb.tb_frame.f_locals['n'] == 234
+
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -105,6 +105,35 @@
     assert deco(lambda x: x + "")(10) == -66
     assert deco(lambda x: x + 42)(10) == 52
 
+def test_ffi_callback_onerror():
+    ffi = _cffi1_backend.FFI()
+    seen = []
+    def oops(*args):
+        seen.append(args)
+
+    @ffi.callback("int(int)", onerror=oops)
+    def fn1(x):
+        return x + ""
+    assert fn1(10) == 0
+
+    @ffi.callback("int(int)", onerror=oops, error=-66)
+    def fn2(x):
+        return x + ""
+    assert fn2(10) == -66
+
+    assert len(seen) == 2
+    exc, val, tb = seen[0]
+    assert exc is TypeError
+    assert isinstance(val, TypeError)
+    assert tb.tb_frame.f_code.co_name == "fn1"
+    exc, val, tb = seen[1]
+    assert exc is TypeError
+    assert isinstance(val, TypeError)
+    assert tb.tb_frame.f_code.co_name == "fn2"
+    #
+    py.test.raises(TypeError, ffi.callback, "int(int)",
+                   lambda x: x, onerror=42)   # <- not callable
+
 def test_ffi_getctype():
     ffi = _cffi1_backend.FFI()
     assert ffi.getctype("int") == "int"

From noreply at buildbot.pypy.org  Sun Jul  5 11:39:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  5 Jul 2015 11:39:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150705093921.76AA51C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78440:ac9cff797efb
Date: 2015-07-05 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/ac9cff797efb/

Log:	merge heads

diff --git a/pypy/module/_cffi_backend/cglob.py b/pypy/module/_cffi_backend/cglob.py
--- a/pypy/module/_cffi_backend/cglob.py
+++ b/pypy/module/_cffi_backend/cglob.py
@@ -4,8 +4,7 @@
 from pypy.module._cffi_backend import newtype
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem import lltype, rffi
-
-FNPTR = rffi.CCallback([], rffi.VOIDP)
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 
 class W_GlobSupport(W_Root):
@@ -16,12 +15,20 @@
         self.space = space
         self.w_ctype = w_ctype
         self.ptr = ptr
-        self.fetch_addr = rffi.cast(FNPTR, fetch_addr)
+        self.fetch_addr = fetch_addr
 
     def fetch_global_var_addr(self):
         if self.ptr:
             return self.ptr
-        result = self.fetch_addr()
+        if not we_are_translated():
+            FNPTR = rffi.CCallback([], rffi.VOIDP)
+            fetch_addr = rffi.cast(FNPTR, self.fetch_addr)
+            result = fetch_addr()
+        else:
+            # careful in translated versions: we need to call fetch_addr,
+            # but in a GIL-releasing way.  The easiest is to invoke a
+            # llexternal() helper.
+            result = pypy__cffi_fetch_var(self.fetch_addr)
         return rffi.cast(rffi.CCHARP, result)
 
     def read_global_var(self):
@@ -37,3 +44,14 @@
 
 W_GlobSupport.typedef = TypeDef("FFIGlobSupport")
 W_GlobSupport.typedef.acceptable_as_base_class = False
+
+
+eci = ExternalCompilationInfo(post_include_bits=["""
+static void *pypy__cffi_fetch_var(void *fetch_addr) {
+    return ((void*(*)(void))fetch_addr)();
+}
+"""])
+
+pypy__cffi_fetch_var = rffi.llexternal(
+    "pypy__cffi_fetch_var", [rffi.VOIDP], rffi.VOIDP,
+    compilation_info=eci)

From noreply at buildbot.pypy.org  Sun Jul  5 18:06:55 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  5 Jul 2015 18:06:55 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start writing direct tests of
 short preamble support
Message-ID: <20150705160655.19B0E1C1048@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78441:60495ed32c01
Date: 2015-07-05 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/60495ed32c01/

Log:	start writing direct tests of short preamble support

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -103,19 +103,6 @@
 
 # ____________________________________________________________
 
-class Memo(object):
-    def __init__(self, initial_args=None, initial_res=None):
-        self.snapshots = {}
-        self.box_mapping = {}
-        if initial_args is not None:
-            for i in range(len(initial_args)):
-                self.box_mapping[initial_args[i]] = initial_res[i]
-
-    def get(self, box, default):
-        return self.box_mapping.get(box, default)
-
-    def set(self, box, newbox):
-        self.box_mapping[box] = newbox
 
 def compile_loop(metainterp, greenkey, start,
                  inputargs, jumpargs,
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -221,10 +221,11 @@
         ops = sb.optimizer._newoperations
         for i, op in enumerate(ops):
             if op.is_always_pure():
-                sb.add_potential(op, op)
+                sb.add_potential(op)
             if op.is_ovf() and ops[i + 1].getopnum() == rop.GUARD_NO_OVERFLOW:
-                sb.add_potential(op, op)
+                sb.add_potential(op)
         for i in self.call_pure_positions:
+            yyy
             op = ops[i]
             assert op.getopnum() == rop.CALL
             op = op.copy_and_change(rop.CALL_PURE)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -0,0 +1,118 @@
+
+from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.history import Const
+
+class BoxNotProducable(Exception):
+    pass
+
+
+class ShortBoxes(object):
+    def __init__(self):
+        self.potential_ops = {}
+        self.inputarg_boxes = {}
+        self.synthetic = {}
+        self.alternatives = {}
+        self.extra_same_as = []
+
+    def create_short_boxes(self, optimizer, inputargs):
+        for box in inputargs:
+            self.inputarg_boxes[box] = None
+        optimizer.produce_potential_short_preamble_ops(self)
+
+        self.short_boxes = {}
+        self.short_boxes_in_production = {}
+
+        for op in self.potential_ops.keys():
+            try:
+                self.produce_short_preamble_op(op)
+            except BoxNotProducable:
+                pass
+
+        self.short_boxes_in_production = None # Not needed anymore
+
+    def prioritized_alternatives(self, box):
+        if box not in self.alternatives:
+            return [self.potential_ops[box]]
+        alts = self.alternatives[box]
+        hi, lo = 0, len(alts) - 1
+        while hi < lo:
+            if alts[lo] is None: # Inputarg, lowest priority
+                alts[lo], alts[-1] = alts[-1], alts[lo]
+                lo -= 1
+            elif alts[lo] not in self.synthetic: # Hi priority
+                alts[hi], alts[lo] = alts[lo], alts[hi]
+                hi += 1
+            else: # Low priority
+                lo -= 1
+        return alts
+
+    def add_to_short(self, op):
+        if op in self.short_boxes:
+            xxx
+            #if op is None:
+            #    xxx
+            #    oldop = self.short_boxes[box]
+            #    self.rename[op] = oldop
+            #    self.short_boxes[box] = None
+            #    self.short_boxes[oldop] = oldop
+            #else:
+            #    xxxx
+            #    newop = op.clone()
+            #    newbox = newop.result = op.result.clonebox()
+            #    self.short_boxes[newop.result] = newop
+            #xxx
+            #value = self.optimizer.getvalue(box)
+            #self.optimizer.emit_operation(ResOperation(rop.SAME_AS, [box], newbox))
+            #self.optimizer.make_equal_to(newbox, value)
+            #if op is None:
+            #    if self.short_boxes[box] is not box:
+            #        xxx
+            #else:
+            #    if self.short_boxes[box] is not op:
+            #        if self.short_boxes[box] is None:
+            #            self.short_boxes[box] = op
+            #        else:
+            #            xxx
+        else:
+            self.short_boxes[op] = None
+
+    def produce_short_preamble_op(self, op):
+        if op in self.short_boxes:
+            return
+        if op in self.inputarg_boxes:
+            return
+        if isinstance(op, Const):
+            return
+        if op in self.short_boxes_in_production:
+            raise BoxNotProducable
+        self.short_boxes_in_production[op] = None
+
+        if op in self.potential_ops:
+            ops = self.prioritized_alternatives(op)
+            produced_one = False
+            for newop in ops:
+                try:
+                    if newop:
+                        for arg in newop.getarglist():
+                            self.produce_short_preamble_op(arg)
+                except BoxNotProducable:
+                    pass
+                else:
+                    produced_one = True
+                    self.add_to_short(newop)
+            if not produced_one:
+                raise BoxNotProducable
+        else:
+            raise BoxNotProducable
+
+    def add_potential(self, op, result=None, synthetic=False):
+        if result is None:
+            result = op
+        if result in self.potential_ops:
+            if result not in self.alternatives:
+                self.alternatives[result] = [self.potential_ops[result]]
+            self.alternatives[result].append(op)
+        else:
+            self.potential_ops[result] = op
+        if synthetic:
+            self.synthetic[result] = True
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_short.py b/rpython/jit/metainterp/optimizeopt/test/test_short.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/test/test_short.py
@@ -0,0 +1,48 @@
+
+""" Short preamble tests
+"""
+
+from rpython.jit.metainterp.resoperation import InputArgInt, ResOperation, rop
+from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes
+from rpython.jit.metainterp.history import AbstractDescr
+
+class Descr(AbstractDescr):
+    pass
+
+class Opt(object):
+    def __init__(self, oplist):
+        self.oplist = oplist
+    
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.oplist:
+            if isinstance(op, tuple):
+                op, r = op
+            else:
+                op, r = op, op
+            sb.add_potential(op, r)
+
+class TestShortBoxes(object):
+    def test_pure_ops(self):
+        i0 = InputArgInt()
+        i1 = InputArgInt()
+        op = ResOperation(rop.INT_ADD, [i0, i1])
+        sb = ShortBoxes()
+        sb.create_short_boxes(Opt([op]), [i0, i1])
+        assert sb.short_boxes == {op: None}
+
+    def test_pure_ops_does_not_work(self):
+        i0 = InputArgInt()
+        i1 = InputArgInt()
+        op = ResOperation(rop.INT_ADD, [i0, i1])
+        sb = ShortBoxes()
+        sb.create_short_boxes(Opt([op]), [i0])
+        assert sb.short_boxes == {}
+
+    def test_multiple_similar_ops(self):
+        i0 = InputArgInt()
+        i1 = InputArgInt()
+        op = ResOperation(rop.INT_ADD, [i0, i1])
+        op1 = ResOperation(rop.GETFIELD_GC_I, [i0], descr=Descr())
+        sb = ShortBoxes()
+        sb.create_short_boxes(Opt([op, (op1, op)]), [i0, i1])
+        
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -4,11 +4,11 @@
 
 from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest,\
      LLtypeMixin, FakeMetaInterpStaticData
-from rpython.jit.metainterp.history import (TreeLoop, AbstractDescr,
+from rpython.jit.metainterp.history import (TreeLoop, AbstractDescr, ConstInt,
                                             JitCellToken, TargetToken)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
-     NotVirtualStateInfo, LEVEL_CONSTANT
+     NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN
 
 class FakeOptimizer(object):
     optearlyforce = None
@@ -38,7 +38,7 @@
         start_state = self._do_optimize_loop(preamble, None,
                                              export_state=True)
         vs = preamble.operations[-1].getdescr().virtual_state
-        return start_state, vs
+        return start_state, vs, loop
 
     def test_simple(self):
         loop = """
@@ -47,7 +47,22 @@
         guard_value(i1, 1) []
         jump(i1)
         """
-        es, vs = self.optimize(loop)
+        es, vs, loop = self.optimize(loop)
         assert isinstance(vs.state[0], NotVirtualStateInfo)
+        # the virtual state is constant, so we don't need to have it in
+        # inputargs
         assert vs.make_inputargs([1], FakeOptimizer()) == []
         assert vs.state[0].level == LEVEL_CONSTANT
+        # we have exported values for i1, which happens to be an inputarg
+        assert es.inputarg_mapping[0][1].getint() == 1
+        assert isinstance(es.inputarg_mapping[0][1], ConstInt)
+
+    def test_not_constant(self):
+        loop = """
+        [i0]
+        i1 = int_add(i0, 1)
+        jump(i0)
+        """
+        es, vs, loop = self.optimize(loop)
+        assert isinstance(vs.state[0], NotVirtualStateInfo)
+        assert vs.state[0].level == LEVEL_UNKNOWN
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -464,7 +464,6 @@
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
-        start_state.orig_inputargs = inputargs
         self._do_optimize_loop(loop, call_pure_results, start_state,
                                export_state=False)
         extra_same_as = []
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,6 +1,5 @@
 import sys
 
-from rpython.jit.metainterp.compile import Memo
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
 from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.metainterp.optimize import InvalidLoop
@@ -8,10 +7,9 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
      Optimization
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
-        ShortBoxes, BadVirtualState, VirtualStatesCantMatch)
+        BadVirtualState, VirtualStatesCantMatch)
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      OpHelpers, AbstractInputArg, GuardResOp, AbstractResOp
-from rpython.jit.metainterp.resume import Snapshot
 from rpython.jit.metainterp import compile
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
 
@@ -44,12 +42,6 @@
         self.emitting_dissabled = False
         self.emitted_guards = 0
 
-    def ensure_imported(self, value):
-        if not self.emitting_dissabled and value in self.importable_values:
-            imp = self.importable_values[value]
-            del self.importable_values[value]
-            imp.import_value(value)
-
     def emit_operation(self, op):
         if self.emitting_dissabled:
             return
@@ -82,21 +74,14 @@
         modifier = VirtualStateConstructor(self.optimizer)
         return modifier.get_virtual_state(args)
 
-    def fix_snapshot(self, jump_args, snapshot):
-        if snapshot is None:
-            return None
-        snapshot_args = snapshot.boxes
-        new_snapshot_args = []
-        for a in snapshot_args:
-            a = self.getvalue(a).get_key_box()
-            new_snapshot_args.append(a)
-        prev = self.fix_snapshot(jump_args, snapshot.prev)
-        return Snapshot(prev, new_snapshot_args)
-
     def propagate_all_forward(self, starting_state, export_state=True):
         self.optimizer.exporting_state = export_state
         loop = self.optimizer.loop
         self.optimizer.clear_newoperations()
+        for op in loop.operations:
+            assert op.get_forwarded() is None
+        for op in loop.inputargs:
+            assert op.get_forwarded() is None
 
         start_label = loop.operations[0]
         if start_label.getopnum() == rop.LABEL:
@@ -220,30 +205,30 @@
 
     def export_state(self, targetop):
         original_jump_args = targetop.getarglist()
+        label_op = self.optimizer.loop.operations[0]
         jump_args = [self.get_box_replacement(a) for a in original_jump_args]
+        virtual_state = self.get_virtual_state(jump_args)
+        target_token = targetop.getdescr()
+        assert isinstance(target_token, TargetToken)
+        target_token.virtual_state = virtual_state
+        return ExportedState([(label_op.getarg(i), jump_args[i]) for i in range(len(jump_args))], [], [], [])
+        xxx
 
-        virtual_state = self.get_virtual_state(jump_args)
 
         inputargs = virtual_state.make_inputargs(jump_args, self.optimizer)
         short_inputargs = virtual_state.make_inputargs(jump_args,
                                             self.optimizer, keyboxes=True)
 
-        if self.boxes_created_this_iteration is not None:
-            for box in self.inputargs:
-                self.boxes_created_this_iteration[box] = None
-
-        short_boxes = ShortBoxes(self.optimizer, inputargs)
-
-        proven_constants = []
-        for i in range(len(original_jump_args)):
-            srcbox = jump_args[i]
-            if srcbox is not original_jump_args[i]:
-                if srcbox.type == 'r':
-                    info = self.optimizer.getptrinfo(srcbox)
-                    if info and info.is_virtual():
-                        xxx
-            if original_jump_args[i] is not srcbox and srcbox.is_constant():
-                proven_constants.append((original_jump_args[i], srcbox))
+        #proven_constants = []
+        #for i in range(len(original_jump_args)):
+        #    srcbox = jump_args[i]
+        ##    if srcbox is not original_jump_args[i]:
+        #        if srcbox.type == 'r':
+        #            info = self.optimizer.getptrinfo(srcbox)
+        #            if info and info.is_virtual():
+        #                xxx
+        #    if original_jump_args[i] is not srcbox and srcbox.is_constant():
+        #        proven_constants.append((original_jump_args[i], srcbox))
                 #opnum = OpHelpers.same_as_for_type(original_jump_args[i].type)
                 #op = ResOperation(opnum, [srcbox])
                 #self.optimizer.emit_operation(op)
@@ -263,20 +248,16 @@
         #inputarg_setup_ops = original_jump_args
         #inputarg_setup_ops = self.optimizer.get_newoperations()
 
-        target_token = targetop.getdescr()
-        assert isinstance(target_token, TargetToken)
-        targetop.initarglist(inputargs)
-        target_token.virtual_state = virtual_state
         target_token.short_preamble = [ResOperation(rop.LABEL, short_inputargs, None)]
 
-        exported_values = {}
-        for box in inputargs:
-            exported_values[box] = self.optimizer.getinfo(box)
-        for op in short_boxes.operations():
-            if op and op.type != 'v':
-                exported_values[op] = self.optimizer.getinfo(op)
+        #exported_values = {}
+        #for box in inputargs:
+        #    exported_values[box] = self.optimizer.getinfo(box)
+        #for op in short_boxes.operations():
+        #    if op and op.type != 'v':
+        #        exported_values[op] = self.optimizer.getinfo(op)
 
-        return ExportedState(short_boxes, proven_constants, exported_values)
+        return ExportedState([], [])
 
     def import_state(self, targetop, exported_state):
         if not targetop: # Trace did not start with a label
@@ -295,21 +276,20 @@
             self.initial_virtual_state = virtual_state
             return
 
-        self.short = target_token.short_preamble[:]
-        self.short_seen = {}
-        self.short_boxes = exported_state.short_boxes
+        self.short = [] # target_token.short_preamble[:]
+        #self.short_seen = {}
         self.initial_virtual_state = target_token.virtual_state
 
-        inpargs = self.initial_virtual_state.make_inputargs(
-            exported_state.orig_inputargs, self.optimizer)
-        for i, arg in enumerate(inpargs):
-            if arg is not self.inputargs[i]:
-                arg.set_forwarded(self.inputargs[i])
-        for box in self.inputargs:
-            preamble_info = exported_state.exported_values[box]
-            self.optimizer.setinfo_from_preamble(box, preamble_info)
-        for box, const in exported_state.proven_constants:
-            box.set_forwarded(const)
+        #inpargs = self.initial_virtual_state.make_inputargs(
+        #    exported_state.orig_inputargs, self.optimizer)
+        #for i, arg in enumerate(inpargs):
+        #    if arg is not self.inputargs[i]:
+        #        arg.set_forwarded(self.inputargs[i])
+        #for box in self.inputargs:
+        #    preamble_info = exported_state.exported_values[box]
+        #    self.optimizer.setinfo_from_preamble(box, preamble_info)
+        #for box, const in exported_state.state:
+        #    box.set_forwarded(const)
 
         # Setup the state of the new optimizer by emiting the
         # short operations and discarding the result
@@ -318,14 +298,14 @@
         #for source, target in exported_state.inputarg_setup_ops:
         #    source.set_forwarded(target)
 
-        for op in self.short_boxes.operations():
-            if not op:
-                continue
-            if op.is_always_pure():
-                self.pure(op.getopnum(),
-                          PreambleOp(op, self.optimizer.getinfo(op)))
-            else:
-                yyy
+        #for op in self.short_boxes.operations():
+        #    if not op:
+        #        continue
+        #    if op.is_always_pure():
+        #        self.pure(op.getopnum(),
+        #                  PreambleOp(op, self.optimizer.getinfo(op)))
+        #    else:
+        #        yyy
         return
         seen = {}
         for op in self.short_boxes.operations():
@@ -717,23 +697,30 @@
                                       'it has at the start of the target loop')
             i += 1
 
-
-class ValueImporter(object):
-    def __init__(self, unroll, value, op):
-        self.unroll = unroll
-        self.preamble_value = value
-        self.op = op
-
-    def import_value(self, value):
-        value.import_from(self.preamble_value, self.unroll.optimizer)
-        self.unroll.add_op_to_short(self.op, False, True)
-
+class ShortPreambleBuilder(object):
+    """ A place that builds short preamble and the necessary
+    arguments for the label and the jump
+    """
 
 class ExportedState(object):
-    def __init__(self, short_boxes, proven_constants, exported_values):
-        self.short_boxes = short_boxes
-        self.proven_constants = proven_constants
-        self.exported_values = exported_values
+    """ Exported state consists of a few pieces of information:
+
+    * inputarg_mapping - a list of tuples with original inputarg box
+                         as the first element and the second element being
+                         what it maps to (potentially const)
+    * exported_infos - a mapping from ops to infos, including inputargs
+    * pure operations - a list of pure operations that can produce various
+                        ops
+    * heap cache operations - an additional list of how to produce various
+                              ops
+    """
+    
+    def __init__(self, inputarg_mapping, exported_infos, pure_ops,
+                 heap_cache_ops):
+        self.inputarg_mapping = inputarg_mapping
+        self.exported_infos = exported_infos
+        self.pure_ops = pure_ops
+        self.heap_cache_ops = heap_cache_ops
 
     def dump(self, metainterp_sd):
         debug_start("jit-exported-state")
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -5,12 +5,12 @@
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractInputArg
-from rpython.jit.metainterp.compile import Memo
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.objectmodel import we_are_translated
 
 LEVEL_UNKNOWN = '\x00'
-LEVEL_CONSTANT = '\x01'
+LEVEL_NONNULL = '\x01'
+LEVEL_CONSTANT = '\x02'
 
 class BadVirtualState(Exception):
     pass
@@ -286,6 +286,8 @@
         info = optimizer.getinfo(box)
         if info and info.is_constant():
             self.level = LEVEL_CONSTANT
+        elif box.type == 'r' and info and info.is_nonnull():
+            self.level = LEVEL_NONNULL
         else:
             self.level = LEVEL_UNKNOWN
         return
@@ -617,172 +619,3 @@
     def visit_varraystruct(self, arraydescr, fielddescrs):
         return VArrayStructStateInfo(arraydescr, fielddescrs)
 
-
-class BoxNotProducable(Exception):
-    pass
-
-
-class ShortBoxes(object):
-    def __init__(self, optimizer, surviving_boxes):
-        self.potential_ops = {}
-        self.alternatives = {}
-        self.synthetic = {}
-        self.optimizer = optimizer
-        self.assumed_classes = {}
-
-        assert surviving_boxes is not None
-        for box in surviving_boxes:
-            self.potential_ops[box] = None
-        optimizer.produce_potential_short_preamble_ops(self)
-
-        self.short_boxes = {}
-        self.short_boxes_in_production = {}
-
-        for op in self.potential_ops.keys():
-            try:
-                self.produce_short_preamble_op(op)
-            except BoxNotProducable:
-                pass
-
-        self.short_boxes_in_production = None # Not needed anymore
-        #else:
-        #    self.short_boxes = {}
-
-    def dump(self, logops):
-        if 0:
-            for op, resop in self.short_boxes.iteritems():
-                if isinstance(op, AbstractInputArg):
-                    assert resop is None
-                    debug_print(logops.repr_of_arg(op))
-                else:
-                    debug_print("%s:%s"% (logops.repr_of_resop(op),
-                                          logops.repr_of_resop(resop)))
-
-    def prioritized_alternatives(self, box):
-        if box not in self.alternatives:
-            return [self.potential_ops[box]]
-        alts = self.alternatives[box]
-        hi, lo = 0, len(alts) - 1
-        while hi < lo:
-            if alts[lo] is None: # Inputarg, lowest priority
-                alts[lo], alts[-1] = alts[-1], alts[lo]
-                lo -= 1
-            elif alts[lo] not in self.synthetic: # Hi priority
-                alts[hi], alts[lo] = alts[lo], alts[hi]
-                hi += 1
-            else: # Low priority
-                lo -= 1
-        return alts
-
-    def add_to_short(self, box, op):
-        if box in self.short_boxes:
-            xxx
-            return # XXX avoid those corner cases
-            #if op is None:
-            #    xxx
-            #    oldop = self.short_boxes[box]
-            #    self.rename[op] = oldop
-            #    self.short_boxes[box] = None
-            #    self.short_boxes[oldop] = oldop
-            #else:
-            #    xxxx
-            #    newop = op.clone()
-            #    newbox = newop.result = op.result.clonebox()
-            #    self.short_boxes[newop.result] = newop
-            #xxx
-            #value = self.optimizer.getvalue(box)
-            #self.optimizer.emit_operation(ResOperation(rop.SAME_AS, [box], newbox))
-            #self.optimizer.make_equal_to(newbox, value)
-            #if op is None:
-            #    if self.short_boxes[box] is not box:
-            #        xxx
-            #else:
-            #    if self.short_boxes[box] is not op:
-            #        if self.short_boxes[box] is None:
-            #            self.short_boxes[box] = op
-            #        else:
-            #            xxx
-            if op is None:
-                oldop = self.short_boxes[box].clone()
-                oldres = oldop.result
-                newbox = oldop.result = oldres.clonebox()
-                self.rename[box] = newbox
-                self.short_boxes[box] = None
-                self.short_boxes[newbox] = oldop
-            else:
-                newop = op.clone()
-                newbox = newop.result = op.result.clonebox()
-                self.short_boxes[newop.result] = newop
-            value = self.optimizer.getvalue(box)
-            self.optimizer.emit_operation(ResOperation(rop.SAME_AS, [box], newbox))
-            self.optimizer.make_equal_to(newbox, value)
-        else:
-            self.short_boxes[box] = op
-
-    def produce_short_preamble_op(self, op):
-        if op in self.short_boxes:
-            return
-        if isinstance(op, Const):
-            return
-        if op in self.short_boxes_in_production:
-            raise BoxNotProducable
-        self.short_boxes_in_production[op] = None
-
-        if op in self.potential_ops:
-            ops = self.prioritized_alternatives(op)
-            produced_one = False
-            for newop in ops:
-                try:
-                    if newop:
-                        for arg in newop.getarglist():
-                            self.produce_short_preamble_op(arg)
-                except BoxNotProducable:
-                    pass
-                else:
-                    produced_one = True
-                    self.add_to_short(op, newop)
-            if not produced_one:
-                raise BoxNotProducable
-        else:
-            raise BoxNotProducable
-
-    def add_potential(self, key, op, synthetic=False):
-        #try:
-        #    value = self.optimizer.values[key]
-        #    if value in self.optimizer.opaque_pointers:
-        #        classbox = value.get_constant_class(self.optimizer.cpu)
-        #        if classbox:
-        #            self.assumed_classes[key] = classbox
-        #except KeyError:
-        #    pass
-        if key not in self.potential_ops:
-            self.potential_ops[key] = op
-        else:
-            if key not in self.alternatives:
-                self.alternatives[key] = [self.potential_ops[key]]
-            self.alternatives[key].append(op)
-        if synthetic:
-            self.synthetic[op] = True
-
-    def debug_print(self, logops):
-        if 0:
-            debug_start('jit-short-boxes')
-            for box, op in self.short_boxes.items():
-                if op:
-                    debug_print(logops.repr_of_arg(box) + ': ' + logops.repr_of_resop(op))
-                else:
-                    debug_print(logops.repr_of_arg(box) + ': None')
-            debug_stop('jit-short-boxes')
-
-    def operations(self):
-        if not we_are_translated(): # For tests
-            ops = self.short_boxes.values()
-            ops.sort(key=str, reverse=True)
-            return ops
-        return self.short_boxes.values()
-
-    def producer(self, box):
-        return self.short_boxes[box]
-
-    def has_producer(self, box):
-        return box in self.short_boxes

From noreply at buildbot.pypy.org  Sun Jul  5 18:49:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  5 Jul 2015 18:49:30 +0200 (CEST)
Subject: [pypy-commit] cffi default: Expand the explanation of the hack
Message-ID: <20150705164930.E6FF41C080A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2210:56ce956f2e0d
Date: 2015-07-05 18:50 +0200
http://bitbucket.org/cffi/cffi/changeset/56ce956f2e0d/

Log:	Expand the explanation of the hack

diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -534,9 +534,14 @@
    ``onerror`` itself fails---then the value of ``error`` will be
    used, if any.
 
-   Note the following hack: in ``onerror``, you can access some of the
-   original callback arguments by attempting to read
-   ``traceback.tb_frame.f_locals['argname']``.
+   Note the following hack: in ``onerror``, you can access the original
+   callback arguments as follows.  First check if ``traceback`` is not
+   None (it is None e.g. if the whole function ran successfully but
+   there was an error converting the value returned: this occurs after
+   the call).  If ``traceback`` is not None, then ``traceback.tb_frame``
+   is the frame of the outermost function, i.e. directly the one invoked
+   by the callback handler.  So you can get the value of ``argname`` in
+   that frame by reading ``traceback.tb_frame.f_locals['argname']``.
 
 
 FFI Interface

From noreply at buildbot.pypy.org  Mon Jul  6 09:30:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 09:30:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: 32-bit fix
Message-ID: <20150706073057.746731C1C48@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78442:3c9b5167cbbe
Date: 2015-07-06 09:30 +0200
http://bitbucket.org/pypy/pypy/changeset/3c9b5167cbbe/

Log:	32-bit fix

diff --git a/rpython/rlib/test/test_longlong2float.py b/rpython/rlib/test/test_longlong2float.py
--- a/rpython/rlib/test/test_longlong2float.py
+++ b/rpython/rlib/test/test_longlong2float.py
@@ -88,7 +88,7 @@
 
 def test_compiled_encode_nan():
     fn2 = compile(fn_encode_nan, [float, int])
-    ints = [-2**31, 2**31-1, 42]
+    ints = [int(-2**31), int(2**31-1), 42]
     for x in enum_floats():
         y = ints.pop()
         ints.insert(0, y)

From noreply at buildbot.pypy.org  Mon Jul  6 09:36:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 09:36:34 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: CFFI Issue #115: trying to
 implement ffi.new_allocator()
Message-ID: <20150706073634.C55C11C1C96@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78443:877d00ea2662
Date: 2015-07-05 20:52 +0200
http://bitbucket.org/pypy/pypy/changeset/877d00ea2662/

Log:	CFFI Issue #115: trying to implement ffi.new_allocator()


From noreply at buildbot.pypy.org  Mon Jul  6 09:36:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 09:36:36 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: in-progress
Message-ID: <20150706073636.14AE81C1C96@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78444:0514cea3666b
Date: 2015-07-06 09:36 +0200
http://bitbucket.org/pypy/pypy/changeset/0514cea3666b/

Log:	in-progress

diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -0,0 +1,48 @@
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+class W_Allocator(W_Root):
+    _immutable_ = True
+
+    def __init__(self, ffi, should_clear_after_alloc):
+        self.ffi = ffi
+        self.should_clear_after_alloc = should_clear_after_alloc
+
+    def allocate(self, space, datasize, ctype, length=-1):
+        from pypy.module._cffi_backend.cdataobj import W_CDataNewStd
+        if self.should_clear_after_alloc:
+            ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                flavor='raw', zero=True)
+        else:
+            ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                flavor='raw', zero=False)
+        return W_CDataNewStd(space, ptr, ctype, length)
+
+    @unwrap_spec(w_init=WrappedDefault(None))
+    def descr_call(self, space, w_arg, w_init):
+        from pypy.module._cffi_backend.ctypeobj import W_CType
+        if isinstance(w_arg, W_CType):
+            w_ctype = w_arg
+        else:
+            ffi = self.ffi
+            if ffi is None:
+                raise oefmt(space.w_TypeError,
+                            "expected a ctype object, got '%T'", w_arg)
+            w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING)
+        return w_ctype.newp(w_init, self)
+
+
+W_Allocator.typedef = TypeDef(
+        'FFIAllocator',
+        __call__ = interp2app(W_Allocator.descr_call),
+        )
+W_Allocator.typedef.acceptable_as_base_class = False
+
+
+default_allocator = W_Allocator(ffi=None, should_clear_after_alloc=True)
+nonzero_allocator = W_Allocator(ffi=None, should_clear_after_alloc=False)
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -365,14 +365,13 @@
 
 
 class W_CDataMem(W_CData):
-    """This is the base class used for cdata objects that own and free
-    their memory.  Used directly by the results of cffi.cast('int', x)
-    or other primitive explicitly-casted types.  It is further subclassed
-    by W_CDataNewOwning."""
+    """This is used only by the results of cffi.cast('int', x)
+    or other primitive explicitly-casted types."""
     _attrs_ = []
 
-    def __init__(self, space, size, ctype):
-        cdata = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw', zero=True)
+    def __init__(self, space, ctype):
+        cdata = lltype.malloc(rffi.CCHARP.TO, ctype.size, flavor='raw',
+                              zero=False)
         W_CData.__init__(self, space, cdata, ctype)
 
     @rgc.must_be_light_finalizer
@@ -380,34 +379,51 @@
         lltype.free(self._ptr, flavor='raw')
 
 
-class W_CDataNewOwning(W_CDataMem):
-    """This is the class used for the cata objects created by newp()."""
-    _attrs_ = []
+class W_CDataNewOwning(W_CData):
+    """This is the abstract base class used for cdata objects created
+    by newp().  They create and free their own memory according to an
+    allocator."""
+
+    # the 'length' is either >= 0 for arrays, or -1 for pointers.
+    _attrs_ = ['length']
+    _immutable_fields_ = ['length']
+
+    def __init__(self, space, cdata, ctype, length=-1):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.length = length
 
     def _repr_extra(self):
         return self._repr_extra_owning()
 
-
-class W_CDataNewOwningLength(W_CDataNewOwning):
-    """Subclass with an explicit length, for allocated instances of
-    the C type 'foo[]'."""
-    _attrs_ = ['length']
-    _immutable_fields_ = ['length']
-
-    def __init__(self, space, size, ctype, length):
-        W_CDataNewOwning.__init__(self, space, size, ctype)
-        self.length = length
-
     def _sizeof(self):
-        from pypy.module._cffi_backend import ctypearray
         ctype = self.ctype
-        assert isinstance(ctype, ctypearray.W_CTypeArray)
-        return self.length * ctype.ctitem.size
+        if self.length >= 0:
+            from pypy.module._cffi_backend import ctypearray
+            assert isinstance(ctype, ctypearray.W_CTypeArray)
+            return self.length * ctype.ctitem.size
+        else:
+            return ctype.size
 
     def get_array_length(self):
         return self.length
 
 
+class W_CDataNewStd(W_CDataNewOwning):
+    """Subclass using the standard allocator, lltype.malloc()/lltype.free()"""
+    _attrs_ = []
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        lltype.free(self._ptr, flavor='raw')
+
+
+class W_CDataMemNonStd(W_CDataNewOwning):
+    """Subclass using a non-standard allocator"""
+    _attrs_ = []
+
+    # XXXXXXXXX
+
+
 class W_CDataPtrToStructOrUnion(W_CData):
     """This subclass is used for the pointer returned by new('struct foo').
     It has a strong reference to a W_CDataNewOwning that really owns the
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -28,7 +28,7 @@
     def _alignof(self):
         return self.ctitem.alignof()
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         datasize = self.size
         #
@@ -40,12 +40,10 @@
             except OverflowError:
                 raise OperationError(space.w_OverflowError,
                     space.wrap("array size would overflow a ssize_t"))
-            #
-            cdata = cdataobj.W_CDataNewOwningLength(space, datasize,
-                                                    self, length)
+        else:
+            length = self.length
         #
-        else:
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+        cdata = allocator.allocate(space, datasize, self, length)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -55,7 +55,7 @@
     def pack_list_of_items(self, cdata, w_ob):
         return False
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         raise oefmt(space.w_TypeError,
                     "expected a pointer or array ctype, got '%s'", self.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -63,7 +63,7 @@
             value = self._cast_result(value)
         else:
             value = self._cast_generic(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         self.write_raw_integer_data(w_cdata, value)
         return w_cdata
 
@@ -353,7 +353,7 @@
             value = self.cast_unicode(w_ob)
         else:
             value = space.float_w(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         if not isinstance(self, W_CTypePrimitiveLongDouble):
             w_cdata.write_raw_float_data(value)
         else:
@@ -446,7 +446,7 @@
         return self.space.wrap(value)
 
     def convert_to_object(self, cdata):
-        w_cdata = cdataobj.W_CDataMem(self.space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(self.space, self)
         with w_cdata as ptr:
             self._copy_longdouble(cdata, ptr)
         return w_cdata
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -185,7 +185,7 @@
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
         space = self.space
         ctitem = self.ctitem
@@ -205,14 +205,14 @@
                     datasize = ctitem.convert_struct_from_object(
                         lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
             #
-            cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
+            cdatastruct = allocator.allocate(space, datasize, ctitem)
             ptr = cdatastruct.unsafe_escaping_ptr()
             cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
                                                        self, cdatastruct)
         else:
             if self.is_char_or_unichar_ptr_or_array():
                 datasize *= 2       # forcefully add a null character
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+            cdata = allocator.allocate(space, datasize, self)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -81,10 +81,9 @@
     def copy_and_convert_to_object(self, source):
         space = self.space
         self.check_complete()
-        ob = cdataobj.W_CDataNewOwning(space, self.size, self)
-        with ob as target:
-            misc._raw_memcopy(source, target, self.size)
-        return ob
+        ptr = lltype.malloc(rffi.CCHARP.TO, self.size, flavor='raw', zero=False)
+        misc._raw_memcopy(source, ptr, self.size)
+        return cdataobj.W_CDataNewStd(space, ptr, self)
 
     def typeoffsetof_field(self, fieldname, following):
         self.force_lazy_struct()
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -14,6 +14,7 @@
 from pypy.module._cffi_backend import cffi_opcode
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend.cdataobj import W_CData
+from pypy.module._cffi_backend.allocator import W_Allocator, default_allocator
 
 
 ACCEPT_STRING   = 1
@@ -44,6 +45,10 @@
 
 
 class W_FFIObject(W_Root):
+    ACCEPT_STRING = ACCEPT_STRING
+    ACCEPT_CTYPE  = ACCEPT_CTYPE
+    ACCEPT_CDATA  = ACCEPT_CDATA
+
     w_gc_wref_remove = None
 
     @jit.dont_look_inside
@@ -414,7 +419,17 @@
 pointer to the memory somewhere else, e.g. into another structure."""
         #
         w_ctype = self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CTYPE)
-        return w_ctype.newp(w_init)
+        return w_ctype.newp(w_init, default_allocator)
+
+
+    @unwrap_spec(should_clear_after_alloc=int)
+    def descr_new_allocator(self, should_clear_after_alloc=1):
+        """\
+Return a new allocator. Xxx
+        """
+        #
+        alloc = W_Allocator(self, bool(should_clear_after_alloc))
+        return self.space.wrap(alloc)
 
 
     def descr_new_handle(self, w_arg):
@@ -600,6 +615,7 @@
         getctype    = interp2app(W_FFIObject.descr_getctype),
         integer_const = interp2app(W_FFIObject.descr_integer_const),
         new         = interp2app(W_FFIObject.descr_new),
+        new_allocator = interp2app(W_FFIObject.descr_new_allocator),
         new_handle  = interp2app(W_FFIObject.descr_new_handle),
         offsetof    = interp2app(W_FFIObject.descr_offsetof),
         sizeof      = interp2app(W_FFIObject.descr_sizeof),
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,13 +1,21 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.module._cffi_backend import ctypeobj, cdataobj
+from pypy.module._cffi_backend.allocator import W_Allocator, default_allocator
 
 
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_init=WrappedDefault(None))
 def newp(space, w_ctype, w_init):
-    return w_ctype.newp(w_init)
+    return w_ctype.newp(w_init, default_allocator)
+
+# ____________________________________________________________
+
+ at unwrap_spec(should_clear_after_alloc=int)
+def new_allocator(space, should_clear_after_alloc):
+    alloc = W_Allocator(None, bool(should_clear_after_alloc))
+    return space.wrap(alloc)
 
 # ____________________________________________________________
 
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -274,3 +274,25 @@
             import gc
             gc.collect()
         assert seen == [1, 1]
+
+    def test_ffi_new_allocator_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        alloc1 = ffi.new_allocator()
+        alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
+        for retry in range(100):
+            p1 = alloc1("int[10]")
+            p2 = alloc2("int[10]")
+            combination = 0
+            for i in range(10):
+                assert p1[i] == 0
+                combination |= p2[i]
+                p1[i] = -42
+                p2[i] = -43
+            if combination != 0:
+                break
+            del p1, p2
+            import gc; gc.collect()
+        else:
+            raise AssertionError("cannot seem to get an int[10] not "
+                                 "completely cleared")
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -9,6 +9,7 @@
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
+from pypy.module._cffi_backend import allocator
 
 
 class W_FunctionWrapper(W_Root):
@@ -74,7 +75,8 @@
             # then internally allocate the struct and pass a pointer to it as
             # a first argument.
             if locs[0] == 'R':
-                w_result_cdata = ctype.fargs[0].newp(space.w_None)
+                w_result_cdata = ctype.fargs[0].newp(space.w_None,
+                                                    allocator.nonzero_allocator)
                 args_w = [w_result_cdata] + args_w
                 prepare_args(space, rawfunctype, args_w, 1)
                 #
@@ -116,7 +118,7 @@
             # the equivalent of ffi.new()
             if space.is_w(w_arg, space.w_None):
                 continue
-            w_arg = farg.newp(w_arg)
+            w_arg = farg.newp(w_arg, allocator.default_allocator)
         args_w[i] = w_arg
 
 

From noreply at buildbot.pypy.org  Mon Jul  6 10:07:53 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  6 Jul 2015 10:07:53 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: rework short boxes
Message-ID: <20150706080753.C87F71C06AD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78445:eeaaac148347
Date: 2015-07-06 10:07 +0200
http://bitbucket.org/pypy/pypy/changeset/eeaaac148347/

Log:	rework short boxes

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -1,118 +1,51 @@
 
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers
 from rpython.jit.metainterp.history import Const
 
-class BoxNotProducable(Exception):
-    pass
 
 
 class ShortBoxes(object):
     def __init__(self):
-        self.potential_ops = {}
-        self.inputarg_boxes = {}
-        self.synthetic = {}
-        self.alternatives = {}
+        self.potential_ops = []
+        self.ops_used = {}
         self.extra_same_as = []
 
     def create_short_boxes(self, optimizer, inputargs):
         for box in inputargs:
-            self.inputarg_boxes[box] = None
+            self.ops_used[box] = None
         optimizer.produce_potential_short_preamble_ops(self)
 
         self.short_boxes = {}
-        self.short_boxes_in_production = {}
+        # short boxes has a map of "op from preamble" ->
+        # "op going to short preamble", where "op from preamble" can be
+        # anything, but the one going to short_preamble has to be either pure
+        # or a heap cache op
 
-        for op in self.potential_ops.keys():
-            try:
-                self.produce_short_preamble_op(op)
-            except BoxNotProducable:
+        for op, preamble_op in self.potential_ops:
+            self.produce_short_preamble_op(op, preamble_op)
+
+    def add_to_short(self, op, short_op):
+        self.short_boxes[op] = short_op
+
+    def produce_short_preamble_op(self, op, preamble_op):
+        for arg in op.getarglist():
+            if isinstance(arg, Const):
                 pass
+            elif arg in self.ops_used:
+                pass
+            else:
+                return # can't produce
+        if op in self.short_boxes:
+            opnum = OpHelpers.same_as_for_type(op.type)
+            same_as_op = ResOperation(opnum, [op])
+            self.extra_same_as.append(same_as_op)
+            self.add_to_short(same_as_op, preamble_op)
+        else:
+            self.add_to_short(op, preamble_op)
 
-        self.short_boxes_in_production = None # Not needed anymore
-
-    def prioritized_alternatives(self, box):
-        if box not in self.alternatives:
-            return [self.potential_ops[box]]
-        alts = self.alternatives[box]
-        hi, lo = 0, len(alts) - 1
-        while hi < lo:
-            if alts[lo] is None: # Inputarg, lowest priority
-                alts[lo], alts[-1] = alts[-1], alts[lo]
-                lo -= 1
-            elif alts[lo] not in self.synthetic: # Hi priority
-                alts[hi], alts[lo] = alts[lo], alts[hi]
-                hi += 1
-            else: # Low priority
-                lo -= 1
-        return alts
-
-    def add_to_short(self, op):
-        if op in self.short_boxes:
-            xxx
-            #if op is None:
-            #    xxx
-            #    oldop = self.short_boxes[box]
-            #    self.rename[op] = oldop
-            #    self.short_boxes[box] = None
-            #    self.short_boxes[oldop] = oldop
-            #else:
-            #    xxxx
-            #    newop = op.clone()
-            #    newbox = newop.result = op.result.clonebox()
-            #    self.short_boxes[newop.result] = newop
-            #xxx
-            #value = self.optimizer.getvalue(box)
-            #self.optimizer.emit_operation(ResOperation(rop.SAME_AS, [box], newbox))
-            #self.optimizer.make_equal_to(newbox, value)
-            #if op is None:
-            #    if self.short_boxes[box] is not box:
-            #        xxx
-            #else:
-            #    if self.short_boxes[box] is not op:
-            #        if self.short_boxes[box] is None:
-            #            self.short_boxes[box] = op
-            #        else:
-            #            xxx
+    def add_potential(self, op, short_preamble_op=None):
+        if short_preamble_op is None:
+            self.potential_ops.append((op, op))
         else:
-            self.short_boxes[op] = None
-
-    def produce_short_preamble_op(self, op):
-        if op in self.short_boxes:
-            return
-        if op in self.inputarg_boxes:
-            return
-        if isinstance(op, Const):
-            return
-        if op in self.short_boxes_in_production:
-            raise BoxNotProducable
-        self.short_boxes_in_production[op] = None
-
-        if op in self.potential_ops:
-            ops = self.prioritized_alternatives(op)
-            produced_one = False
-            for newop in ops:
-                try:
-                    if newop:
-                        for arg in newop.getarglist():
-                            self.produce_short_preamble_op(arg)
-                except BoxNotProducable:
-                    pass
-                else:
-                    produced_one = True
-                    self.add_to_short(newop)
-            if not produced_one:
-                raise BoxNotProducable
-        else:
-            raise BoxNotProducable
-
-    def add_potential(self, op, result=None, synthetic=False):
-        if result is None:
-            result = op
-        if result in self.potential_ops:
-            if result not in self.alternatives:
-                self.alternatives[result] = [self.potential_ops[result]]
-            self.alternatives[result].append(op)
-        else:
-            self.potential_ops[result] = op
-        if synthetic:
-            self.synthetic[result] = True
+            self.potential_ops.append((op, short_preamble_op))
+        self.ops_used[op] = None
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_short.py b/rpython/jit/metainterp/optimizeopt/test/test_short.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_short.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_short.py
@@ -28,7 +28,7 @@
         op = ResOperation(rop.INT_ADD, [i0, i1])
         sb = ShortBoxes()
         sb.create_short_boxes(Opt([op]), [i0, i1])
-        assert sb.short_boxes == {op: None}
+        assert sb.short_boxes == {op: op}
 
     def test_pure_ops_does_not_work(self):
         i0 = InputArgInt()
@@ -39,10 +39,26 @@
         assert sb.short_boxes == {}
 
     def test_multiple_similar_ops(self):
+        """ This can happen e.g. if heap cache and pure ops produce
+        the same thing. So let's say we have:
+
+        i0 = int_add(i0, 1)
+        setfield_gc(p0, i0)
+
+        now i0 can be gotten in two ways - from getfield or from int_add,
+        we store both in short preamble (in case someone else who inlines
+        the short preamble does not share them)
+        """
         i0 = InputArgInt()
         i1 = InputArgInt()
         op = ResOperation(rop.INT_ADD, [i0, i1])
         op1 = ResOperation(rop.GETFIELD_GC_I, [i0], descr=Descr())
         sb = ShortBoxes()
-        sb.create_short_boxes(Opt([op, (op1, op)]), [i0, i1])
-        
+        sb.create_short_boxes(Opt([op, (op, op1)]), [i0, i1])
+        assert len(sb.short_boxes) == 2
+        l = [x.getopnum() for x in sb.short_boxes.keys()]
+        l.sort()
+        assert l == [rop.INT_ADD, rop.SAME_AS_I]
+        assert op1 in sb.short_boxes.values()
+        assert op in sb.short_boxes.values()
+        assert op in sb.short_boxes.keys()

From noreply at buildbot.pypy.org  Mon Jul  6 10:14:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 10:14:03 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: Non-standard allocators
Message-ID: <20150706081403.C61B41C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78446:ceb59c88b235
Date: 2015-07-06 10:14 +0200
http://bitbucket.org/pypy/pypy/changeset/ceb59c88b235/

Log:	Non-standard allocators

diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -9,19 +9,50 @@
 class W_Allocator(W_Root):
     _immutable_ = True
 
-    def __init__(self, ffi, should_clear_after_alloc):
+    def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc):
         self.ffi = ffi
+        if w_alloc is not None and ffi.space.is_none(w_alloc): w_alloc = None
+        if w_free  is not None and ffi.space.is_none(w_free):  w_free = None
+        self.w_alloc = w_alloc
+        self.w_free = w_free
         self.should_clear_after_alloc = should_clear_after_alloc
 
     def allocate(self, space, datasize, ctype, length=-1):
-        from pypy.module._cffi_backend.cdataobj import W_CDataNewStd
-        if self.should_clear_after_alloc:
-            ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
-                                flavor='raw', zero=True)
+        from pypy.module._cffi_backend import cdataobj, ctypeptr, ffi_obj
+        if self.w_alloc is None:
+            if self.should_clear_after_alloc:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=True)
+            else:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=False)
+            return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
         else:
-            ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
-                                flavor='raw', zero=False)
-        return W_CDataNewStd(space, ptr, ctype, length)
+            w_raw_cdata = space.call_function(self.w_alloc,
+                                              space.wrap(datasize))
+            if not isinstance(w_raw_cdata, cdataobj.W_CData):
+                raise oefmt(ffi_obj.get_ffi_error(space),
+                            "expected cdata object from the call to %R, "
+                            "got '%T'", self.w_alloc, w_raw_cdata)
+            if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray):
+                raise oefmt(ffi_obj.get_ffi_error(space),
+                            "expected cdata pointer from the call to %R, "
+                            "got '%s'", self.w_alloc, w_raw_cdata.ctype.name)
+            #
+            ptr = w_raw_cdata.unsafe_escaping_ptr()
+            if self.should_clear_after_alloc:
+                rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0,
+                              rffi.cast(rffi.SIZE_T, datasize))
+            #
+            if self.w_free is None:
+                # use this class which does not have a __del__, but still
+                # keeps alive w_raw_cdata
+                res = cdataobj.W_CDataNewNonStdNoFree(space, ptr, ctype, length)
+            else:
+                res = cdataobj.W_CDataNewNonStdFree(space, ptr, ctype, length)
+                res.w_free = self.w_free
+            res.w_raw_cdata = w_raw_cdata
+            return res
 
     @unwrap_spec(w_init=WrappedDefault(None))
     def descr_call(self, space, w_arg, w_init):
@@ -44,5 +75,5 @@
 W_Allocator.typedef.acceptable_as_base_class = False
 
 
-default_allocator = W_Allocator(ffi=None, should_clear_after_alloc=True)
-nonzero_allocator = W_Allocator(ffi=None, should_clear_after_alloc=False)
+default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True)
+nonzero_allocator = W_Allocator(None, None, None,should_clear_after_alloc=False)
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -417,11 +417,16 @@
         lltype.free(self._ptr, flavor='raw')
 
 
-class W_CDataMemNonStd(W_CDataNewOwning):
-    """Subclass using a non-standard allocator"""
-    _attrs_ = []
+class W_CDataNewNonStdNoFree(W_CDataNewOwning):
+    """Subclass using a non-standard allocator, no free()"""
+    _attrs_ = ['w_raw_cdata']
 
-    # XXXXXXXXX
+class W_CDataNewNonStdFree(W_CDataNewNonStdNoFree):
+    """Subclass using a non-standard allocator, with a free()"""
+    _attrs_ = ['w_free']
+
+    def __del__(self):
+        self.space.call_function(self.w_free, self.w_raw_cdata)
 
 
 class W_CDataPtrToStructOrUnion(W_CData):
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -422,13 +422,30 @@
         return w_ctype.newp(w_init, default_allocator)
 
 
-    @unwrap_spec(should_clear_after_alloc=int)
-    def descr_new_allocator(self, should_clear_after_alloc=1):
+    @unwrap_spec(w_alloc=WrappedDefault(None),
+                 w_free=WrappedDefault(None),
+                 should_clear_after_alloc=int)
+    def descr_new_allocator(self, w_alloc, w_free,
+                            should_clear_after_alloc=1):
         """\
-Return a new allocator. Xxx
+Return a new allocator.
+
+'alloc' is called with the size as argument.  If it returns NULL, a
+MemoryError is raised.  'free' is called with the result of 'alloc'
+as argument.  Both can be either Python function or directly C
+functions.  If 'free' is explicitly set to None, then no free function
+is called.
+
+If both 'alloc' and 'free' are None, the default is used.
+If 'should_clear_after_alloc' is set to False, then the memory
+returned by 'alloc' is assumed to be already cleared; otherwise
+CFFI will clear it.
         """
         #
-        alloc = W_Allocator(self, bool(should_clear_after_alloc))
+        if self.space.is_none(w_alloc) and not self.space.is_none(w_free):
+            raise oefmt(self.w_TypeError, "cannot give free without alloc")
+        alloc = W_Allocator(self, w_alloc, w_free,
+                            bool(should_clear_after_alloc))
         return self.space.wrap(alloc)
 
 
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -296,3 +296,46 @@
         else:
             raise AssertionError("cannot seem to get an int[10] not "
                                  "completely cleared")
+
+    def test_ffi_new_allocator_2(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        def myfree(raw):
+            seen.append('free')
+        alloc1 = ffi.new_allocator(myalloc, myfree)
+        alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+                                   should_clear_after_alloc=False)
+        p1 = alloc1("int[10]")
+        p2 = alloc2("int[]", 10)
+        assert seen == [40, 40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert ffi.typeof(p2) == ffi.typeof("int[]")
+        assert ffi.sizeof(p2) == 40
+        assert p1[5] == 0
+        assert p2[6] != 0
+        del p1, p2
+        retries = 0
+        while len(seen) != 4:
+            retries += 1
+            assert retries <= 5
+            import gc; gc.collect()
+        assert seen == [40, 40, 'free', 'free']
+
+    def test_ffi_new_allocator_3(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        alloc1 = ffi.new_allocator(myalloc)    # no 'free'
+        p1 = alloc1("int[10]")
+        assert seen == [40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert p1[5] == 0
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1249,3 +1249,8 @@
             lltype.Void,
             releasegil=False
         )
+c_memset = llexternal("memset",
+            [VOIDP, lltype.Signed, SIZE_T],
+            lltype.Void,
+            releasegil=False
+        )

From noreply at buildbot.pypy.org  Mon Jul  6 10:37:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  6 Jul 2015 10:37:03 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start fighting the unrolling -
 step one is calling the short boxes
Message-ID: <20150706083703.F2B5B1C14DC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78447:966c24dd62ad
Date: 2015-07-06 10:37 +0200
http://bitbucket.org/pypy/pypy/changeset/966c24dd62ad/

Log:	start fighting the unrolling - step one is calling the short boxes

diff --git a/rpython/jit/metainterp/optimizeopt/generalize.py b/rpython/jit/metainterp/optimizeopt/generalize.py
deleted file mode 100644
--- a/rpython/jit/metainterp/optimizeopt/generalize.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT
-
-
-class GeneralizationStrategy(object):
-    def __init__(self, optimizer):
-        self.optimizer = optimizer
-
-    def apply(self):
-        raise NotImplementedError
-
-
-class KillHugeIntBounds(GeneralizationStrategy):
-    def apply(self):
-        pass
-        #for v in self.optimizer.values.values():
-        #    if v.is_constant():
-        #        continue
-        #    if isinstance(v, IntOptValue):
-        #        if v.intbound.lower < MININT / 2:
-        #            v.intbound.lower = MININT
-        #        if v.intbound.upper > MAXINT / 2:
-        #            v.intbound.upper = MAXINT
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -218,7 +218,7 @@
         return recentops.lookup(self.optimizer, op)
 
     def produce_potential_short_preamble_ops(self, sb):
-        ops = sb.optimizer._newoperations
+        ops = self.optimizer._newoperations
         for i, op in enumerate(ops):
             if op.is_always_pure():
                 sb.add_potential(op)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -23,6 +23,7 @@
 
         for op, preamble_op in self.potential_ops:
             self.produce_short_preamble_op(op, preamble_op)
+        return self.short_boxes
 
     def add_to_short(self, op, short_op):
         self.short_boxes[op] = short_op
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_short.py b/rpython/jit/metainterp/optimizeopt/test/test_short.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_short.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_short.py
@@ -62,3 +62,4 @@
         assert op1 in sb.short_boxes.values()
         assert op in sb.short_boxes.values()
         assert op in sb.short_boxes.keys()
+
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -13,7 +13,7 @@
 class FakeOptimizer(object):
     optearlyforce = None
      
-class TestUnroll(BaseTest, LLtypeMixin):
+class BaseTestUnroll(BaseTest, LLtypeMixin):
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
     
     def optimize(self, ops):
@@ -38,8 +38,9 @@
         start_state = self._do_optimize_loop(preamble, None,
                                              export_state=True)
         vs = preamble.operations[-1].getdescr().virtual_state
-        return start_state, vs, loop
+        return start_state, vs, loop, preamble
 
+class TestUnroll(BaseTestUnroll):
     def test_simple(self):
         loop = """
         [i0]
@@ -47,7 +48,7 @@
         guard_value(i1, 1) []
         jump(i1)
         """
-        es, vs, loop = self.optimize(loop)
+        es, vs, loop, preamble = self.optimize(loop)
         assert isinstance(vs.state[0], NotVirtualStateInfo)
         # the virtual state is constant, so we don't need to have it in
         # inputargs
@@ -56,6 +57,7 @@
         # we have exported values for i1, which happens to be an inputarg
         assert es.inputarg_mapping[0][1].getint() == 1
         assert isinstance(es.inputarg_mapping[0][1], ConstInt)
+        assert es.short_boxes == {}
 
     def test_not_constant(self):
         loop = """
@@ -63,6 +65,8 @@
         i1 = int_add(i0, 1)
         jump(i0)
         """
-        es, vs, loop = self.optimize(loop)
+        es, vs, loop, preamble = self.optimize(loop)
         assert isinstance(vs.state[0], NotVirtualStateInfo)
         assert vs.state[0].level == LEVEL_UNKNOWN
+        op = preamble.operations[1]
+        assert es.short_boxes == {op: op}
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,9 +1,8 @@
 import sys
 
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
-from rpython.jit.metainterp.logger import LogOperations
+from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes
 from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
      Optimization
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
@@ -165,8 +164,6 @@
             self.close_bridge(start_label)
 
         self.optimizer.flush()
-        if export_state:
-            KillHugeIntBounds(self.optimizer).apply()
 
         loop.operations = self.optimizer.get_newoperations()
         if export_state:
@@ -211,8 +208,11 @@
         target_token = targetop.getdescr()
         assert isinstance(target_token, TargetToken)
         target_token.virtual_state = virtual_state
-        return ExportedState([(label_op.getarg(i), jump_args[i]) for i in range(len(jump_args))], [], [], [])
-        xxx
+        sb = ShortBoxes()
+        sb.create_short_boxes(self.optimizer, jump_args)
+        inparg_mapping = [(label_op.getarg(i), jump_args[i])
+                          for i in range(len(jump_args))]
+        return ExportedState(inparg_mapping, [], sb.short_boxes)
 
 
         inputargs = virtual_state.make_inputargs(jump_args, self.optimizer)
@@ -298,14 +298,11 @@
         #for source, target in exported_state.inputarg_setup_ops:
         #    source.set_forwarded(target)
 
-        #for op in self.short_boxes.operations():
-        #    if not op:
-        #        continue
-        #    if op.is_always_pure():
-        #        self.pure(op.getopnum(),
-        #                  PreambleOp(op, self.optimizer.getinfo(op)))
-        #    else:
-        #        yyy
+        for op, preamble_op in exported_state.short_boxes.iteritems():
+            if preamble_op.is_always_pure():
+                self.pure(op.getopnum(), PreambleOp(op, None))
+            else:
+                xxx
         return
         seen = {}
         for op in self.short_boxes.operations():
@@ -336,8 +333,8 @@
                     #    debug_print('  Falling back to add extra: ' +
                     #                self.optimizer.loop.logops.repr_of_resop(op))
 
-        self.optimizer.flush()
-        self.optimizer.emitting_dissabled = False
+        #self.optimizer.flush()
+        #self.optimizer.emitting_dissabled = False
 
     def close_bridge(self, start_label):
         inputargs = self.inputargs
@@ -709,28 +706,10 @@
                          as the first element and the second element being
                          what it maps to (potentially const)
     * exported_infos - a mapping from ops to infos, including inputargs
-    * pure operations - a list of pure operations that can produce various
-                        ops
-    * heap cache operations - an additional list of how to produce various
-                              ops
+    * short boxes - a mapping op -> preamble_op
     """
     
-    def __init__(self, inputarg_mapping, exported_infos, pure_ops,
-                 heap_cache_ops):
+    def __init__(self, inputarg_mapping, exported_infos, short_boxes):
         self.inputarg_mapping = inputarg_mapping
         self.exported_infos = exported_infos
-        self.pure_ops = pure_ops
-        self.heap_cache_ops = heap_cache_ops
-
-    def dump(self, metainterp_sd):
-        debug_start("jit-exported-state")
-        logops = LogOperations(metainterp_sd, False)
-        debug_print("   inputarg setup")
-        logops._log_operations([], self.inputarg_setup_ops)
-        debug_print("   short boxes")
-        self.short_boxes.dump(logops)
-        debug_print("   exported values")
-        for k, v in self.exported_values.iteritems():
-            debug_print("%s:%s" % (logops.repr_of_resop(k),
-                                   logops.repr_of_resop(v.box)))
-        debug_stop("jit-exported-state")
+        self.short_boxes = short_boxes

From noreply at buildbot.pypy.org  Mon Jul  6 10:41:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 10:41:07 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: The in-line mode,
 internally _cffi_backend.newp_allocator()
Message-ID: <20150706084107.D36331C14DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78448:30a33ea339da
Date: 2015-07-06 10:41 +0200
http://bitbucket.org/pypy/pypy/changeset/30a33ea339da/

Log:	The in-line mode, internally _cffi_backend.newp_allocator()

diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -25,6 +25,7 @@
         'new_function_type': 'newtype.new_function_type',
 
         'newp': 'func.newp',
+        'newp_allocator': 'func.newp_allocator',
         'cast': 'func.cast',
         'callback': 'func.callback',
         'alignof': 'func.alignof',
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -10,9 +10,7 @@
     _immutable_ = True
 
     def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc):
-        self.ffi = ffi
-        if w_alloc is not None and ffi.space.is_none(w_alloc): w_alloc = None
-        if w_free  is not None and ffi.space.is_none(w_free):  w_free = None
+        self.ffi = ffi    # may be None
         self.w_alloc = w_alloc
         self.w_free = w_free
         self.should_clear_after_alloc = should_clear_after_alloc
@@ -75,5 +73,16 @@
 W_Allocator.typedef.acceptable_as_base_class = False
 
 
+def new_allocator(space, ffi, w_alloc, w_free, should_clear_after_alloc):
+    if space.is_none(w_alloc):
+        w_alloc = None
+    if space.is_none(w_free):
+        w_free = None
+    if w_alloc is None and w_free is not None:
+        raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'")
+    alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc))
+    return space.wrap(alloc)
+
+
 default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True)
 nonzero_allocator = W_Allocator(None, None, None,should_clear_after_alloc=False)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -11,10 +11,9 @@
 from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray
 from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle
 from pypy.module._cffi_backend import cbuffer, func, cgc, wrapper
-from pypy.module._cffi_backend import cffi_opcode
+from pypy.module._cffi_backend import cffi_opcode, allocator
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend.cdataobj import W_CData
-from pypy.module._cffi_backend.allocator import W_Allocator, default_allocator
 
 
 ACCEPT_STRING   = 1
@@ -419,7 +418,7 @@
 pointer to the memory somewhere else, e.g. into another structure."""
         #
         w_ctype = self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CTYPE)
-        return w_ctype.newp(w_init, default_allocator)
+        return w_ctype.newp(w_init, allocator.default_allocator)
 
 
     @unwrap_spec(w_alloc=WrappedDefault(None),
@@ -442,11 +441,8 @@
 CFFI will clear it.
         """
         #
-        if self.space.is_none(w_alloc) and not self.space.is_none(w_free):
-            raise oefmt(self.w_TypeError, "cannot give free without alloc")
-        alloc = W_Allocator(self, w_alloc, w_free,
-                            bool(should_clear_after_alloc))
-        return self.space.wrap(alloc)
+        return allocator.new_allocator(self.space, self, w_alloc, w_free,
+                                       should_clear_after_alloc)
 
 
     def descr_new_handle(self, w_arg):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,21 +1,20 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.module._cffi_backend import ctypeobj, cdataobj
-from pypy.module._cffi_backend.allocator import W_Allocator, default_allocator
+from pypy.module._cffi_backend import ctypeobj, cdataobj, allocator
 
 
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_init=WrappedDefault(None))
 def newp(space, w_ctype, w_init):
-    return w_ctype.newp(w_init, default_allocator)
+    return w_ctype.newp(w_init, allocator.default_allocator)
 
 # ____________________________________________________________
 
 @unwrap_spec(should_clear_after_alloc=int)
-def new_allocator(space, should_clear_after_alloc):
-    alloc = W_Allocator(None, bool(should_clear_after_alloc))
-    return space.wrap(alloc)
+def newp_allocator(space, w_alloc, w_free, should_clear_after_alloc):
+    return allocator.new_allocator(space, None, w_alloc, w_free,
+                                   should_clear_after_alloc)
 
 # ____________________________________________________________
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3402,6 +3402,37 @@
     py.test.raises(RuntimeError, "p[42]")
     py.test.raises(RuntimeError, "p[42] = -1")
 
+def test_newp_allocator():
+    BChar = new_primitive_type("char")
+    BCharPtr = new_pointer_type(BChar)
+    BCharArray = new_array_type(BCharPtr, None)
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    seen = []
+    def myalloc(size):
+        seen.append(size)
+        return newp(BCharArray, "#" * size)
+    def myfree(raw):
+        seen.append(raw)
+    alloc1 = newp_allocator(None, None, True)
+    alloc2 = newp_allocator(myalloc, myfree, False)
+    p1 = alloc1(BIntPtr)
+    p2 = alloc2(BIntPtr)
+    assert typeof(p1) is BIntPtr
+    assert typeof(p2) is BIntPtr
+    assert p1[0] == 0
+    assert p2[0] == ord('#') * 0x01010101
+    assert seen == [sizeof(BInt)]
+    raw2 = cast(BCharPtr, p2)
+    del p1, p2
+    retries = 0
+    while len(seen) != 2:
+        retries += 1
+        assert retries <= 5
+        import gc; gc.collect()
+    assert seen == [sizeof(BInt), raw2]
+    assert repr(seen[1]) == "<cdata 'char[]' owning 5 bytes>"
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "1.2.0"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -317,7 +317,7 @@
         assert ffi.typeof(p2) == ffi.typeof("int[]")
         assert ffi.sizeof(p2) == 40
         assert p1[5] == 0
-        assert p2[6] != 0
+        assert p2[6] == ord('X') * 0x01010101
         del p1, p2
         retries = 0
         while len(seen) != 4:

From noreply at buildbot.pypy.org  Mon Jul  6 11:07:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 11:07:11 +0200 (CEST)
Subject: [pypy-commit] cffi default: Found a simpler way to implement the
 in-line ffi.gc() on top of the
Message-ID: <20150706090711.5CA981C1C9C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2211:88b59dfa8176
Date: 2015-07-06 11:07 +0200
http://bitbucket.org/cffi/cffi/changeset/88b59dfa8176/

Log:	Found a simpler way to implement the in-line ffi.gc() on top of the
	out-of-line one

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5554,9 +5554,6 @@
                            (PyObject *)&CTypeDescr_Type);
 }
 
-/* forward: implemented in ffi_obj.c */
-static PyObject *b_gcp(PyObject *self, PyObject *args);
-
 /************************************************************/
 
 static char _testfunc0(char a, char b)
@@ -5862,7 +5859,6 @@
     {"newp_handle", b_newp_handle, METH_VARARGS},
     {"from_handle", b_from_handle, METH_O},
     {"from_buffer", b_from_buffer, METH_VARARGS},
-    {"gcp", b_gcp, METH_VARARGS},
 #ifdef MS_WIN32
     {"getwinerror", (PyCFunction)b_getwinerror, METH_VARARGS | METH_KEYWORDS},
 #endif
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -680,19 +680,6 @@
     return gc_weakrefs_build(self, cd, destructor);
 }
 
-static PyObject *b_gcp(PyObject *self, PyObject *args)
-{
-    /* for in-line mode */
-    static FFIObject *ffi1 = NULL;
-
-    if (ffi1 == NULL) {
-        ffi1 = ffi_internal_new(&FFI_Type, NULL);
-        if (ffi1 == NULL)
-            return NULL;
-    }
-    return ffi_gc(ffi1, args, NULL);
-}
-
 PyDoc_STRVAR(ffi_callback_doc,
 "Return a callback object or a decorator making such a callback object.\n"
 "'cdecl' must name a C function pointer type.  The callback invokes the\n"
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -72,6 +72,7 @@
         self._cdefsources = []
         self._included_ffis = []
         self._windows_unicode = None
+        self._gcp = None
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
         for name in backend.__dict__:
@@ -328,13 +329,14 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
-        try:
-            gcp = self._backend.gcp
-        except AttributeError:
-            pass
-        else:
-            return gcp(cdata, destructor)
+        if self._gcp is not None:
+            return self._gcp(cdata, destructor)
+        if hasattr(self._backend, 'FFI'):
+            compiled_ffi = self._backend.FFI()
+            self._gcp = compiled_ffi.gc
+            return self._gcp(cdata, destructor)
         #
+        # the rest is for the ctypes backend only
         with self._lock:
             try:
                 gc_weakrefs = self.gc_weakrefs

From noreply at buildbot.pypy.org  Mon Jul  6 11:08:51 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 11:08:51 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: Kill gcp() again
Message-ID: <20150706090851.DFBBF1C1C9C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78449:70f788fd9dca
Date: 2015-07-06 11:06 +0200
http://bitbucket.org/pypy/pypy/changeset/70f788fd9dca/

Log:	Kill gcp() again

diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -38,7 +38,6 @@
         'from_handle': 'handle.from_handle',
         '_get_types': 'func._get_types',
         'from_buffer': 'func.from_buffer',
-        'gcp': 'func.gcp',
 
         'string': 'func.string',
         'buffer': 'cbuffer.buffer',
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -112,18 +112,3 @@
                     "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
-
-# ____________________________________________________________
-
-class ConstantFFI:
-    ffi1 = None
-    def _cleanup_(self):
-        self.ffi1 = None
-constant_ffi = ConstantFFI()
-
- at unwrap_spec(w_cdata=cdataobj.W_CData)
-def gcp(space, w_cdata, w_destructor):
-    if constant_ffi.ffi1 is None:
-        from pypy.module._cffi_backend import ffi_obj
-        constant_ffi.ffi1 = ffi_obj.make_plain_ffi_object(space)
-    return constant_ffi.ffi1.descr_gc(w_cdata, w_destructor)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -4,8 +4,6 @@
     spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
 
     def teardown_method(self, meth):
-        from pypy.module._cffi_backend.func import constant_ffi
-        constant_ffi._cleanup_()
         _clean_cache(self.space)
 
     def test_ffi_new(self):
@@ -267,13 +265,12 @@
             assert p1[0] == 123
             seen.append(1)
         ffi.gc(p, destructor=destructor)    # instantly forgotten
-        _cffi1_backend.gcp(p, destructor=destructor)
         for i in range(5):
             if seen:
                 break
             import gc
             gc.collect()
-        assert seen == [1, 1]
+        assert seen == [1]
 
     def test_ffi_new_allocator_1(self):
         import _cffi_backend as _cffi1_backend

From noreply at buildbot.pypy.org  Mon Jul  6 12:09:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 12:09:30 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: Kill again newp_allocator()
Message-ID: <20150706100930.BD2B01C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78450:2094c5d180ef
Date: 2015-07-06 11:19 +0200
http://bitbucket.org/pypy/pypy/changeset/2094c5d180ef/

Log:	Kill again newp_allocator()

diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -25,7 +25,6 @@
         'new_function_type': 'newtype.new_function_type',
 
         'newp': 'func.newp',
-        'newp_allocator': 'func.newp_allocator',
         'cast': 'func.cast',
         'callback': 'func.callback',
         'alignof': 'func.alignof',
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -54,15 +54,9 @@
 
     @unwrap_spec(w_init=WrappedDefault(None))
     def descr_call(self, space, w_arg, w_init):
-        from pypy.module._cffi_backend.ctypeobj import W_CType
-        if isinstance(w_arg, W_CType):
-            w_ctype = w_arg
-        else:
-            ffi = self.ffi
-            if ffi is None:
-                raise oefmt(space.w_TypeError,
-                            "expected a ctype object, got '%T'", w_arg)
-            w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING)
+        ffi = self.ffi
+        assert ffi is not None
+        w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING | ffi.ACCEPT_CTYPE)
         return w_ctype.newp(w_init, self)
 
 
@@ -73,7 +67,8 @@
 W_Allocator.typedef.acceptable_as_base_class = False
 
 
-def new_allocator(space, ffi, w_alloc, w_free, should_clear_after_alloc):
+def new_allocator(ffi, w_alloc, w_free, should_clear_after_alloc):
+    space = ffi.space
     if space.is_none(w_alloc):
         w_alloc = None
     if space.is_none(w_free):
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -437,11 +437,11 @@
 
 If both 'alloc' and 'free' are None, the default is used.
 If 'should_clear_after_alloc' is set to False, then the memory
-returned by 'alloc' is assumed to be already cleared; otherwise
-CFFI will clear it.
+returned by 'alloc' is assumed to be already cleared (or you are
+fine with garbage); otherwise CFFI will clear it.
         """
         #
-        return allocator.new_allocator(self.space, self, w_alloc, w_free,
+        return allocator.new_allocator(self, w_alloc, w_free,
                                        should_clear_after_alloc)
 
 
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -11,13 +11,6 @@
 
 # ____________________________________________________________
 
- at unwrap_spec(should_clear_after_alloc=int)
-def newp_allocator(space, w_alloc, w_free, should_clear_after_alloc):
-    return allocator.new_allocator(space, None, w_alloc, w_free,
-                                   should_clear_after_alloc)
-
-# ____________________________________________________________
-
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
 def cast(space, w_ctype, w_ob):
     return w_ctype.cast(w_ob)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3402,37 +3402,6 @@
     py.test.raises(RuntimeError, "p[42]")
     py.test.raises(RuntimeError, "p[42] = -1")
 
-def test_newp_allocator():
-    BChar = new_primitive_type("char")
-    BCharPtr = new_pointer_type(BChar)
-    BCharArray = new_array_type(BCharPtr, None)
-    BInt = new_primitive_type("int")
-    BIntPtr = new_pointer_type(BInt)
-    seen = []
-    def myalloc(size):
-        seen.append(size)
-        return newp(BCharArray, "#" * size)
-    def myfree(raw):
-        seen.append(raw)
-    alloc1 = newp_allocator(None, None, True)
-    alloc2 = newp_allocator(myalloc, myfree, False)
-    p1 = alloc1(BIntPtr)
-    p2 = alloc2(BIntPtr)
-    assert typeof(p1) is BIntPtr
-    assert typeof(p2) is BIntPtr
-    assert p1[0] == 0
-    assert p2[0] == ord('#') * 0x01010101
-    assert seen == [sizeof(BInt)]
-    raw2 = cast(BCharPtr, p2)
-    del p1, p2
-    retries = 0
-    while len(seen) != 2:
-        retries += 1
-        assert retries <= 5
-        import gc; gc.collect()
-    assert seen == [sizeof(BInt), raw2]
-    assert repr(seen[1]) == "<cdata 'char[]' owning 5 bytes>"
-
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "1.2.0"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -302,7 +302,7 @@
             seen.append(size)
             return ffi.new("char[]", "X" * size)
         def myfree(raw):
-            seen.append('free')
+            seen.append(raw)
         alloc1 = ffi.new_allocator(myalloc, myfree)
         alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
                                    should_clear_after_alloc=False)
@@ -315,13 +315,17 @@
         assert ffi.sizeof(p2) == 40
         assert p1[5] == 0
         assert p2[6] == ord('X') * 0x01010101
+        raw1 = ffi.cast("char *", p1)
+        raw2 = ffi.cast("char *", p2)
         del p1, p2
         retries = 0
         while len(seen) != 4:
             retries += 1
             assert retries <= 5
             import gc; gc.collect()
-        assert seen == [40, 40, 'free', 'free']
+        assert seen == [40, 40, raw1, raw2]
+        assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+        assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
 
     def test_ffi_new_allocator_3(self):
         import _cffi_backend as _cffi1_backend

From noreply at buildbot.pypy.org  Mon Jul  6 12:09:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 12:09:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill gcp() again
Message-ID: <20150706100931.E494D1C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78451:94805f64e621
Date: 2015-07-06 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/94805f64e621/

Log:	Kill gcp() again

diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -37,7 +37,6 @@
         'from_handle': 'handle.from_handle',
         '_get_types': 'func._get_types',
         'from_buffer': 'func.from_buffer',
-        'gcp': 'func.gcp',
 
         'string': 'func.string',
         'buffer': 'cbuffer.buffer',
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -105,18 +105,3 @@
                     "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
-
-# ____________________________________________________________
-
-class ConstantFFI:
-    ffi1 = None
-    def _cleanup_(self):
-        self.ffi1 = None
-constant_ffi = ConstantFFI()
-
- at unwrap_spec(w_cdata=cdataobj.W_CData)
-def gcp(space, w_cdata, w_destructor):
-    if constant_ffi.ffi1 is None:
-        from pypy.module._cffi_backend import ffi_obj
-        constant_ffi.ffi1 = ffi_obj.make_plain_ffi_object(space)
-    return constant_ffi.ffi1.descr_gc(w_cdata, w_destructor)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -4,8 +4,6 @@
     spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
 
     def teardown_method(self, meth):
-        from pypy.module._cffi_backend.func import constant_ffi
-        constant_ffi._cleanup_()
         _clean_cache(self.space)
 
     def test_ffi_new(self):
@@ -267,10 +265,9 @@
             assert p1[0] == 123
             seen.append(1)
         ffi.gc(p, destructor=destructor)    # instantly forgotten
-        _cffi1_backend.gcp(p, destructor=destructor)
         for i in range(5):
             if seen:
                 break
             import gc
             gc.collect()
-        assert seen == [1, 1]
+        assert seen == [1]

From noreply at buildbot.pypy.org  Mon Jul  6 12:20:41 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  6 Jul 2015 12:20:41 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: hg merge default
Message-ID: <20150706102041.C97571C0207@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78452:a93da01911d0
Date: 2015-07-06 10:58 +0100
http://bitbucket.org/pypy/pypy/changeset/a93da01911d0/

Log:	hg merge default

diff too long, truncating to 2000 out of 17394 lines

diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -528,12 +528,13 @@
 # result, the parsing rules here are less strict.
 #
 
-_LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_LegalKeyChars  = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
+_LegalValueChars = _LegalKeyChars + r"\[\]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
     r"\s*"                        # Optional whitespace at start of cookie
     r"(?P<key>"                   # Start of group 'key'
-    ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
+    "["+ _LegalKeyChars +"]+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
     r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
@@ -542,7 +543,7 @@
     r"|"                            # or
     r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
     r"|"                            # or
-    ""+ _LegalCharsPatt +"*"        # Any word or empty string
+    "["+ _LegalValueChars +"]*"        # Any word or empty string
     r")"                          # End of group 'val'
     r")?"                         # End of optional value group
     r"\s*"                        # Any number of spaces.
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -14,6 +14,7 @@
 import posixpath
 import BaseHTTPServer
 import urllib
+import urlparse
 import cgi
 import sys
 import shutil
@@ -68,10 +69,14 @@
         path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
-            if not self.path.endswith('/'):
+            parts = urlparse.urlsplit(self.path)
+            if not parts.path.endswith('/'):
                 # redirect browser - doing basically what apache does
                 self.send_response(301)
-                self.send_header("Location", self.path + "/")
+                new_parts = (parts[0], parts[1], parts[2] + '/',
+                             parts[3], parts[4])
+                new_url = urlparse.urlunsplit(new_parts)
+                self.send_header("Location", new_url)
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py
--- a/lib-python/2.7/_LWPCookieJar.py
+++ b/lib-python/2.7/_LWPCookieJar.py
@@ -18,7 +18,7 @@
                        iso2time, time2isoz)
 
 def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in an the LWP cookie file format.
+    """Return string representation of Cookie in the LWP cookie file format.
 
     Actually, the format is extended a bit -- see module docstring.
 
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -548,23 +548,25 @@
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if len(args) > 2:
-            raise TypeError("update() takes at most 2 positional "
-                            "arguments ({} given)".format(len(args)))
-        elif not args:
-            raise TypeError("update() takes at least 1 argument (0 given)")
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
         self = args[0]
-        other = args[1] if len(args) >= 2 else ()
-
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            if isinstance(other, Mapping):
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -25,8 +25,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
-# defined in io.py. We don't use real inheritance though, because we don't
-# want to inherit the C implementations.
+# defined in io.py. We don't use real inheritance though, because we don't want
+# to inherit the C implementations.
 
 
 class BlockingIOError(IOError):
@@ -775,7 +775,7 @@
         clsname = self.__class__.__name__
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.{0}>".format(clsname)
         else:
             return "<_pyio.{0} name={1!r}>".format(clsname, name)
@@ -1216,8 +1216,10 @@
         return self.writer.flush()
 
     def close(self):
-        self.writer.close()
-        self.reader.close()
+        try:
+            self.writer.close()
+        finally:
+            self.reader.close()
 
     def isatty(self):
         return self.reader.isatty() or self.writer.isatty()
@@ -1538,7 +1540,7 @@
     def __repr__(self):
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
             return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -335,9 +335,9 @@
     # though
     week_of_year = -1
     week_of_year_start = -1
-    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # weekday and julian defaulted to None so as to signal need to calculate
     # values
-    weekday = julian = -1
+    weekday = julian = None
     found_dict = found.groupdict()
     for group_key in found_dict.iterkeys():
         # Directives not explicitly handled below:
@@ -434,14 +434,14 @@
         year = 1900
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian == -1 and week_of_year != -1 and weekday != -1:
+    if julian is None and week_of_year != -1 and weekday is not None:
         week_starts_Mon = True if week_of_year_start == 0 else False
         julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
                                             week_starts_Mon)
     # Cannot pre-calculate datetime_date() since can change in Julian
     # calculation and thus could have different value for the day of the week
     # calculation.
-    if julian == -1:
+    if julian is None:
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
@@ -451,7 +451,7 @@
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
-    if weekday == -1:
+    if weekday is None:
         weekday = datetime_date(year, month, day).weekday()
     if leap_year_fix:
         # the caller didn't supply a year but asked for Feb 29th. We couldn't
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -357,10 +357,13 @@
         self._soundpos = 0
 
     def close(self):
-        if self._decomp:
-            self._decomp.CloseDecompressor()
-            self._decomp = None
-        self._file.close()
+        decomp = self._decomp
+        try:
+            if decomp:
+                self._decomp = None
+                decomp.CloseDecompressor()
+        finally:
+            self._file.close()
 
     def tell(self):
         return self._soundpos
diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py
--- a/lib-python/2.7/binhex.py
+++ b/lib-python/2.7/binhex.py
@@ -32,7 +32,8 @@
     pass
 
 # States (what have we written)
-[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3)
+_DID_HEADER = 0
+_DID_DATA = 1
 
 # Various constants
 REASONABLY_LARGE=32768  # Minimal amount we pass the rle-coder
@@ -235,17 +236,22 @@
         self._write(data)
 
     def close(self):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error, 'Close at the wrong time'
-        if self.rlen != 0:
-            raise Error, \
-                  "Incorrect resource-datasize, diff=%r" % (self.rlen,)
-        self._writecrc()
-        self.ofp.close()
-        self.state = None
-        del self.ofp
+        if self.state is None:
+            return
+        try:
+            if self.state < _DID_DATA:
+                self.close_data()
+            if self.state != _DID_DATA:
+                raise Error, 'Close at the wrong time'
+            if self.rlen != 0:
+                raise Error, \
+                    "Incorrect resource-datasize, diff=%r" % (self.rlen,)
+            self._writecrc()
+        finally:
+            self.state = None
+            ofp = self.ofp
+            del self.ofp
+            ofp.close()
 
 def binhex(inp, out):
     """(infilename, outfilename) - Create binhex-encoded copy of a file"""
@@ -463,11 +469,15 @@
         return self._read(n)
 
     def close(self):
-        if self.rlen:
-            dummy = self.read_rsrc(self.rlen)
-        self._checkcrc()
-        self.state = _DID_RSRC
-        self.ifp.close()
+        if self.state is None:
+            return
+        try:
+            if self.rlen:
+                dummy = self.read_rsrc(self.rlen)
+            self._checkcrc()
+        finally:
+            self.state = None
+            self.ifp.close()
 
 def hexbin(inp, out):
     """(infilename, outfilename) - Decode binhexed file"""
diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py
--- a/lib-python/2.7/bsddb/test/test_all.py
+++ b/lib-python/2.7/bsddb/test/test_all.py
@@ -412,9 +412,6 @@
         def get_dbp(self) :
             return self._db
 
-    import string
-    string.letters=[chr(i) for i in xrange(65,91)]
-
     bsddb._db.DBEnv_orig = bsddb._db.DBEnv
     bsddb._db.DB_orig = bsddb._db.DB
     if bsddb.db.version() <= (4, 3) :
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -999,7 +999,7 @@
         for x in "The quick brown fox jumped over the lazy dog".split():
             d2.put(x, self.makeData(x))
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d3.put(x, x*70)
 
         d1.sync()
@@ -1047,7 +1047,7 @@
             if verbose:
                 print rec
             rec = c3.next()
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
 
         c1.close()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -59,7 +59,7 @@
             return bytes(key, "iso8859-1")  # 8 bits
 
     def populateDB(self, d):
-        for x in string.letters:
+        for x in string.ascii_letters:
             d[self.mk('S' + x)] = 10 * x           # add a string
             d[self.mk('I' + x)] = ord(x)           # add an integer
             d[self.mk('L' + x)] = [x] * 10         # add a list
diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py
--- a/lib-python/2.7/bsddb/test/test_get_none.py
+++ b/lib-python/2.7/bsddb/test/test_get_none.py
@@ -26,14 +26,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(1)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         data = d.get('bad key')
         self.assertEqual(data, None)
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         c = d.cursor()
@@ -43,7 +43,7 @@
             rec = c.next()
 
         self.assertEqual(rec, None)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
@@ -54,14 +54,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(0)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         self.assertRaises(db.DBNotFoundError, d.get, 'bad key')
         self.assertRaises(KeyError, d.get, 'bad key')
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         exceptionHappened = 0
@@ -77,7 +77,7 @@
 
         self.assertNotEqual(rec, None)
         self.assertTrue(exceptionHappened)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,7 +10,6 @@
 
 #----------------------------------------------------------------------
 
- at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
@@ -37,17 +36,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
@@ -108,17 +107,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -4,12 +4,11 @@
 import os, sys
 import errno
 from pprint import pprint
+import string
 import unittest
 
 from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path
 
-letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
 
 #----------------------------------------------------------------------
 
@@ -39,7 +38,7 @@
 
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             recno = d.append(x * 60)
             self.assertIsInstance(recno, int)
             self.assertGreaterEqual(recno, 1)
@@ -270,7 +269,7 @@
         d.set_re_pad(45)  # ...test both int and char
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             d.append(x * 35)    # These will be padded
 
         d.append('.' * 40)      # this one will be exact
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -85,8 +85,10 @@
 
     def close(self):
         if not self.closed:
-            self.skip()
-            self.closed = True
+            try:
+                self.skip()
+            finally:
+                self.closed = True
 
     def isatty(self):
         if self.closed:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -20,8 +20,14 @@
            "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
            "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
+           "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder",
+           "StreamReader", "StreamWriter",
+           "StreamReaderWriter", "StreamRecoder",
+           "getencoder", "getdecoder", "getincrementalencoder",
+           "getincrementaldecoder", "getreader", "getwriter",
+           "encode", "decode", "iterencode", "iterdecode",
            "strict_errors", "ignore_errors", "replace_errors",
-           "xmlcharrefreplace_errors",
+           "xmlcharrefreplace_errors", "backslashreplace_errors",
            "register_error", "lookup_error"]
 
 ### Constants
@@ -1051,7 +1057,7 @@
         during translation.
 
         One example where this happens is cp875.py which decodes
-        multiple character to \u001a.
+        multiple character to \\u001a.
 
     """
     m = {}
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -330,7 +330,7 @@
     #   http://code.activestate.com/recipes/259174/
     #   Knuth, TAOCP Vol. II section 4.6.3
 
-    def __init__(self, iterable=None, **kwds):
+    def __init__(*args, **kwds):
         '''Create a new, empty Counter object.  And if given, count elements
         from an input iterable.  Or, initialize the count from another mapping
         of elements to their counts.
@@ -341,8 +341,15 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
         super(Counter, self).__init__()
-        self.update(iterable, **kwds)
+        self.update(*args, **kwds)
 
     def __missing__(self, key):
         'The count of elements not in the Counter is zero.'
@@ -393,7 +400,7 @@
         raise NotImplementedError(
             'Counter.fromkeys() is undefined.  Use Counter(iterable) instead.')
 
-    def update(self, iterable=None, **kwds):
+    def update(*args, **kwds):
         '''Like dict.update() but add counts instead of replacing them.
 
         Source can be an iterable, a dictionary, or another Counter instance.
@@ -413,6 +420,14 @@
         # contexts.  Instead, we implement straight-addition.  Both the inputs
         # and outputs are allowed to contain zero and negative counts.
 
+        if not args:
+            raise TypeError("descriptor 'update' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             if isinstance(iterable, Mapping):
                 if self:
@@ -428,7 +443,7 @@
         if kwds:
             self.update(kwds)
 
-    def subtract(self, iterable=None, **kwds):
+    def subtract(*args, **kwds):
         '''Like dict.update() but subtracts counts instead of replacing them.
         Counts can be reduced below zero.  Both the inputs and outputs are
         allowed to contain zero and negative counts.
@@ -444,6 +459,14 @@
         -1
 
         '''
+        if not args:
+            raise TypeError("descriptor 'subtract' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             self_get = self.get
             if isinstance(iterable, Mapping):
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -464,26 +464,42 @@
     for ns_header in ns_headers:
         pairs = []
         version_set = False
-        for ii, param in enumerate(re.split(r";\s*", ns_header)):
-            param = param.rstrip()
-            if param == "": continue
-            if "=" not in param:
-                k, v = param, None
-            else:
-                k, v = re.split(r"\s*=\s*", param, 1)
-                k = k.lstrip()
+
+        # XXX: The following does not strictly adhere to RFCs in that empty
+        # names and values are legal (the former will only appear once and will
+        # be overwritten if multiple occurrences are present). This is
+        # mostly to deal with backwards compatibility.
+        for ii, param in enumerate(ns_header.split(';')):
+            param = param.strip()
+
+            key, sep, val = param.partition('=')
+            key = key.strip()
+
+            if not key:
+                if ii == 0:
+                    break
+                else:
+                    continue
+
+            # allow for a distinction between present and empty and missing
+            # altogether
+            val = val.strip() if sep else None
+
             if ii != 0:
-                lc = k.lower()
+                lc = key.lower()
                 if lc in known_attrs:
-                    k = lc
-                if k == "version":
+                    key = lc
+
+                if key == "version":
                     # This is an RFC 2109 cookie.
-                    v = _strip_quotes(v)
+                    if val is not None:
+                        val = _strip_quotes(val)
                     version_set = True
-                if k == "expires":
+                elif key == "expires":
                     # convert expires date to seconds since epoch
-                    v = http2time(_strip_quotes(v))  # None if invalid
-            pairs.append((k, v))
+                    if val is not None:
+                        val = http2time(_strip_quotes(val))  # None if invalid
+            pairs.append((key, val))
 
         if pairs:
             if not version_set:
diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
--- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
+++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
@@ -1,1 +1,1 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -32,15 +32,24 @@
     def setUp(self):
         self.gl = self.glu = self.gle = None
         if lib_gl:
-            self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            try:
+                self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_glu:
-            self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            try:
+                self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_gle:
             try:
                 self.gle = CDLL(lib_gle)
             except OSError:
                 pass
 
+    def tearDown(self):
+        self.gl = self.glu = self.gle = None
+
     @unittest.skipUnless(lib_gl, 'lib_gl not available')
     def test_gl(self):
         if self.gl:
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -15,9 +15,9 @@
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-class PickleTest(unittest.TestCase):
+class PickleTest:
     def dumps(self, item):
-        return pickle.dumps(item)
+        return pickle.dumps(item, self.proto)
 
     def loads(self, item):
         return pickle.loads(item)
@@ -72,17 +72,15 @@
 
     @xfail
     def test_wchar(self):
-        pickle.dumps(c_char("x"))
+        self.dumps(c_char(b"x"))
         # Issue 5049
-        pickle.dumps(c_wchar(u"x"))
+        self.dumps(c_wchar(u"x"))
 
-class PickleTest_1(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 1)
-
-class PickleTest_2(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 2)
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,8 +7,6 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
-LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
-large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -191,9 +189,11 @@
             self.assertEqual(bool(mth), True)
 
     def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
         self.assertTrue(POINTER(LargeNamedType))
 
     def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
         self.assertTrue(POINTER(large_string))
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -178,7 +178,7 @@
             res = re.findall(expr, data)
             if not res:
                 return _get_soname(_findLib_gcc(name))
-            res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y)))
+            res.sort(key=_num_version)
             return res[-1]
 
     elif sys.platform == "sunos5":
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.9"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf8 -*-
 """Tests for distutils.command.check."""
+import textwrap
 import unittest
 from test.test_support import run_unittest
 
@@ -93,6 +94,36 @@
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
+    @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils")
+    def test_check_restructuredtext_with_syntax_highlight(self):
+        # Don't fail if there is a `code` or `code-block` directive
+
+        example_rst_docs = []
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code:: python
+
+                def foo():
+                    pass
+            """))
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code-block:: python
+
+                def foo():
+                    pass
+            """))
+
+        for rest_with_code in example_rst_docs:
+            pkg_info, dist = self.create_dist(long_description=rest_with_code)
+            cmd = check(dist)
+            cmd.check_restructuredtext()
+            self.assertEqual(cmd._warnings, 0)
+            msgs = cmd._check_rst_data(rest_with_code)
+            self.assertEqual(len(msgs), 0)
+
     def test_check_all(self):
 
         metadata = {'url': 'xxx', 'author': 'xxx'}
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py
--- a/lib-python/2.7/dumbdbm.py
+++ b/lib-python/2.7/dumbdbm.py
@@ -21,6 +21,7 @@
 
 """
 
+import ast as _ast
 import os as _os
 import __builtin__
 import UserDict
@@ -85,7 +86,7 @@
             with f:
                 for line in f:
                     line = line.rstrip()
-                    key, pos_and_siz_pair = eval(line)
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
                     self._index[key] = pos_and_siz_pair
 
     # Write the index dict to the directory file.  The original directory
@@ -208,8 +209,10 @@
         return len(self._index)
 
     def close(self):
-        self._commit()
-        self._index = self._datfile = self._dirfile = self._bakfile = None
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
 
     __del__ = close
 
diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py
--- a/lib-python/2.7/encodings/uu_codec.py
+++ b/lib-python/2.7/encodings/uu_codec.py
@@ -84,7 +84,7 @@
             data = a2b_uu(s)
         except binascii.Error, v:
             # Workaround for broken uuencoders by /Fredrik Lundh
-            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3
+            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3
             data = a2b_uu(s[:nbytes])
             #sys.stderr.write("Warning: %s\n" % str(v))
         write(data)
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "7.0"
+_SETUPTOOLS_VERSION = "15.2"
 
-_PIP_VERSION = "1.5.6"
+_PIP_VERSION = "6.1.1"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py
--- a/lib-python/2.7/fileinput.py
+++ b/lib-python/2.7/fileinput.py
@@ -233,8 +233,10 @@
         self.close()
 
     def close(self):
-        self.nextfile()
-        self._files = ()
+        try:
+            self.nextfile()
+        finally:
+            self._files = ()
 
     def __iter__(self):
         return self
@@ -270,23 +272,25 @@
 
         output = self._output
         self._output = 0
-        if output:
-            output.close()
+        try:
+            if output:
+                output.close()
+        finally:
+            file = self._file
+            self._file = 0
+            try:
+                if file and not self._isstdin:
+                    file.close()
+            finally:
+                backupfilename = self._backupfilename
+                self._backupfilename = 0
+                if backupfilename and not self._backup:
+                    try: os.unlink(backupfilename)
+                    except OSError: pass
 
-        file = self._file
-        self._file = 0
-        if file and not self._isstdin:
-            file.close()
-
-        backupfilename = self._backupfilename
-        self._backupfilename = 0
-        if backupfilename and not self._backup:
-            try: os.unlink(backupfilename)
-            except OSError: pass
-
-        self._isstdin = False
-        self._buffer = []
-        self._bufindex = 0
+                self._isstdin = False
+                self._buffer = []
+                self._bufindex = 0
 
     def readline(self):
         try:
diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py
--- a/lib-python/2.7/fnmatch.py
+++ b/lib-python/2.7/fnmatch.py
@@ -47,12 +47,14 @@
     import os,posixpath
     result=[]
     pat=os.path.normcase(pat)
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    match=_cache[pat].match
+        _cache[pat] = re_pat = re.compile(res)
+    match = re_pat.match
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -71,12 +73,14 @@
     its arguments.
     """
 
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    return _cache[pat].match(name) is not None
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
 
 def translate(pat):
     """Translate a shell PATTERN to a regular expression.
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -594,11 +594,16 @@
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file is not None:
-            self.file.close()
-        if self.sock is not None:
-            self.sock.close()
-        self.file = self.sock = None
+        try:
+            file = self.file
+            self.file = None
+            if file is not None:
+                file.close()
+        finally:
+            sock = self.sock
+            self.sock = None
+            if sock is not None:
+                sock.close()
 
 try:
     import ssl
@@ -638,12 +643,24 @@
         '221 Goodbye.'
         >>>
         '''
-        ssl_version = ssl.PROTOCOL_TLSv1
+        ssl_version = ssl.PROTOCOL_SSLv23
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
-                     certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     certfile=None, context=None,
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
+            if context is None:
+                context = ssl._create_stdlib_context(self.ssl_version,
+                                                     certfile=certfile,
+                                                     keyfile=keyfile)
+            self.context = context
             self._prot_p = False
             FTP.__init__(self, host, user, passwd, acct, timeout)
 
@@ -656,12 +673,12 @@
             '''Set up secure control connection by using TLS/SSL.'''
             if isinstance(self.sock, ssl.SSLSocket):
                 raise ValueError("Already using TLS")
-            if self.ssl_version == ssl.PROTOCOL_TLSv1:
+            if self.ssl_version >= ssl.PROTOCOL_SSLv23:
                 resp = self.voidcmd('AUTH TLS')
             else:
                 resp = self.voidcmd('AUTH SSL')
-            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
-                                        ssl_version=self.ssl_version)
+            self.sock = self.context.wrap_socket(self.sock,
+                                                 server_hostname=self.host)
             self.file = self.sock.makefile(mode='rb')
             return resp
 
@@ -692,8 +709,8 @@
         def ntransfercmd(self, cmd, rest=None):
             conn, size = FTP.ntransfercmd(self, cmd, rest)
             if self._prot_p:
-                conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
-                                       ssl_version=self.ssl_version)
+                conn = self.context.wrap_socket(conn,
+                                                server_hostname=self.host)
             return conn, size
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py
--- a/lib-python/2.7/genericpath.py
+++ b/lib-python/2.7/genericpath.py
@@ -10,6 +10,14 @@
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -52,7 +52,9 @@
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
-           'dgettext', 'dngettext', 'gettext', 'ngettext',
+           'bind_textdomain_codeset',
+           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ldngettext', 'lngettext', 'ngettext',
            ]
 
 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
@@ -294,11 +296,12 @@
             # See if we're looking at GNU .mo conventions for metadata
             if mlen == 0:
                 # Catalog description
-                lastk = k = None
+                lastk = None
                 for item in tmsg.splitlines():
                     item = item.strip()
                     if not item:
                         continue
+                    k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
                         k = k.strip().lower()
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -238,9 +238,9 @@
             data = data.tobytes()
 
         if len(data) > 0:
-            self.size = self.size + len(data)
+            self.fileobj.write(self.compress.compress(data))
+            self.size += len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
         return len(data)
@@ -369,19 +369,21 @@
         return self.fileobj is None
 
     def close(self):
-        if self.fileobj is None:
+        fileobj = self.fileobj
+        if fileobj is None:
             return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
+        self.fileobj = None
+        try:
+            if self.mode == WRITE:
+                fileobj.write(self.compress.flush())
+                write32u(fileobj, self.crc)
+                # self.size may exceed 2GB, or even 4GB
+                write32u(fileobj, self.size & 0xffffffffL)
+        finally:
+            myfileobj = self.myfileobj
+            if myfileobj:
+                self.myfileobj = None
+                myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_closed()
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -187,7 +187,7 @@
 
         def prf(msg, inner=inner, outer=outer):
             # PBKDF2_HMAC uses the password as key. We can re-use the same
-            # digest objects and and just update copies to skip initialization.
+            # digest objects and just update copies to skip initialization.
             icpy = inner.copy()
             ocpy = outer.copy()
             icpy.update(msg)
diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py
--- a/lib-python/2.7/htmlentitydefs.py
+++ b/lib-python/2.7/htmlentitydefs.py
@@ -1,6 +1,6 @@
 """HTML character entity references."""
 
-# maps the HTML entity name to the Unicode codepoint
+# maps the HTML entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -256,7 +256,7 @@
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
-# maps the Unicode codepoint to the HTML entity name
+# maps the Unicode code point to the HTML entity name
 codepoint2name = {}
 
 # maps the HTML entity name to the character
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -68,6 +68,7 @@
 
 from array import array
 import os
+import re
 import socket
 from sys import py3kwarning
 from urlparse import urlsplit
@@ -218,6 +219,38 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
+#
+# VCHAR          = %x21-7E
+# obs-text       = %x80-FF
+# header-field   = field-name ":" OWS field-value OWS
+# field-name     = token
+# field-value    = *( field-content / obs-fold )
+# field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+# field-vchar    = VCHAR / obs-text
+#
+# obs-fold       = CRLF 1*( SP / HTAB )
+#                ; obsolete line folding
+#                ; see Section 3.2.4
+
+# token          = 1*tchar
+#
+# tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+#                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+#                / DIGIT / ALPHA
+#                ; any VCHAR, except delimiters
+#
+# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
+
+# the patterns for both name and value are more leniant than RFC
+# definitions to allow for backwards compatibility
+_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match
+_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
+
+# We always set the Content-Length header for these methods because some
+# servers will otherwise respond with a 411
+_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
+
 
 class HTTPMessage(mimetools.Message):
 
@@ -313,6 +346,11 @@
                 hlist.append(line)
                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
                 continue
+            elif headerseen is not None:
+                # An empty header name. These aren't allowed in HTTP, but it's
+                # probably a benign mistake. Don't add the header, just keep
+                # going.
+                continue
             else:
                 # It's not a header line; throw it back and stop here.
                 if not self.dict:
@@ -522,9 +560,10 @@
         return True
 
     def close(self):
-        if self.fp:
-            self.fp.close()
+        fp = self.fp
+        if fp:
             self.fp = None
+            fp.close()
 
     def isclosed(self):
         # NOTE: it is possible that we will not ever call self.close(). This
@@ -723,7 +762,7 @@
         endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
         request to the proxy server when the connection is established.
 
-        This method must be called before the HTML connection has been
+        This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers
@@ -797,13 +836,17 @@
 
     def close(self):
         """Close the connection to the HTTP server."""
-        if self.sock:
-            self.sock.close()   # close it manually... there may be other refs
-            self.sock = None
-        if self.__response:
-            self.__response.close()
-            self.__response = None
         self.__state = _CS_IDLE
+        try:
+            sock = self.sock
+            if sock:
+                self.sock = None
+                sock.close()   # close it manually... there may be other refs
+        finally:
+            response = self.__response
+            if response:
+                self.__response = None
+                response.close()
 
     def send(self, data):
         """Send `data' to the server."""
@@ -978,7 +1021,16 @@
         if self.__state != _CS_REQ_STARTED:
             raise CannotSendHeader()
 
-        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        header = '%s' % header
+        if not _is_legal_header_name(header):
+            raise ValueError('Invalid header name %r' % (header,))
+
+        values = [str(v) for v in values]
+        for one_value in values:
+            if _is_illegal_header_value(one_value):
+                raise ValueError('Invalid header value %r' % (one_value,))
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join(values))
         self._output(hdr)
 
     def endheaders(self, message_body=None):
@@ -1000,19 +1052,25 @@
         """Send a complete request to the server."""
         self._send_request(method, url, body, headers)
 
-    def _set_content_length(self, body):
-        # Set the content-length based on the body.
+    def _set_content_length(self, body, method):
+        # Set the content-length based on the body. If the body is "empty", we
+        # set Content-Length: 0 for methods that expect a body (RFC 7230,
+        # Section 3.3.2). If the body is set for other methods, we set the
+        # header provided we can figure out what the length is.
         thelen = None
-        try:
-            thelen = str(len(body))
-        except TypeError, te:
-            # If this is a file-like object, try to
-            # fstat its file descriptor
+        if body is None and method.upper() in _METHODS_EXPECTING_BODY:
+            thelen = '0'
+        elif body is not None:
             try:
-                thelen = str(os.fstat(body.fileno()).st_size)
-            except (AttributeError, OSError):
-                # Don't send a length if this failed
-                if self.debuglevel > 0: print "Cannot stat!!"
+                thelen = str(len(body))
+            except TypeError:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0: print "Cannot stat!!"
 
         if thelen is not None:
             self.putheader('Content-Length', thelen)
@@ -1028,8 +1086,8 @@
 
         self.putrequest(method, url, **skips)
 
-        if body is not None and 'content-length' not in header_names:
-            self._set_content_length(body)
+        if 'content-length' not in header_names:
+            self._set_content_length(body, method)
         for hdr, value in headers.iteritems():
             self.putheader(hdr, value)
         self.endheaders(body)
@@ -1072,20 +1130,20 @@
 
         try:
             response.begin()
+            assert response.will_close != _UNKNOWN
+            self.__state = _CS_IDLE
+
+            if response.will_close:
+                # this effectively passes the connection to the response
+                self.close()
+            else:
+                # remember this, so we can tell when it is complete
+                self.__response = response
+
+            return response
         except:
             response.close()
             raise
-        assert response.will_close != _UNKNOWN
-        self.__state = _CS_IDLE
-
-        if response.will_close:
-            # this effectively passes the connection to the response
-            self.close()
-        else:
-            # remember this, so we can tell when it is complete
-            self.__response = response
-
-        return response
 
 
 class HTTP:
@@ -1129,7 +1187,7 @@
         "Accept arguments to set the host/port, since the superclass doesn't."
 
         if host is not None:
-            self._conn._set_hostport(host, port)
+            (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port)
         self._conn.connect()
 
     def getfile(self):
diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py
--- a/lib-python/2.7/idlelib/CodeContext.py
+++ b/lib-python/2.7/idlelib/CodeContext.py
@@ -15,8 +15,8 @@
 from sys import maxint as INFINITY
 from idlelib.configHandler import idleConf
 
-BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for",
-                    "if", "try", "while", "with"])
+BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for",
+                    "if", "try", "while", "with"}
 UPDATEINTERVAL = 100 # millisec
 FONTUPDATEINTERVAL = 1000 # millisec
 
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -469,13 +469,10 @@
         ("format", "F_ormat"),
         ("run", "_Run"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     def createmenubar(self):
         mbar = self.menubar
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -44,9 +44,11 @@
 
         The length limit parameter is for testing with a known value.
         """
-        if limit == None:
+        if limit is None:
+            # The default length limit is that defined by pep8
             limit = idleConf.GetOption(
-                    'main', 'FormatParagraph', 'paragraph', type='int')
+                'extensions', 'FormatParagraph', 'max-width',
+                type='int', default=72)
         text = self.editwin.text
         first, last = self.editwin.get_selection_indices()
         if first and last:
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -871,13 +871,10 @@
         ("edit", "_Edit"),
         ("debug", "_Debug"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     # New classes
     from idlelib.IdleHistory import History
@@ -1350,7 +1347,7 @@
         if type(s) not in (unicode, str, bytearray):
             # See issue #19481
             if isinstance(s, unicode):
-                s = unicode.__getslice__(s, None, None)
+                s = unicode.__getitem__(s, slice(None))
             elif isinstance(s, str):
                 s = str.__str__(s)
             elif isinstance(s, bytearray):
diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py
--- a/lib-python/2.7/idlelib/SearchEngine.py
+++ b/lib-python/2.7/idlelib/SearchEngine.py
@@ -191,7 +191,7 @@
 
     This is done by searching forwards until there is no match.
     Prog: compiled re object with a search method returning a match.
-    Chars: line of text, without \n.
+    Chars: line of text, without \\n.
     Col: stop index for the search; the limit for match.end().
     '''
     m = prog.search(chars)
diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def
--- a/lib-python/2.7/idlelib/config-extensions.def
+++ b/lib-python/2.7/idlelib/config-extensions.def
@@ -66,6 +66,7 @@
 
 [FormatParagraph]
 enable=True
+max-width=72
 [FormatParagraph_cfgBindings]
 format-paragraph=<Alt-Key-q>
 
diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def
--- a/lib-python/2.7/idlelib/config-main.def
+++ b/lib-python/2.7/idlelib/config-main.def
@@ -58,9 +58,6 @@
 font-bold= 0
 encoding= none
 
-[FormatParagraph]
-paragraph=72
-
 [Indent]
 use-spaces= 1
 num-spaces= 4
diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py
--- a/lib-python/2.7/idlelib/configDialog.py
+++ b/lib-python/2.7/idlelib/configDialog.py
@@ -371,7 +371,6 @@
         parent = self.parent
         self.winWidth = StringVar(parent)
         self.winHeight = StringVar(parent)
-        self.paraWidth = StringVar(parent)
         self.startupEdit = IntVar(parent)
         self.autoSave = IntVar(parent)
         self.encoding = StringVar(parent)
@@ -387,7 +386,6 @@
         frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Autosave Preferences ')
         frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE)
-        frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE)
         frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE)
         frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Additional Help Sources ')
@@ -416,11 +414,6 @@
         labelWinHeightTitle = Label(frameWinSize, text='Height')
         entryWinHeight = Entry(
                 frameWinSize, textvariable=self.winHeight, width=3)
-        #paragraphFormatWidth
-        labelParaWidthTitle = Label(
-                frameParaSize, text='Paragraph reformat width (in characters)')
-        entryParaWidth = Entry(
-                frameParaSize, textvariable=self.paraWidth, width=3)
         #frameEncoding
         labelEncodingTitle = Label(
                 frameEncoding, text="Default Source Encoding")
@@ -458,7 +451,6 @@
         frameRun.pack(side=TOP, padx=5, pady=5, fill=X)
         frameSave.pack(side=TOP, padx=5, pady=5, fill=X)
         frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X)
-        frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X)
         frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X)
         frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH)
         #frameRun
@@ -475,9 +467,6 @@
         labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5)
         entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5)
-        #paragraphFormatWidth
-        labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
-        entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         #frameEncoding
         labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
         radioEncNone.pack(side=RIGHT, anchor=E, pady=5)
@@ -509,7 +498,6 @@
         self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin)
         self.winWidth.trace_variable('w', self.VarChanged_winWidth)
         self.winHeight.trace_variable('w', self.VarChanged_winHeight)
-        self.paraWidth.trace_variable('w', self.VarChanged_paraWidth)
         self.startupEdit.trace_variable('w', self.VarChanged_startupEdit)
         self.autoSave.trace_variable('w', self.VarChanged_autoSave)
         self.encoding.trace_variable('w', self.VarChanged_encoding)
@@ -594,10 +582,6 @@
         value = self.winHeight.get()
         self.AddChangedItem('main', 'EditorWindow', 'height', value)
 
-    def VarChanged_paraWidth(self, *params):
-        value = self.paraWidth.get()
-        self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value)
-
     def VarChanged_startupEdit(self, *params):
         value = self.startupEdit.get()
         self.AddChangedItem('main', 'General', 'editor-on-startup', value)
@@ -1094,9 +1078,6 @@
                 'main', 'EditorWindow', 'width', type='int'))
         self.winHeight.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'height', type='int'))
-        #initial paragraph reformat size
-        self.paraWidth.set(idleConf.GetOption(
-                'main', 'FormatParagraph', 'paragraph', type='int'))
         # default source encoding
         self.encoding.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'encoding', default='none'))
diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt
--- a/lib-python/2.7/idlelib/help.txt
+++ b/lib-python/2.7/idlelib/help.txt
@@ -100,7 +100,7 @@
 			  which is scrolling off the top or the window.
 			  (Not present in Shell window.)
 
-Windows Menu:
+Window Menu:
 
 	Zoom Height -- toggles the window between configured size
 	and maximum height.
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
- at echo off
-rem Start IDLE using the appropriate Python interpreter
-set CURRDIR=%~dp0
-start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+ at echo off
+rem Start IDLE using the appropriate Python interpreter
+set CURRDIR=%~dp0
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py
--- a/lib-python/2.7/idlelib/idle_test/test_calltips.py
+++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py
@@ -55,7 +55,8 @@
         def gtest(obj, out):
             self.assertEqual(signature(obj), out)
 
-        gtest(List, '()\n' + List.__doc__)
+        if List.__doc__ is not None:
+            gtest(List, '()\n' + List.__doc__)
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
@@ -70,7 +71,8 @@
 
     def test_signature_wrap(self):
         # This is also a test of an old-style class
-        self.assertEqual(signature(textwrap.TextWrapper), '''\
+        if textwrap.TextWrapper.__doc__ is not None:
+            self.assertEqual(signature(textwrap.TextWrapper), '''\
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True)''')
@@ -106,20 +108,23 @@
         def t5(a, b=None, *args, **kwds): 'doc'
         t5.tip = "(a, b=None, *args, **kwargs)"
 
+        doc = '\ndoc' if t1.__doc__ is not None else ''
         for func in (t1, t2, t3, t4, t5, TC):
-            self.assertEqual(signature(func), func.tip + '\ndoc')
+            self.assertEqual(signature(func), func.tip + doc)
 
     def test_methods(self):
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__):
-            self.assertEqual(signature(meth), meth.tip + "\ndoc")
-        self.assertEqual(signature(TC.cm), "(a)\ndoc")
-        self.assertEqual(signature(TC.sm), "(b)\ndoc")
+            self.assertEqual(signature(meth), meth.tip + doc)
+        self.assertEqual(signature(TC.cm), "(a)" + doc)
+        self.assertEqual(signature(TC.sm), "(b)" + doc)
 
     def test_bound_methods(self):
         # test that first parameter is correctly removed from argspec
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth, mtip  in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"),
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
-            self.assertEqual(signature(meth), mtip + "\ndoc")
+            self.assertEqual(signature(meth), mtip + doc)
 
     def test_starred_parameter(self):
         # test that starred first parameter is *not* removed from argspec
diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/idlelib/idle_test/test_io.py
@@ -0,0 +1,267 @@
+import unittest
+import io
+from idlelib.PyShell import PseudoInputFile, PseudoOutputFile
+from test import test_support as support
+
+
+class Base(object):
+    def __str__(self):
+        return '%s:str' % type(self).__name__
+    def __unicode__(self):
+        return '%s:unicode' % type(self).__name__
+    def __len__(self):
+        return 3
+    def __iter__(self):
+        return iter('abc')
+    def __getitem__(self, *args):
+        return '%s:item' % type(self).__name__
+    def __getslice__(self, *args):
+        return '%s:slice' % type(self).__name__
+
+class S(Base, str):
+    pass
+
+class U(Base, unicode):
+    pass
+
+class BA(Base, bytearray):
+    pass
+
+class MockShell:
+    def __init__(self):
+        self.reset()
+
+    def write(self, *args):
+        self.written.append(args)
+
+    def readline(self):
+        return self.lines.pop()
+
+    def close(self):
+        pass
+
+    def reset(self):
+        self.written = []
+
+    def push(self, lines):
+        self.lines = list(lines)[::-1]
+
+
+class PseudeOutputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdout>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.read, 0)
+        self.assertRaises(IOError, f.readline, 0)
+
+    def test_write(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.write('test')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        shell.reset()
+        f.write('t\xe8st')
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        shell.reset()
+        f.write(u't\xe8st')
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        shell.reset()
+
+        f.write(S('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(BA('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(U(u't\xe8st'))
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.write)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 'test', 'spam')
+        self.assertEqual(shell.written, [])
+
+    def test_writelines(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.writelines([])
+        self.assertEqual(shell.written, [])
+        shell.reset()
+        f.writelines(['one\n', 'two'])
+        self.assertEqual(shell.written,
+                         [('one\n', 'stdout'), ('two', 'stdout')])
+        shell.reset()
+        f.writelines(['on\xe8\n', 'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')])
+        shell.reset()
+        f.writelines([u'on\xe8\n', u'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [(u'on\xe8\n', 'stdout'), (u'tw\xf2', 'stdout')])
+        shell.reset()
+
+        f.writelines([S('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([BA('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([U(u't\xe8st')])
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.writelines)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [123])
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [], [])
+        self.assertEqual(shell.written, [])
+
+    def test_close(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertFalse(f.closed)
+        f.write('test')
+        f.close()
+        self.assertTrue(f.closed)
+        self.assertRaises(ValueError, f.write, 'x')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        f.close()
+        self.assertRaises(TypeError, f.close, 1)
+
+
+class PseudeInputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '<stdin>')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertTrue(f.readable())
+        self.assertFalse(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.write, 'x')
+        self.assertRaises(IOError, f.writelines, ['x'])
+
+    def test_read(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(-1), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(None), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', 'three\n', ''])
+        self.assertEqual(f.read(2), 'on')
+        self.assertEqual(f.read(3), 'e\nt')
+        self.assertEqual(f.read(10), 'wo\nthree\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.read(0), '')
+        self.assertRaises(TypeError, f.read, 1.5)
+        self.assertRaises(TypeError, f.read, '1')
+        self.assertRaises(TypeError, f.read, 1, 1)
+
+    def test_readline(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', 'three\n', 'four\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(-1), 'two\n')
+        self.assertEqual(f.readline(None), 'three\n')
+        shell.push(['one\ntwo\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(), 'two\n')
+        shell.push(['one', 'two', 'three'])
+        self.assertEqual(f.readline(), 'one')
+        self.assertEqual(f.readline(), 'two')
+        shell.push(['one\n', 'two\n', 'three\n'])
+        self.assertEqual(f.readline(2), 'on')
+        self.assertEqual(f.readline(1), 'e')
+        self.assertEqual(f.readline(1), '\n')
+        self.assertEqual(f.readline(10), 'two\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.readline(0), '')
+        self.assertRaises(TypeError, f.readlines, 1.5)
+        self.assertRaises(TypeError, f.readlines, '1')
+        self.assertRaises(TypeError, f.readlines, 1, 1)
+
+    def test_readlines(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(-1), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(None), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(0), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(3), ['one\n'])
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(4), ['one\n', 'two\n'])
+
+        shell.push(['one\n', 'two\n', ''])
+        self.assertRaises(TypeError, f.readlines, 1.5)
+        self.assertRaises(TypeError, f.readlines, '1')
+        self.assertRaises(TypeError, f.readlines, 1, 1)
+

From noreply at buildbot.pypy.org  Mon Jul  6 13:20:38 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  6 Jul 2015 13:20:38 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: refactored assembler,
 moved vector regalloc & assembler methods to a separate file
Message-ID: <20150706112038.0FECC1C0207@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78453:a3e41e7fb44b
Date: 2015-07-03 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/a3e41e7fb44b/

Log:	refactored assembler, moved vector regalloc & assembler methods to a
	separate file added implementation to automatically generate a exit
	bridge out of the vectorized trace

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -10,6 +10,7 @@
 from rpython.jit.metainterp.history import (Const, Box, VOID,
     BoxVector, ConstInt, BoxVectorAccum)
 from rpython.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT
+from rpython.jit.metainterp.compile import CompileLoopVersionDescr
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref
@@ -29,6 +30,7 @@
     imm0, imm1, FloatImmedLoc, RawEbpLoc, RawEspLoc)
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.backend.x86 import rx86, codebuf, callbuilder
+from rpython.jit.backend.x86.vector_ext import VectorAssemblerMixin
 from rpython.jit.backend.x86.callbuilder import follow_jump
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.backend.x86 import support
@@ -40,7 +42,7 @@
 from rpython.rlib.objectmodel import compute_unique_id
 
 
-class Assembler386(BaseAssembler):
+class Assembler386(BaseAssembler, VectorAssemblerMixin):
     _regalloc = None
     _output_loop_log = None
     _second_tmp_reg = ecx
@@ -550,7 +552,6 @@
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
                                                      'b', descr_number)
-
         arglocs = self.rebuild_faillocs_from_descr(faildescr, inputargs)
         regalloc = RegAlloc(self, self.cpu.translate_support_code)
         startpos = self.mc.get_relative_pos()
@@ -592,8 +593,12 @@
         # for each pending guard, generate the code of the recovery stub
         # at the end of self.mc.
         for tok in self.pending_guard_tokens:
-            regalloc.position = tok.position
-            tok.pos_recovery_stub = self.generate_quick_failure(tok, regalloc)
+            descr = tok.faildescr
+            if not isinstance(descr, CompileLoopVersionDescr):
+                regalloc.position = tok.position
+                tok.pos_recovery_stub = self.generate_quick_failure(tok, regalloc)
+            else:
+                self.store_info_on_descr(0, tok)
         if WORD == 8 and len(self.pending_memoryerror_trampoline_from) > 0:
             self.error_trampoline_64 = self.generate_propagate_error_64()
 
@@ -606,6 +611,9 @@
         for tok in self.pending_guard_tokens:
             addr = rawstart + tok.pos_jump_offset
             tok.faildescr.adr_jump_offset = addr
+            descr = tok.faildescr
+            if isinstance(descr, CompileLoopVersionDescr):
+                continue # patch them later
             relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4)
             assert rx86.fits_in_32bits(relative_target)
             #
@@ -1645,30 +1653,6 @@
             self.mc.MOVD32_xr(resloc.value, eax.value)
             self.mc.PUNPCKLDQ_xx(resloc.value, loc1.value)
 
-    def _guard_vector_true(self, guard_op, loc, zero=False):
-        arg = guard_op.getarg(0)
-        assert isinstance(arg, BoxVector)
-        size = arg.item_size
-        temp = X86_64_XMM_SCRATCH_REG
-        #
-        self.mc.PXOR(temp, temp)
-        # if the vector is not fully packed blend 1s
-        if not arg.fully_packed(self.cpu.vector_register_size):
-            self.mc.PCMPEQQ(temp, temp) # fill with ones
-            select = 0
-            bits_used = (arg.item_count * arg.item_size * 8)
-            index = bits_used // 16
-            while index < 8:
-                select |= (1 << index)
-                index += 1
-            self.mc.PBLENDW_xxi(loc.value, temp.value, select)
-            # reset to zeros
-            self.mc.PXOR(temp, temp)
-
-        self.mc.PCMPEQ(size, loc, temp)
-        self.mc.PCMPEQQ(temp, temp)
-        self.mc.PTEST(loc, temp)
-
     def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
         if isinstance(loc, RegLoc):
@@ -2527,393 +2511,6 @@
                 self.save_into_mem(addr, imm0, imm(current))
             i += current
 
-    # vector operations
-    # ________________________________________
-
-    def _accum_update_at_exit(self, fail_locs, fail_args, faildescr, regalloc):
-        """ If accumulation is done in this loop, at the guard exit
-        some vector registers must be adjusted to yield the correct value"""
-        assert regalloc is not None
-        accum_info = faildescr.rd_accum_list
-        while accum_info:
-            pos = accum_info.position
-            loc = fail_locs[pos]
-            assert isinstance(loc, RegLoc)
-            arg = fail_args[pos]
-            if isinstance(arg, BoxVectorAccum):
-                arg = arg.scalar_var
-            assert arg is not None
-            tgtloc = regalloc.force_allocate_reg(arg, fail_args)
-            if accum_info.operation == '+':
-                # reduction using plus
-                self._accum_reduce_sum(arg, loc, tgtloc)
-            elif accum_info.operation == '*':
-                self._accum_reduce_mul(arg, loc, tgtloc)
-            else:
-                not_implemented("accum operator %s not implemented" %
-                                            (accum_info.operation)) 
-            fail_locs[pos] = tgtloc
-            regalloc.possibly_free_var(arg)
-            accum_info = accum_info.prev
-
-    def _accum_reduce_mul(self, arg, accumloc, targetloc):
-        scratchloc = X86_64_XMM_SCRATCH_REG
-        self.mov(accumloc, scratchloc)
-        # swap the two elements
-        self.mc.SHUFPD_xxi(scratchloc.value, scratchloc.value, 0x01)
-        self.mc.MULSD(accumloc, scratchloc)
-        if accumloc is not targetloc:
-            self.mov(accumloc, targetloc)
-
-    def _accum_reduce_sum(self, arg, accumloc, targetloc):
-        # Currently the accumulator can ONLY be the biggest
-        # size for X86 -> 64 bit float/int
-        if arg.type == FLOAT:
-            # r = (r[0]+r[1],r[0]+r[1])
-            self.mc.HADDPD(accumloc, accumloc)
-            # upper bits (> 64) are dirty (but does not matter)
-            if accumloc is not targetloc:
-                self.mov(accumloc, targetloc)
-            return
-        elif arg.type == INT:
-            scratchloc = X86_64_SCRATCH_REG
-            self.mc.PEXTRQ_rxi(targetloc.value, accumloc.value, 0)
-            self.mc.PEXTRQ_rxi(scratchloc.value, accumloc.value, 1)
-            self.mc.ADD(targetloc, scratchloc)
-            return
-
-        not_implemented("reduce sum for %s not impl." % arg)
-
-    def genop_vec_getarrayitem_raw(self, op, arglocs, resloc):
-        # considers item scale (raw_load does not)
-        base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
-        scale = get_scale(size_loc.value)
-        src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale)
-        self._vec_load(resloc, src_addr, integer_loc.value,
-                       size_loc.value, aligned_loc.value)
-
-    def genop_vec_raw_load(self, op, arglocs, resloc):
-        base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
-        src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0)
-        self._vec_load(resloc, src_addr, integer_loc.value,
-                       size_loc.value, aligned_loc.value)
-
-    def _vec_load(self, resloc, src_addr, integer, itemsize, aligned):
-        if integer:
-            if aligned:
-                self.mc.MOVDQA(resloc, src_addr)
-            else:
-                self.mc.MOVDQU(resloc, src_addr)
-        else:
-            if itemsize == 4:
-                self.mc.MOVUPS(resloc, src_addr)
-            elif itemsize == 8:
-                self.mc.MOVUPD(resloc, src_addr)
-
-    def genop_discard_vec_setarrayitem_raw(self, op, arglocs):
-        # considers item scale (raw_store does not)
-        base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
-        scale = get_scale(size_loc.value)
-        dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, scale)
-        self._vec_store(dest_loc, value_loc, integer_loc.value,
-                        size_loc.value, aligned_loc.value)
-
-    def genop_discard_vec_raw_store(self, op, arglocs):
-        base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
-        dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0)
-        self._vec_store(dest_loc, value_loc, integer_loc.value,
-                        size_loc.value, aligned_loc.value)
-
-    def _vec_store(self, dest_loc, value_loc, integer, itemsize, aligned):
-        if integer:
-            if aligned:
-                self.mc.MOVDQA(dest_loc, value_loc)
-            else:
-                self.mc.MOVDQU(dest_loc, value_loc)
-        else:
-            if itemsize == 4:
-                self.mc.MOVUPS(dest_loc, value_loc)
-            elif itemsize == 8:
-                self.mc.MOVUPD(dest_loc, value_loc)
-
-    def genop_vec_int_mul(self, op, arglocs, resloc):
-        loc0, loc1, itemsize_loc = arglocs
-        itemsize = itemsize_loc.value
-        if itemsize == 2:
-            self.mc.PMULLW(loc0, loc1)
-        elif itemsize == 4:
-            self.mc.PMULLD(loc0, loc1)
-        else:
-            # NOTE see http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025
-            # There is no 64x64 bit packed mul and I did not find one
-            # for 8 bit either. It is questionable if it gives any benefit
-            # for 8 bit.
-            not_implemented("int8/64 mul")
-
-    def genop_vec_int_add(self, op, arglocs, resloc):
-        loc0, loc1, size_loc = arglocs
-        size = size_loc.value
-        if size == 1:
-            self.mc.PADDB(loc0, loc1)
-        elif size == 2:
-            self.mc.PADDW(loc0, loc1)
-        elif size == 4:
-            self.mc.PADDD(loc0, loc1)
-        elif size == 8:
-            self.mc.PADDQ(loc0, loc1)
-
-    def genop_vec_int_sub(self, op, arglocs, resloc):
-        loc0, loc1, size_loc = arglocs
-        size = size_loc.value
-        if size == 1:
-            self.mc.PSUBB(loc0, loc1)
-        elif size == 2:
-            self.mc.PSUBW(loc0, loc1)
-        elif size == 4:
-            self.mc.PSUBD(loc0, loc1)
-        elif size == 8:
-            self.mc.PSUBQ(loc0, loc1)
-
-    def genop_vec_int_and(self, op, arglocs, resloc):
-        self.mc.PAND(resloc, arglocs[0])
-
-    def genop_vec_int_or(self, op, arglocs, resloc):
-        self.mc.POR(resloc, arglocs[0])
-
-    def genop_vec_int_xor(self, op, arglocs, resloc):
-        self.mc.PXOR(resloc, arglocs[0])
-
-    genop_vec_float_arith = """
-    def genop_vec_float_{type}(self, op, arglocs, resloc):
-        loc0, loc1, itemsize_loc = arglocs
-        itemsize = itemsize_loc.value
-        if itemsize == 4:
-            self.mc.{p_op_s}(loc0, loc1)
-        elif itemsize == 8:
-            self.mc.{p_op_d}(loc0, loc1)
-    """
-    for op in ['add','mul','sub']:
-        OP = op.upper()
-        _source = genop_vec_float_arith.format(type=op,
-                                               p_op_s=OP+'PS',
-                                               p_op_d=OP+'PD')
-        exec py.code.Source(_source).compile()
-    del genop_vec_float_arith
-
-    def genop_vec_float_truediv(self, op, arglocs, resloc):
-        loc0, loc1, sizeloc = arglocs
-        size = sizeloc.value
-        if size == 4:
-            self.mc.DIVPS(loc0, loc1)
-        elif size == 8:
-            self.mc.DIVPD(loc0, loc1)
-
-    def genop_vec_float_abs(self, op, arglocs, resloc):
-        src, sizeloc = arglocs
-        size = sizeloc.value
-        if size == 4:
-            self.mc.ANDPS(src, heap(self.single_float_const_abs_addr))
-        elif size == 8:
-            self.mc.ANDPD(src, heap(self.float_const_abs_addr))
-
-    def genop_vec_float_neg(self, op, arglocs, resloc):
-        src, sizeloc = arglocs
-        size = sizeloc.value
-        if size == 4:
-            self.mc.XORPS(src, heap(self.single_float_const_neg_addr))
-        elif size == 8:
-            self.mc.XORPD(src, heap(self.float_const_neg_addr))
-
-    def genop_vec_int_signext(self, op, arglocs, resloc):
-        srcloc, sizeloc, tosizeloc = arglocs
-        size = sizeloc.value
-        tosize = tosizeloc.value
-        if size == tosize:
-            return # already the right size
-        if size == 4 and tosize == 8:
-            scratch = X86_64_SCRATCH_REG.value
-            self.mc.PEXTRD_rxi(scratch, srcloc.value, 1)
-            self.mc.PINSRQ_xri(resloc.value, scratch, 1)
-            self.mc.PEXTRD_rxi(scratch, srcloc.value, 0)
-            self.mc.PINSRQ_xri(resloc.value, scratch, 0)
-        elif size == 8 and tosize == 4:
-            # is there a better sequence to move them?
-            scratch = X86_64_SCRATCH_REG.value
-            self.mc.PEXTRQ_rxi(scratch, srcloc.value, 0)
-            self.mc.PINSRD_xri(resloc.value, scratch, 0)
-            self.mc.PEXTRQ_rxi(scratch, srcloc.value, 1)
-            self.mc.PINSRD_xri(resloc.value, scratch, 1)
-        else:
-            # note that all other conversions are not implemented
-            # on purpose. it needs many x86 op codes to implement
-            # the missing combinations. even if they are implemented
-            # the speedup might only be modest...
-            # the optimization does not emit such code!
-            msg = "vec int signext (%d->%d)" % (size, tosize)
-            not_implemented(msg)
-
-    def genop_vec_float_expand(self, op, arglocs, resloc):
-        srcloc, sizeloc = arglocs
-        size = sizeloc.value
-        if isinstance(srcloc, ConstFloatLoc):
-            # they are aligned!
-            self.mc.MOVAPD(resloc, srcloc)
-        elif size == 4:
-            # the register allocator forces src to be the same as resloc
-            # r = (s[0], s[0], r[0], r[0])
-            # since resloc == srcloc: r = (r[0], r[0], r[0], r[0])
-            self.mc.SHUFPS_xxi(resloc.value, srcloc.value, 0)
-        elif size == 8:
-            self.mc.MOVDDUP(resloc, srcloc)
-        else:
-            raise AssertionError("float of size %d not supported" % (size,))
-
-    def genop_vec_int_expand(self, op, arglocs, resloc):
-        srcloc, sizeloc = arglocs
-        if not isinstance(srcloc, RegLoc):
-            self.mov(srcloc, X86_64_SCRATCH_REG)
-            srcloc = X86_64_SCRATCH_REG
-        assert not srcloc.is_xmm
-        size = sizeloc.value
-        if size == 1:
-            self.mc.PINSRB_xri(resloc.value, srcloc.value, 0)
-            self.mc.PSHUFB(resloc, heap(self.expand_byte_mask_addr))
-        elif size == 2:
-            self.mc.PINSRW_xri(resloc.value, srcloc.value, 0)
-            self.mc.PINSRW_xri(resloc.value, srcloc.value, 4)
-            self.mc.PSHUFLW_xxi(resloc.value, resloc.value, 0)
-            self.mc.PSHUFHW_xxi(resloc.value, resloc.value, 0)
-        elif size == 4:
-            self.mc.PINSRD_xri(resloc.value, srcloc.value, 0)
-            self.mc.PSHUFD_xxi(resloc.value, resloc.value, 0)
-        elif size == 8:
-            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0)
-            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1)
-        else:
-            raise AssertionError("cannot handle size %d (int expand)" % (size,))
-
-    def genop_vec_int_pack(self, op, arglocs, resloc):
-        resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
-        assert isinstance(resultloc, RegLoc)
-        assert isinstance(sourceloc, RegLoc)
-        size = sizeloc.value
-        srcidx = srcidxloc.value
-        residx = residxloc.value
-        count = countloc.value
-        # for small data type conversion this can be quite costy
-        # NOTE there might be some combinations that can be handled
-        # more efficiently! e.g.
-        # v2 = pack(v0,v1,4,4)
-        si = srcidx
-        ri = residx
-        k = count
-        while k > 0:
-            if size == 8:
-                if resultloc.is_xmm and sourceloc.is_xmm: # both xmm
-                    self.mc.PEXTRQ_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRQ_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
-                elif resultloc.is_xmm: # xmm <- reg
-                    self.mc.PINSRQ_xri(resultloc.value, sourceloc.value, ri)
-                else: # reg <- xmm
-                    self.mc.PEXTRQ_rxi(resultloc.value, sourceloc.value, si)
-            elif size == 4:
-                if resultloc.is_xmm and sourceloc.is_xmm:
-                    self.mc.PEXTRD_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRD_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
-                elif resultloc.is_xmm:
-                    self.mc.PINSRD_xri(resultloc.value, sourceloc.value, ri)
-                else:
-                    self.mc.PEXTRD_rxi(resultloc.value, sourceloc.value, si)
-            elif size == 2:
-                if resultloc.is_xmm and sourceloc.is_xmm:
-                    self.mc.PEXTRW_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRW_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
-                elif resultloc.is_xmm:
-                    self.mc.PINSRW_xri(resultloc.value, sourceloc.value, ri)
-                else:
-                    self.mc.PEXTRW_rxi(resultloc.value, sourceloc.value, si)
-            elif size == 1:
-                if resultloc.is_xmm and sourceloc.is_xmm:
-                    self.mc.PEXTRB_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
-                    self.mc.PINSRB_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
-                elif resultloc.is_xmm:
-                    self.mc.PINSRB_xri(resultloc.value, sourceloc.value, ri)
-                else:
-                    self.mc.PEXTRB_rxi(resultloc.value, sourceloc.value, si)
-            si += 1
-            ri += 1
-            k -= 1
-
-    genop_vec_int_unpack = genop_vec_int_pack
-
-    def genop_vec_float_pack(self, op, arglocs, resultloc):
-        resloc, srcloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
-        assert isinstance(resloc, RegLoc)
-        assert isinstance(srcloc, RegLoc)
-        count = countloc.value
-        residx = residxloc.value
-        srcidx = srcidxloc.value
-        size = sizeloc.value
-        if size == 4:
-            si = srcidx
-            ri = residx
-            k = count
-            while k > 0:
-                if resloc.is_xmm:
-                    src = srcloc.value
-                    if not srcloc.is_xmm:
-                        # if source is a normal register (unpack)
-                        assert count == 1
-                        assert si == 0
-                        self.mov(srcloc, X86_64_XMM_SCRATCH_REG)
-                        src = X86_64_XMM_SCRATCH_REG.value
-                    select = ((si & 0x3) << 6)|((ri & 0x3) << 4)
-                    self.mc.INSERTPS_xxi(resloc.value, src, select)
-                else:
-                    self.mc.PEXTRD_rxi(resloc.value, srcloc.value, si)
-                si += 1
-                ri += 1
-                k -= 1
-        elif size == 8:
-            assert resloc.is_xmm
-            if srcloc.is_xmm:
-                if srcidx == 0:
-                    if residx == 0:
-                        # r = (s[0], r[1])
-                        self.mc.MOVSD(resloc, srcloc)
-                    else:
-                        assert residx == 1
-                        # r = (r[0], s[0])
-                        self.mc.UNPCKLPD(resloc, srcloc)
-                else:
-                    assert srcidx == 1
-                    if residx == 0:
-                        # r = (s[1], r[1])
-                        if resloc != srcloc:
-                            self.mc.UNPCKHPD(resloc, srcloc)
-                        self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
-                    else:
-                        assert residx == 1
-                        # r = (r[0], s[1])
-                        if resloc != srcloc:
-                            self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
-                            self.mc.UNPCKHPD(resloc, srcloc)
-                        # if they are equal nothing is to be done
-
-    genop_vec_float_unpack = genop_vec_float_pack
-
-    def genop_vec_cast_float_to_singlefloat(self, op, arglocs, resloc):
-        self.mc.CVTPD2PS(resloc, arglocs[0])
-
-    def genop_vec_cast_float_to_int(self, op, arglocs, resloc):
-        self.mc.CVTPD2DQ(resloc, arglocs[0])
-
-    def genop_vec_cast_int_to_float(self, op, arglocs, resloc):
-        self.mc.CVTDQ2PD(resloc, arglocs[0])
-
-    def genop_vec_cast_singlefloat_to_float(self, op, arglocs, resloc):
-        self.mc.CVTPS2PD(resloc, arglocs[0])
-
     # ________________________________________
 
 genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST
@@ -2923,7 +2520,10 @@
 genop_tlref_list = {}
 genop_guard_list = [Assembler386.not_implemented_op_guard] * rop._LAST
 
-for name, value in Assembler386.__dict__.iteritems():
+import itertools
+iterate = itertools.chain(Assembler386.__dict__.iteritems(),
+                          VectorAssemblerMixin.__dict__.iteritems())
+for name, value in iterate:
     if name.startswith('genop_discard_'):
         opname = name[len('genop_discard_'):]
         num = getattr(rop, opname.upper())
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -19,6 +19,7 @@
     ebp, r8, r9, r10, r11, r12, r13, r14, r15, xmm0, xmm1, xmm2, xmm3, xmm4,
     xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
     X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG)
+from rpython.jit.backend.x86.vector_ext import VectorRegallocMixin
 from rpython.jit.codewriter import longlong
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
@@ -146,7 +147,7 @@
     gpr_reg_mgr_cls.all_reg_indexes[_reg.value] = _i
 
 
-class RegAlloc(BaseRegalloc):
+class RegAlloc(BaseRegalloc, VectorRegallocMixin):
 
     def __init__(self, assembler, translate_support_code=False):
         assert isinstance(translate_support_code, bool)
@@ -1503,190 +1504,10 @@
             self.rm.possibly_free_var(length_box)
             self.rm.possibly_free_var(dstaddr_box)
 
-    # vector operations
-    # ________________________________________
-
-    def consider_vec_getarrayitem_raw(self, op):
-        descr = op.getdescr()
-        assert isinstance(descr, ArrayDescr)
-        assert not descr.is_array_of_pointers() and \
-               not descr.is_array_of_structs()
-        itemsize, ofs, _ = unpack_arraydescr(descr)
-        integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
-        aligned = False
-        args = op.getarglist()
-        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
-        result_loc = self.force_allocate_reg(op.result)
-        self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs),
-                          imm(integer), imm(aligned)], result_loc)
-
-    consider_vec_raw_load = consider_vec_getarrayitem_raw
-
-    def consider_vec_setarrayitem_raw(self, op):
-        descr = op.getdescr()
-        assert isinstance(descr, ArrayDescr)
-        assert not descr.is_array_of_pointers() and \
-               not descr.is_array_of_structs()
-        itemsize, ofs, _ = unpack_arraydescr(descr)
-        args = op.getarglist()
-        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
-        ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
-
-        integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
-        aligned = False
-        self.perform_discard(op, [base_loc, ofs_loc, value_loc,
-                                 imm(itemsize), imm(ofs), imm(integer), imm(aligned)])
-
-    consider_vec_raw_store = consider_vec_setarrayitem_raw
-
-    def consider_vec_arith(self, op):
-        lhs = op.getarg(0)
-        assert isinstance(lhs, BoxVector)
-        size = lhs.item_size
-        args = op.getarglist()
-        loc1 = self.make_sure_var_in_reg(op.getarg(1), args)
-        loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
-        self.perform(op, [loc0, loc1, imm(size)], loc0)
-
-    consider_vec_int_add = consider_vec_arith
-    consider_vec_int_sub = consider_vec_arith
-    consider_vec_int_mul = consider_vec_arith
-    consider_vec_float_add = consider_vec_arith
-    consider_vec_float_sub = consider_vec_arith
-    consider_vec_float_mul = consider_vec_arith
-    consider_vec_float_truediv = consider_vec_arith
-    del consider_vec_arith
-
-    def consider_vec_arith_unary(self, op):
-        lhs = op.getarg(0)
-        assert isinstance(lhs, BoxVector)
-        size = lhs.item_size
-        args = op.getarglist()
-        res = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
-        self.perform(op, [res, imm(size)], res)
-
-    consider_vec_float_neg = consider_vec_arith_unary
-    consider_vec_float_abs = consider_vec_arith_unary
-    del consider_vec_arith_unary
-
-    def consider_vec_logic(self, op):
-        lhs = op.getarg(0)
-        assert isinstance(lhs, BoxVector)
-        size = lhs.item_size
-        args = op.getarglist()
-        source = self.make_sure_var_in_reg(op.getarg(1), args)
-        result = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
-        self.perform(op, [source, imm(size)], result)
-
-    consider_vec_float_eq = consider_vec_logic
-    consider_vec_int_and = consider_vec_logic
-    consider_vec_int_or = consider_vec_logic
-    consider_vec_int_xor = consider_vec_logic
-    del consider_vec_logic
-
-    def consider_vec_int_pack(self, op):
-        # new_res = vec_int_pack(res, src, index, count)
-        arg = op.getarg(1)
-        index = op.getarg(2)
-        count = op.getarg(3)
-        assert isinstance(index, ConstInt)
-        assert isinstance(count, ConstInt)
-        args = op.getarglist()
-        srcloc = self.make_sure_var_in_reg(arg, args)
-        resloc =  self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
-        residx = index.value # where to put it in result?
-        srcidx = 0
-        assert isinstance(op.result, BoxVector)
-        size = op.result.getsize()
-        arglocs = [resloc, srcloc, imm(residx), imm(srcidx), imm(count.value), imm(size)]
-        self.perform(op, arglocs, resloc)
-
-    consider_vec_float_pack = consider_vec_int_pack
-
-    def consider_vec_int_unpack(self, op):
-        index = op.getarg(1)
-        count = op.getarg(2)
-        assert isinstance(index, ConstInt)
-        assert isinstance(count, ConstInt)
-        args = op.getarglist()
-        srcloc = self.make_sure_var_in_reg(op.getarg(0), args)
-        if isinstance(op.result, BoxVector):
-            resloc =  self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
-            assert isinstance(op.result, BoxVector)
-            size = op.result.getsize()
-        else:
-            # unpack into iX box
-            resloc =  self.force_allocate_reg(op.result, args)
-            arg = op.getarg(0)
-            assert isinstance(arg, BoxVector)
-            size = arg.getsize()
-        residx = 0
-        args = op.getarglist()
-        arglocs = [resloc, srcloc, imm(residx), imm(index.value), imm(count.value), imm(size)]
-        self.perform(op, arglocs, resloc)
-
-    consider_vec_float_unpack = consider_vec_int_unpack
-
-    def consider_vec_float_expand(self, op):
-        result = op.result
-        assert isinstance(result, BoxVector)
-        arg = op.getarg(0)
-        args = op.getarglist()
-        if isinstance(arg, Const):
-            resloc = self.xrm.force_allocate_reg(result)
-            srcloc = self.xrm.expand_float(result.getsize(), arg)
-        else:
-            resloc = self.xrm.force_result_in_reg(op.result, arg, args)
-            srcloc = resloc
-
-        size = op.result.getsize()
-        self.perform(op, [srcloc, imm(size)], resloc)
-
-    def consider_vec_int_expand(self, op):
-        arg = op.getarg(0)
-        args = op.getarglist()
-        if isinstance(arg, Const):
-            srcloc = self.rm.convert_to_imm(arg)
-        else:
-            srcloc = self.make_sure_var_in_reg(arg, args)
-        resloc = self.xrm.force_allocate_reg(op.result, args)
-        assert isinstance(op.result, BoxVector)
-        size = op.result.getsize()
-        self.perform(op, [srcloc, imm(size)], resloc)
-
-    def consider_vec_int_signext(self, op):
-        args = op.getarglist()
-        resloc = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
-        sizearg = op.getarg(0)
-        result = op.result
-        assert isinstance(sizearg, BoxVector)
-        assert isinstance(result, BoxVector)
-        size = sizearg.getsize()
-        tosize = result.getsize()
-        self.perform(op, [resloc, imm(size), imm(tosize)], resloc)
-
-    def consider_vec_box(self, op):
-        # pseudo instruction, needed to create a new variable
-        self.xrm.force_allocate_reg(op.result)
-
-    def consider_guard_early_exit(self, op):
-        pass
-
-    def consider_vec_cast_float_to_int(self, op):
-        args = op.getarglist()
-        srcloc = self.make_sure_var_in_reg(op.getarg(0), args)
-        resloc = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
-        self.perform(op, [srcloc], resloc)
-
-    consider_vec_cast_int_to_float = consider_vec_cast_float_to_int
-    consider_vec_cast_float_to_singlefloat = consider_vec_cast_float_to_int
-    consider_vec_cast_singlefloat_to_float = consider_vec_cast_float_to_int
-
     # ________________________________________
 
     def not_implemented_op(self, op):
+        import pdb; pdb.set_trace()
         not_implemented("not implemented operation: %s" % op.getopname())
 
     def not_implemented_op_with_guard(self, op, guard_op):
@@ -1699,7 +1520,10 @@
 def add_none_argument(fn):
     return lambda self, op: fn(self, op, None)
 
-for name, value in RegAlloc.__dict__.iteritems():
+import itertools
+iterate = itertools.chain(RegAlloc.__dict__.iteritems(),
+                          VectorRegallocMixin.__dict__.iteritems())
+for name, value in iterate:
     if name.startswith('consider_'):
         name = name[len('consider_'):]
         num = getattr(rop, name.upper())
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -0,0 +1,610 @@
+import py
+from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
+    ConstFloat, BoxInt, BoxFloat, BoxVector, BoxVectorAccum, INT, REF,
+    FLOAT, VECTOR, TargetToken)
+from rpython.jit.backend.llsupport.descr import (ArrayDescr, CallDescr,
+    unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr)
+from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
+    FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
+    ebp, r8, r9, r10, r11, r12, r13, r14, r15, xmm0, xmm1, xmm2, xmm3, xmm4,
+    xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
+    X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, AddressLoc)
+
+def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0):
+    # duplicated for easy migration, def in assembler.py as well
+    return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset)
+
+class VectorAssemblerMixin(object):
+    _mixin_ = True
+
+    def _guard_vector_true(self, guard_op, loc, zero=False):
+        arg = guard_op.getarg(0)
+        assert isinstance(arg, BoxVector)
+        size = arg.item_size
+        temp = X86_64_XMM_SCRATCH_REG
+        #
+        self.mc.PXOR(temp, temp)
+        # if the vector is not fully packed blend 1s
+        if not arg.fully_packed(self.cpu.vector_register_size):
+            self.mc.PCMPEQQ(temp, temp) # fill with ones
+            select = 0
+            bits_used = (arg.item_count * arg.item_size * 8)
+            index = bits_used // 16
+            while index < 8:
+                select |= (1 << index)
+                index += 1
+            self.mc.PBLENDW_xxi(loc.value, temp.value, select)
+            # reset to zeros
+            self.mc.PXOR(temp, temp)
+
+        self.mc.PCMPEQ(size, loc, temp)
+        self.mc.PCMPEQQ(temp, temp)
+        self.mc.PTEST(loc, temp)
+
+    # vector operations
+    # ________________________________________
+
+    def _accum_update_at_exit(self, fail_locs, fail_args, faildescr, regalloc):
+        """ If accumulation is done in this loop, at the guard exit
+        some vector registers must be adjusted to yield the correct value"""
+        assert regalloc is not None
+        accum_info = faildescr.rd_accum_list
+        while accum_info:
+            pos = accum_info.position
+            loc = fail_locs[pos]
+            assert isinstance(loc, RegLoc)
+            arg = fail_args[pos]
+            if isinstance(arg, BoxVectorAccum):
+                arg = arg.scalar_var
+            assert arg is not None
+            tgtloc = regalloc.force_allocate_reg(arg, fail_args)
+            if accum_info.operation == '+':
+                # reduction using plus
+                self._accum_reduce_sum(arg, loc, tgtloc)
+            elif accum_info.operation == '*':
+                self._accum_reduce_mul(arg, loc, tgtloc)
+            else:
+                not_implemented("accum operator %s not implemented" %
+                                            (accum_info.operation)) 
+            fail_locs[pos] = tgtloc
+            regalloc.possibly_free_var(arg)
+            accum_info = accum_info.prev
+
+    def _accum_reduce_mul(self, arg, accumloc, targetloc):
+        scratchloc = X86_64_XMM_SCRATCH_REG
+        self.mov(accumloc, scratchloc)
+        # swap the two elements
+        self.mc.SHUFPD_xxi(scratchloc.value, scratchloc.value, 0x01)
+        self.mc.MULSD(accumloc, scratchloc)
+        if accumloc is not targetloc:
+            self.mov(accumloc, targetloc)
+
+    def _accum_reduce_sum(self, arg, accumloc, targetloc):
+        # Currently the accumulator can ONLY be the biggest
+        # size for X86 -> 64 bit float/int
+        if arg.type == FLOAT:
+            # r = (r[0]+r[1],r[0]+r[1])
+            self.mc.HADDPD(accumloc, accumloc)
+            # upper bits (> 64) are dirty (but does not matter)
+            if accumloc is not targetloc:
+                self.mov(accumloc, targetloc)
+            return
+        elif arg.type == INT:
+            scratchloc = X86_64_SCRATCH_REG
+            self.mc.PEXTRQ_rxi(targetloc.value, accumloc.value, 0)
+            self.mc.PEXTRQ_rxi(scratchloc.value, accumloc.value, 1)
+            self.mc.ADD(targetloc, scratchloc)
+            return
+
+        not_implemented("reduce sum for %s not impl." % arg)
+
+    def genop_vec_getarrayitem_raw(self, op, arglocs, resloc):
+        # considers item scale (raw_load does not)
+        base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
+        scale = get_scale(size_loc.value)
+        src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale)
+        self._vec_load(resloc, src_addr, integer_loc.value,
+                       size_loc.value, aligned_loc.value)
+
+    def genop_vec_raw_load(self, op, arglocs, resloc):
+        base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
+        src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0)
+        self._vec_load(resloc, src_addr, integer_loc.value,
+                       size_loc.value, aligned_loc.value)
+
+    def _vec_load(self, resloc, src_addr, integer, itemsize, aligned):
+        if integer:
+            if aligned:
+                self.mc.MOVDQA(resloc, src_addr)
+            else:
+                self.mc.MOVDQU(resloc, src_addr)
+        else:
+            if itemsize == 4:
+                self.mc.MOVUPS(resloc, src_addr)
+            elif itemsize == 8:
+                self.mc.MOVUPD(resloc, src_addr)
+
+    def genop_discard_vec_setarrayitem_raw(self, op, arglocs):
+        # considers item scale (raw_store does not)
+        base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
+        scale = get_scale(size_loc.value)
+        dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, scale)
+        self._vec_store(dest_loc, value_loc, integer_loc.value,
+                        size_loc.value, aligned_loc.value)
+
+    def genop_discard_vec_raw_store(self, op, arglocs):
+        base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
+        dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0)
+        self._vec_store(dest_loc, value_loc, integer_loc.value,
+                        size_loc.value, aligned_loc.value)
+
+    def _vec_store(self, dest_loc, value_loc, integer, itemsize, aligned):
+        if integer:
+            if aligned:
+                self.mc.MOVDQA(dest_loc, value_loc)
+            else:
+                self.mc.MOVDQU(dest_loc, value_loc)
+        else:
+            if itemsize == 4:
+                self.mc.MOVUPS(dest_loc, value_loc)
+            elif itemsize == 8:
+                self.mc.MOVUPD(dest_loc, value_loc)
+
+    def genop_vec_int_mul(self, op, arglocs, resloc):
+        loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 2:
+            self.mc.PMULLW(loc0, loc1)
+        elif itemsize == 4:
+            self.mc.PMULLD(loc0, loc1)
+        else:
+            # NOTE see http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025
+            # There is no 64x64 bit packed mul and I did not find one
+            # for 8 bit either. It is questionable if it gives any benefit
+            # for 8 bit.
+            not_implemented("int8/64 mul")
+
+    def genop_vec_int_add(self, op, arglocs, resloc):
+        loc0, loc1, size_loc = arglocs
+        size = size_loc.value
+        if size == 1:
+            self.mc.PADDB(loc0, loc1)
+        elif size == 2:
+            self.mc.PADDW(loc0, loc1)
+        elif size == 4:
+            self.mc.PADDD(loc0, loc1)
+        elif size == 8:
+            self.mc.PADDQ(loc0, loc1)
+
+    def genop_vec_int_sub(self, op, arglocs, resloc):
+        loc0, loc1, size_loc = arglocs
+        size = size_loc.value
+        if size == 1:
+            self.mc.PSUBB(loc0, loc1)
+        elif size == 2:
+            self.mc.PSUBW(loc0, loc1)
+        elif size == 4:
+            self.mc.PSUBD(loc0, loc1)
+        elif size == 8:
+            self.mc.PSUBQ(loc0, loc1)
+
+    def genop_vec_int_and(self, op, arglocs, resloc):
+        self.mc.PAND(resloc, arglocs[0])
+
+    def genop_vec_int_or(self, op, arglocs, resloc):
+        self.mc.POR(resloc, arglocs[0])
+
+    def genop_vec_int_xor(self, op, arglocs, resloc):
+        self.mc.PXOR(resloc, arglocs[0])
+
+    genop_vec_float_arith = """
+    def genop_vec_float_{type}(self, op, arglocs, resloc):
+        loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.{p_op_s}(loc0, loc1)
+        elif itemsize == 8:
+            self.mc.{p_op_d}(loc0, loc1)
+    """
+    for op in ['add','mul','sub']:
+        OP = op.upper()
+        _source = genop_vec_float_arith.format(type=op,
+                                               p_op_s=OP+'PS',
+                                               p_op_d=OP+'PD')
+        exec py.code.Source(_source).compile()
+    del genop_vec_float_arith
+
+    def genop_vec_float_truediv(self, op, arglocs, resloc):
+        loc0, loc1, sizeloc = arglocs
+        size = sizeloc.value
+        if size == 4:
+            self.mc.DIVPS(loc0, loc1)
+        elif size == 8:
+            self.mc.DIVPD(loc0, loc1)
+
+    def genop_vec_float_abs(self, op, arglocs, resloc):
+        src, sizeloc = arglocs
+        size = sizeloc.value
+        if size == 4:
+            self.mc.ANDPS(src, heap(self.single_float_const_abs_addr))
+        elif size == 8:
+            self.mc.ANDPD(src, heap(self.float_const_abs_addr))
+
+    def genop_vec_float_neg(self, op, arglocs, resloc):
+        src, sizeloc = arglocs
+        size = sizeloc.value
+        if size == 4:
+            self.mc.XORPS(src, heap(self.single_float_const_neg_addr))
+        elif size == 8:
+            self.mc.XORPD(src, heap(self.float_const_neg_addr))
+
+    def genop_vec_int_signext(self, op, arglocs, resloc):
+        srcloc, sizeloc, tosizeloc = arglocs
+        size = sizeloc.value
+        tosize = tosizeloc.value
+        if size == tosize:
+            return # already the right size
+        if size == 4 and tosize == 8:
+            scratch = X86_64_SCRATCH_REG.value
+            self.mc.PEXTRD_rxi(scratch, srcloc.value, 1)
+            self.mc.PINSRQ_xri(resloc.value, scratch, 1)
+            self.mc.PEXTRD_rxi(scratch, srcloc.value, 0)
+            self.mc.PINSRQ_xri(resloc.value, scratch, 0)
+        elif size == 8 and tosize == 4:
+            # is there a better sequence to move them?
+            scratch = X86_64_SCRATCH_REG.value
+            self.mc.PEXTRQ_rxi(scratch, srcloc.value, 0)
+            self.mc.PINSRD_xri(resloc.value, scratch, 0)
+            self.mc.PEXTRQ_rxi(scratch, srcloc.value, 1)
+            self.mc.PINSRD_xri(resloc.value, scratch, 1)
+        else:
+            # note that all other conversions are not implemented
+            # on purpose. it needs many x86 op codes to implement
+            # the missing combinations. even if they are implemented
+            # the speedup might only be modest...
+            # the optimization does not emit such code!
+            msg = "vec int signext (%d->%d)" % (size, tosize)
+            not_implemented(msg)
+
+    def genop_vec_float_expand(self, op, arglocs, resloc):
+        srcloc, sizeloc = arglocs
+        size = sizeloc.value
+        if isinstance(srcloc, ConstFloatLoc):
+            # they are aligned!
+            self.mc.MOVAPD(resloc, srcloc)
+        elif size == 4:
+            # the register allocator forces src to be the same as resloc
+            # r = (s[0], s[0], r[0], r[0])
+            # since resloc == srcloc: r = (r[0], r[0], r[0], r[0])
+            self.mc.SHUFPS_xxi(resloc.value, srcloc.value, 0)
+        elif size == 8:
+            self.mc.MOVDDUP(resloc, srcloc)
+        else:
+            raise AssertionError("float of size %d not supported" % (size,))
+
+    def genop_vec_int_expand(self, op, arglocs, resloc):
+        srcloc, sizeloc = arglocs
+        if not isinstance(srcloc, RegLoc):
+            self.mov(srcloc, X86_64_SCRATCH_REG)
+            srcloc = X86_64_SCRATCH_REG
+        assert not srcloc.is_xmm
+        size = sizeloc.value
+        if size == 1:
+            self.mc.PINSRB_xri(resloc.value, srcloc.value, 0)
+            self.mc.PSHUFB(resloc, heap(self.expand_byte_mask_addr))
+        elif size == 2:
+            self.mc.PINSRW_xri(resloc.value, srcloc.value, 0)
+            self.mc.PINSRW_xri(resloc.value, srcloc.value, 4)
+            self.mc.PSHUFLW_xxi(resloc.value, resloc.value, 0)
+            self.mc.PSHUFHW_xxi(resloc.value, resloc.value, 0)
+        elif size == 4:
+            self.mc.PINSRD_xri(resloc.value, srcloc.value, 0)
+            self.mc.PSHUFD_xxi(resloc.value, resloc.value, 0)
+        elif size == 8:
+            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0)
+            self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1)
+        else:
+            raise AssertionError("cannot handle size %d (int expand)" % (size,))
+
+    def genop_vec_int_pack(self, op, arglocs, resloc):
+        resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
+        assert isinstance(resultloc, RegLoc)
+        assert isinstance(sourceloc, RegLoc)
+        size = sizeloc.value
+        srcidx = srcidxloc.value
+        residx = residxloc.value
+        count = countloc.value
+        # for small data type conversion this can be quite costy
+        # NOTE there might be some combinations that can be handled
+        # more efficiently! e.g.
+        # v2 = pack(v0,v1,4,4)
+        si = srcidx
+        ri = residx
+        k = count
+        while k > 0:
+            if size == 8:
+                if resultloc.is_xmm and sourceloc.is_xmm: # both xmm
+                    self.mc.PEXTRQ_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRQ_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                elif resultloc.is_xmm: # xmm <- reg
+                    self.mc.PINSRQ_xri(resultloc.value, sourceloc.value, ri)
+                else: # reg <- xmm
+                    self.mc.PEXTRQ_rxi(resultloc.value, sourceloc.value, si)
+            elif size == 4:
+                if resultloc.is_xmm and sourceloc.is_xmm:
+                    self.mc.PEXTRD_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRD_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                elif resultloc.is_xmm:
+                    self.mc.PINSRD_xri(resultloc.value, sourceloc.value, ri)
+                else:
+                    self.mc.PEXTRD_rxi(resultloc.value, sourceloc.value, si)
+            elif size == 2:
+                if resultloc.is_xmm and sourceloc.is_xmm:
+                    self.mc.PEXTRW_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRW_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                elif resultloc.is_xmm:
+                    self.mc.PINSRW_xri(resultloc.value, sourceloc.value, ri)
+                else:
+                    self.mc.PEXTRW_rxi(resultloc.value, sourceloc.value, si)
+            elif size == 1:
+                if resultloc.is_xmm and sourceloc.is_xmm:
+                    self.mc.PEXTRB_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+                    self.mc.PINSRB_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+                elif resultloc.is_xmm:
+                    self.mc.PINSRB_xri(resultloc.value, sourceloc.value, ri)
+                else:
+                    self.mc.PEXTRB_rxi(resultloc.value, sourceloc.value, si)
+            si += 1
+            ri += 1
+            k -= 1
+
+    genop_vec_int_unpack = genop_vec_int_pack
+
+    def genop_vec_float_pack(self, op, arglocs, resultloc):
+        resloc, srcloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
+        assert isinstance(resloc, RegLoc)
+        assert isinstance(srcloc, RegLoc)
+        count = countloc.value
+        residx = residxloc.value
+        srcidx = srcidxloc.value
+        size = sizeloc.value
+        if size == 4:
+            si = srcidx
+            ri = residx
+            k = count
+            while k > 0:
+                if resloc.is_xmm:
+                    src = srcloc.value
+                    if not srcloc.is_xmm:
+                        # if source is a normal register (unpack)
+                        assert count == 1
+                        assert si == 0
+                        self.mov(srcloc, X86_64_XMM_SCRATCH_REG)
+                        src = X86_64_XMM_SCRATCH_REG.value
+                    select = ((si & 0x3) << 6)|((ri & 0x3) << 4)
+                    self.mc.INSERTPS_xxi(resloc.value, src, select)
+                else:
+                    self.mc.PEXTRD_rxi(resloc.value, srcloc.value, si)
+                si += 1
+                ri += 1
+                k -= 1
+        elif size == 8:
+            assert resloc.is_xmm
+            if srcloc.is_xmm:
+                if srcidx == 0:
+                    if residx == 0:
+                        # r = (s[0], r[1])
+                        self.mc.MOVSD(resloc, srcloc)
+                    else:
+                        assert residx == 1
+                        # r = (r[0], s[0])
+                        self.mc.UNPCKLPD(resloc, srcloc)
+                else:
+                    assert srcidx == 1
+                    if residx == 0:
+                        # r = (s[1], r[1])
+                        if resloc != srcloc:
+                            self.mc.UNPCKHPD(resloc, srcloc)
+                        self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
+                    else:
+                        assert residx == 1
+                        # r = (r[0], s[1])
+                        if resloc != srcloc:
+                            self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
+                            self.mc.UNPCKHPD(resloc, srcloc)
+                        # if they are equal nothing is to be done
+
+    genop_vec_float_unpack = genop_vec_float_pack
+
+    def genop_vec_cast_float_to_singlefloat(self, op, arglocs, resloc):
+        self.mc.CVTPD2PS(resloc, arglocs[0])
+
+    def genop_vec_cast_float_to_int(self, op, arglocs, resloc):
+        self.mc.CVTPD2DQ(resloc, arglocs[0])
+
+    def genop_vec_cast_int_to_float(self, op, arglocs, resloc):
+        self.mc.CVTDQ2PD(resloc, arglocs[0])
+
+    def genop_vec_cast_singlefloat_to_float(self, op, arglocs, resloc):
+        self.mc.CVTPS2PD(resloc, arglocs[0])
+
+class VectorRegallocMixin(object):
+    _mixin_ = True
+
+    def consider_vec_getarrayitem_raw(self, op):
+        descr = op.getdescr()
+        assert isinstance(descr, ArrayDescr)
+        assert not descr.is_array_of_pointers() and \
+               not descr.is_array_of_structs()
+        itemsize, ofs, _ = unpack_arraydescr(descr)
+        integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
+        aligned = False
+        args = op.getarglist()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+        result_loc = self.force_allocate_reg(op.result)
+        self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs),
+                          imm(integer), imm(aligned)], result_loc)
+
+    consider_vec_raw_load = consider_vec_getarrayitem_raw
+
+    def consider_vec_setarrayitem_raw(self, op):
+        descr = op.getdescr()
+        assert isinstance(descr, ArrayDescr)
+        assert not descr.is_array_of_pointers() and \
+               not descr.is_array_of_structs()
+        itemsize, ofs, _ = unpack_arraydescr(descr)
+        args = op.getarglist()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+        ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+
+        integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
+        aligned = False
+        self.perform_discard(op, [base_loc, ofs_loc, value_loc,
+                                 imm(itemsize), imm(ofs), imm(integer), imm(aligned)])
+
+    consider_vec_raw_store = consider_vec_setarrayitem_raw
+
+    def consider_vec_arith(self, op):
+        lhs = op.getarg(0)
+        assert isinstance(lhs, BoxVector)
+        size = lhs.item_size
+        args = op.getarglist()
+        loc1 = self.make_sure_var_in_reg(op.getarg(1), args)
+        loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
+        self.perform(op, [loc0, loc1, imm(size)], loc0)
+
+    consider_vec_int_add = consider_vec_arith
+    consider_vec_int_sub = consider_vec_arith
+    consider_vec_int_mul = consider_vec_arith
+    consider_vec_float_add = consider_vec_arith
+    consider_vec_float_sub = consider_vec_arith
+    consider_vec_float_mul = consider_vec_arith
+    consider_vec_float_truediv = consider_vec_arith
+    del consider_vec_arith
+
+    def consider_vec_arith_unary(self, op):
+        lhs = op.getarg(0)
+        assert isinstance(lhs, BoxVector)
+        size = lhs.item_size
+        args = op.getarglist()
+        res = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
+        self.perform(op, [res, imm(size)], res)
+
+    consider_vec_float_neg = consider_vec_arith_unary
+    consider_vec_float_abs = consider_vec_arith_unary
+    del consider_vec_arith_unary
+
+    def consider_vec_logic(self, op):
+        lhs = op.getarg(0)
+        assert isinstance(lhs, BoxVector)
+        size = lhs.item_size
+        args = op.getarglist()
+        source = self.make_sure_var_in_reg(op.getarg(1), args)
+        result = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
+        self.perform(op, [source, imm(size)], result)
+
+    consider_vec_float_eq = consider_vec_logic
+    consider_vec_int_and = consider_vec_logic
+    consider_vec_int_or = consider_vec_logic
+    consider_vec_int_xor = consider_vec_logic
+    del consider_vec_logic
+
+    def consider_vec_int_pack(self, op):
+        # new_res = vec_int_pack(res, src, index, count)
+        arg = op.getarg(1)
+        index = op.getarg(2)
+        count = op.getarg(3)
+        assert isinstance(index, ConstInt)
+        assert isinstance(count, ConstInt)
+        args = op.getarglist()
+        srcloc = self.make_sure_var_in_reg(arg, args)
+        resloc =  self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
+        residx = index.value # where to put it in result?
+        srcidx = 0
+        assert isinstance(op.result, BoxVector)
+        size = op.result.getsize()
+        arglocs = [resloc, srcloc, imm(residx), imm(srcidx), imm(count.value), imm(size)]
+        self.perform(op, arglocs, resloc)
+
+    consider_vec_float_pack = consider_vec_int_pack
+
+    def consider_vec_int_unpack(self, op):
+        index = op.getarg(1)
+        count = op.getarg(2)
+        assert isinstance(index, ConstInt)
+        assert isinstance(count, ConstInt)
+        args = op.getarglist()
+        srcloc = self.make_sure_var_in_reg(op.getarg(0), args)
+        if isinstance(op.result, BoxVector):
+            resloc =  self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
+            assert isinstance(op.result, BoxVector)
+            size = op.result.getsize()
+        else:
+            # unpack into iX box
+            resloc =  self.force_allocate_reg(op.result, args)
+            arg = op.getarg(0)
+            assert isinstance(arg, BoxVector)
+            size = arg.getsize()
+        residx = 0
+        args = op.getarglist()
+        arglocs = [resloc, srcloc, imm(residx), imm(index.value), imm(count.value), imm(size)]
+        self.perform(op, arglocs, resloc)
+
+    consider_vec_float_unpack = consider_vec_int_unpack
+
+    def consider_vec_float_expand(self, op):
+        result = op.result
+        assert isinstance(result, BoxVector)
+        arg = op.getarg(0)
+        args = op.getarglist()
+        if isinstance(arg, Const):
+            resloc = self.xrm.force_allocate_reg(result)
+            srcloc = self.xrm.expand_float(result.getsize(), arg)
+        else:
+            resloc = self.xrm.force_result_in_reg(op.result, arg, args)
+            srcloc = resloc
+
+        size = op.result.getsize()
+        self.perform(op, [srcloc, imm(size)], resloc)
+
+    def consider_vec_int_expand(self, op):
+        arg = op.getarg(0)
+        args = op.getarglist()
+        if isinstance(arg, Const):
+            srcloc = self.rm.convert_to_imm(arg)
+        else:
+            srcloc = self.make_sure_var_in_reg(arg, args)
+        resloc = self.xrm.force_allocate_reg(op.result, args)
+        assert isinstance(op.result, BoxVector)
+        size = op.result.getsize()
+        self.perform(op, [srcloc, imm(size)], resloc)
+
+    def consider_vec_int_signext(self, op):
+        args = op.getarglist()
+        resloc = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
+        sizearg = op.getarg(0)
+        result = op.result
+        assert isinstance(sizearg, BoxVector)
+        assert isinstance(result, BoxVector)
+        size = sizearg.getsize()
+        tosize = result.getsize()
+        self.perform(op, [resloc, imm(size), imm(tosize)], resloc)
+
+    def consider_vec_box(self, op):
+        # pseudo instruction, needed to create a new variable
+        self.xrm.force_allocate_reg(op.result)
+
+    def consider_guard_early_exit(self, op):
+        pass
+
+    def consider_vec_cast_float_to_int(self, op):
+        args = op.getarglist()
+        srcloc = self.make_sure_var_in_reg(op.getarg(0), args)
+        resloc = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
+        self.perform(op, [srcloc], resloc)
+
+    consider_vec_cast_int_to_float = consider_vec_cast_float_to_int
+    consider_vec_cast_float_to_singlefloat = consider_vec_cast_float_to_int
+    consider_vec_cast_singlefloat_to_float = consider_vec_cast_float_to_int
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -163,10 +163,17 @@
             return None
 
         loop.operations = loop.operations[:-1] + part.operations
+        loop.versions = part.versions
         if part.quasi_immutable_deps:
             loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
     assert part.operations[-1].getopnum() != rop.LABEL
 
+    if loop.versions is not None:
+        # several different loop version have been generated
+        for version in loop.versions:
+            token = version.update_token(jitcell_token)
+            all_target_tokens.append(token)
+
     if not loop.quasi_immutable_deps:
         loop.quasi_immutable_deps = None
     for box in loop.inputargs:
@@ -181,8 +188,21 @@
     propagate_original_jitcell_token(loop)
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
     record_loop_or_bridge(metainterp_sd, loop)
+
+    generate_pending_loop_versions(loop, jitdriver_sd, metainterp_sd, jitcell_token)
+
     return all_target_tokens[0]
 
+def generate_pending_loop_versions(loop, jitdriver_sd, metainterp_sd, jitcell_token):
+    if loop.versions is not None:
+        token = jitcell_token
+        for version in loop.versions:
+            version.update_inputargs()
+            for faildescr  in version.faildescrs:
+                send_bridge_to_backend(jitdriver_sd, metainterp_sd,
+                                       faildescr, version.inputargs,
+                                       version.operations, jitcell_token)
+
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
                     partial_trace, resumekey, start_state):
@@ -689,6 +709,16 @@
 class ResumeAtLoopHeaderDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_EARLY_EXIT
 
+class CompileLoopVersionDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_EARLY_EXIT
+
+    operations = None
+    inputargs = None
+    faillocs = None
+
+    def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
+        assert 0, "this guard must never fail"
+
 class AllVirtuals:
     llopaque = True
     cache = None
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -695,12 +695,49 @@
     def repr_of_descr(self):
         return 'TargetToken(%d)' % compute_unique_id(self)
 
+class LoopVersion(object):
+
+    def __init__(self, loop, aligned=False):
+        self.operations = loop.operations
+        self.aligned = aligned
+        self.faildescrs = []
+        #
+        label = self.operations[0]
+        assert label.getopnum() == rop.LABEL
+        self.enter_args = label.getarglist()
+        self.calling_args = None
+        self.inputargs = None
+
+    def adddescr(self, descr):
+        self.faildescrs.append(descr)
+
+    def update_token(self, jitcell_token):
+        label = self.operations[0]
+        jump = self.operations[-1]
+        #
+        assert label.getopnum() == rop.LABEL
+        assert jump.getopnum() == rop.JUMP
+        #
+        token = TargetToken(jitcell_token)
+        token.original_jitcell_token = jitcell_token
+        label.setdescr(token)
+        jump.setdescr(token)
+        return token
+
+    def update_inputargs(self):
+        assert len(self.enter_args) == len(self.inputargs)
+        rename = { a: b for a,b in zip(self.enter_args, self.calling_args) }
+        for i, arg in enumerate(self.inputargs):
+            self.inputargs[i] = rename[arg]
+
+
 class TreeLoop(object):
     inputargs = None
     operations = None
     call_pure_results = None
     logops = None
     quasi_immutable_deps = None
+    versions = None
 
     def _token(*args):
         raise Exception("TreeLoop.token is killed")
@@ -817,6 +854,7 @@
     def __repr__(self):
         return '<%s>' % (self.name,)
 
+
 def _list_all_operations(result, operations, omit_finish=True):
     if omit_finish and operations[-1].getopnum() == rop.FINISH:
         # xxx obscure
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -104,26 +104,6 @@
     def can_be_relaxed(self):
         return self.op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE)
 
-    def relax_guard_to(self, guard):
-        """ Relaxes a guard operation to an earlier guard. """
-        # clone this operation object. if the vectorizer is
-        # not able to relax guards, it won't leave behind a modified operation
-        tgt_op = self.getoperation().clone()
-        self.op = tgt_op
-
-        op = guard.getoperation()
-        assert isinstance(tgt_op, GuardResOp)
-        assert isinstance(op, GuardResOp)
-        olddescr = op.getdescr()
-        descr = compile.ResumeAtLoopHeaderDescr()
-        if olddescr:
-            descr.copy_all_attributes_from(olddescr)
-        #
-        tgt_op.setdescr(descr)
-        tgt_op.rd_snapshot = op.rd_snapshot
-        #if not we_are_translated():
-        tgt_op.setfailargs(op.getfailargs())
-
     def edge_to(self, to, arg=None, failarg=False, label=None):
         if self is to:
             return
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -11,10 +11,11 @@
 from rpython.jit.metainterp.resume import Snapshot
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
 from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
-from rpython.jit.metainterp.compile import ResumeAtLoopHeaderDescr, invent_fail_descr_for_op
+from rpython.jit.metainterp.compile import (ResumeAtLoopHeaderDescr,
+        CompileLoopVersionDescr, invent_fail_descr_for_op)
 from rpython.jit.metainterp.history import (ConstInt, VECTOR, FLOAT, INT,
         BoxVector, BoxFloat, BoxInt, ConstFloat, TargetToken, JitCellToken, Box,
-        BoxVectorAccum)
+        BoxVectorAccum, LoopVersion)
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method, Renamer
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
@@ -50,6 +51,7 @@
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, False)
     orig_ops = loop.operations
+    orig_version = LoopVersion(loop)
     if len(orig_ops) >= 75:
         # if more than 75 operations are present in this loop,
         # it won't be possible to vectorize. There are too many
@@ -62,11 +64,16 @@
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "pre vectorize")
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZE_TRY)
         start = time.clock()
-        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, cost_threshold)
+        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, cost_threshold, orig_version)
         opt.propagate_all_forward()
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
         gso.propagate_all_forward(opt.loop)
         end = time.clock()
+
+        aligned_vector_version = LoopVersion(loop, aligned=True)
+
+        loop.versions = [orig_version] #, aligned_vector_version]
+
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "post vectorize")
         debug_stop("vec-opt-loop")
@@ -107,8 +114,9 @@
 class VectorizingOptimizer(Optimizer):
     """ Try to unroll the loop and find instructions to group """
 
-    def __init__(self, metainterp_sd, jitdriver_sd, loop, cost_threshold=0):
+    def __init__(self, metainterp_sd, jitdriver_sd, loop, cost_threshold, orig_loop_version):
         Optimizer.__init__(self, metainterp_sd, jitdriver_sd, loop, [])
+        self.orig_loop_version = orig_loop_version
         self.dependency_graph = None
         self.packset = None
         self.unroll_count = 0
@@ -188,6 +196,8 @@
 
         self.emit_unrolled_operation(label_op)
 
+        self.orig_loop_version.calling_args = label_op.getarglist()
+
         renamer = Renamer()
         oi = 0
         pure = True
@@ -247,7 +257,7 @@
                     assert isinstance(copied_op, GuardResOp)
                     target_guard = copied_op
                     # do not overwrite resume at loop header
-                    if not isinstance(target_guard.getdescr(), ResumeAtLoopHeaderDescr):
+                    if target_guard.getdescr().guard_opnum != rop.GUARD_EARLY_EXIT:
                         descr = invent_fail_descr_for_op(copied_op.getopnum(), self)
                         olddescr = copied_op.getdescr()
                         if olddescr:
@@ -573,7 +583,29 @@
                         label_node.edge_to(last_but_one, label='pullup')
                 # only the last guard needs a connection
                 guard_node.edge_to(ee_guard_node, label='pullup-last-guard')
-                guard_node.relax_guard_to(ee_guard_node)
+                self.relax_guard_to(guard_node, ee_guard_node)
+
+    def relax_guard_to(self, guard_node, other_node):
+        """ Relaxes a guard operation to an earlier guard. """
+        # clone this operation object. if the vectorizer is
+        # not able to relax guards, it won't leave behind a modified operation
+        tgt_op = guard_node.getoperation().clone()
+        guard_node.op = tgt_op
+
+        op = other_node.getoperation()
+        assert isinstance(tgt_op, GuardResOp)
+        assert isinstance(op, GuardResOp)
+        olddescr = op.getdescr()
+        descr = CompileLoopVersionDescr()
+        if olddescr:
+            descr.copy_all_attributes_from(olddescr)
+        self.orig_loop_version.inputargs = op.getfailargs()
+        self.orig_loop_version.adddescr(descr)
+        #
+        tgt_op.setdescr(descr)
+        tgt_op.rd_snapshot = op.rd_snapshot
+        tgt_op.setfailargs(op.getfailargs())
+
 
 class CostModel(object):
     def __init__(self, threshold, vec_reg_size):
@@ -754,17 +786,6 @@
 
         del self.packs[j]
         return len(self.packs)
-        # OLD
-        # instead of deleting an item in the center of pack array,
-        # the last element is assigned to position j and
-        # the last slot is freed. Order of packs doesn't matter
-        #last_pos = len(self.packs) - 1
-        #if j == last_pos:
-        #    del self.packs[j]
-        #else:
-        #    self.packs[j] = self.packs[last_pos]
-        #    del self.packs[last_pos]
-        #return last_pos
 
     def accumulates_pair(self, lnode, rnode, origin_pack):
         # lnode and rnode are isomorphic and dependent

From noreply at buildbot.pypy.org  Mon Jul  6 13:20:39 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon,  6 Jul 2015 13:20:39 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: advanced the bridge building from an
 early exit,
 nearly working but there are some problems to solve which accumulation and
 the input arguments...
Message-ID: <20150706112039.5803E1C0207@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78454:bdcb22955f5c
Date: 2015-07-06 13:20 +0200
http://bitbucket.org/pypy/pypy/changeset/bdcb22955f5c/

Log:	advanced the bridge building from an early exit, nearly working but
	there are some problems to solve which accumulation and the input
	arguments...

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -130,7 +130,7 @@
     def test_dot_matrix(self):
         result = self.run("dot_matrix")
         assert int(result) == 86
-        self.check_vectorized(1, 1)
+        self.check_vectorized(2, 1)
 
     def define_float32_copy():
         return """
@@ -529,13 +529,13 @@
         result = self.run("prod")
         assert int(result) == 576
         self.check_trace_count(1)
-        self.check_vectorized(1, 1)
+        self.check_vectorized(2, 1)
 
     def test_prod_zero(self):
         result = self.run("prod_zero")
         assert int(result) == 0
         self.check_trace_count(1)
-        self.check_vectorized(1, 1)
+        self.check_vectorized(2, 1)
 
 
     def define_max():
@@ -768,7 +768,7 @@
         result = self.run("setslice")
         assert result == 5.5
         self.check_trace_count(1)
-        self.check_vectorized(1, 1)
+        self.check_vectorized(2, 1)
 
     def define_virtual_slice():
         return """
@@ -834,7 +834,7 @@
     def test_dot(self):
         result = self.run("dot")
         assert result == 184
-        self.check_trace_count(5)
+        self.check_trace_count(4)
         self.check_vectorized(3,1)
 
     def define_argsort():
@@ -848,7 +848,7 @@
         result = self.run("argsort")
         assert result == 6
         self.check_trace_count(1)
-        self.check_vectorized(1,1) # vec. setslice
+        self.check_vectorized(2,1) # vec. setslice
 
     def define_where():
         return """
@@ -909,7 +909,7 @@
         result = self.run("slice")
         assert result == 18
         self.check_trace_count(1)
-        self.check_vectorized(1,1)
+        self.check_vectorized(2,1)
 
     def define_multidim_slice():
         return """
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -9,11 +9,17 @@
     ebp, r8, r9, r10, r11, r12, r13, r14, r15, xmm0, xmm1, xmm2, xmm3, xmm4,
     xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
     X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, AddressLoc)
+from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 
+# duplicated for easy migration, def in assembler.py as well
+# DUP START
 def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0):
-    # duplicated for easy migration, def in assembler.py as well
     return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset)
 
+def heap(addr):
+    return AddressLoc(ImmedLoc(addr), imm0, 0, 0)
+# DUP END
+
 class VectorAssemblerMixin(object):
     _mixin_ = True
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -189,19 +189,25 @@
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
     record_loop_or_bridge(metainterp_sd, loop)
 
-    generate_pending_loop_versions(loop, jitdriver_sd, metainterp_sd, jitcell_token)
+    generate_pending_loop_versions(loop, jitdriver_sd, metainterp, jitcell_token)
 
     return all_target_tokens[0]
 
-def generate_pending_loop_versions(loop, jitdriver_sd, metainterp_sd, jitcell_token):
+def generate_pending_loop_versions(loop, jitdriver_sd, metainterp, jitcell_token):
+    metainterp_sd = metainterp.staticdata
     if loop.versions is not None:
         token = jitcell_token
         for version in loop.versions:
-            version.update_inputargs()
-            for faildescr  in version.faildescrs:
+            versioned_loop = create_empty_loop(metainterp)
+            versioned_loop.inputargs = version.inputargs
+            versioned_loop.operations = version.operations
+            versioned_loop.original_jitcell_token = jitcell_token
+            for _, faildescr  in version.faildescrs:
                 send_bridge_to_backend(jitdriver_sd, metainterp_sd,
                                        faildescr, version.inputargs,
                                        version.operations, jitcell_token)
+                versioned_loop.original_jitcell_token = jitcell_token
+                record_loop_or_bridge(metainterp_sd, versioned_loop)
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
@@ -395,8 +401,7 @@
                            operations, original_loop_token):
     if not we_are_translated():
         show_procedures(metainterp_sd)
-        seen = dict.fromkeys(inputargs)
-        TreeLoop.check_consistency_of_branch(operations, seen)
+        TreeLoop.check_consistency_of_branch(operations, TreeLoop.seen_args(inputargs))
     if metainterp_sd.warmrunnerdesc is not None:
         hooks = metainterp_sd.warmrunnerdesc.hooks
         debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -702,17 +702,24 @@
         self.aligned = aligned
         self.faildescrs = []
         #
-        label = self.operations[0]
+        i = 0
+        label = self.operations[i]
+        while i < len(self.operations):
+            label = self.operations[i]
+            if label.getopnum() == rop.LABEL:
+                break
+            i += 1
         assert label.getopnum() == rop.LABEL
-        self.enter_args = label.getarglist()
-        self.calling_args = None
+        self.label_pos = i
+        self.parent_trace_label_args = None
+        self.bridge_label_args = label.getarglist()
         self.inputargs = None
 
-    def adddescr(self, descr):
-        self.faildescrs.append(descr)
+    def adddescr(self, op, descr):
+        self.faildescrs.append((op, descr))
 
     def update_token(self, jitcell_token):
-        label = self.operations[0]
+        label = self.operations[self.label_pos]
         jump = self.operations[-1]
         #
         assert label.getopnum() == rop.LABEL
@@ -722,15 +729,16 @@
         token.original_jitcell_token = jitcell_token
         label.setdescr(token)
         jump.setdescr(token)
+
+        assert len(self.bridge_label_args) <= len(self.parent_trace_label_args)
+        for i in range(len(self.bridge_label_args)):
+            arg = self.parent_trace_label_args[i]
+            if isinstance(arg, BoxVectorAccum):
+                self.bridge_label_args[i] = arg
+        self.inputargs = self.bridge_label_args
+
         return token
 
-    def update_inputargs(self):
-        assert len(self.enter_args) == len(self.inputargs)
-        rename = { a: b for a,b in zip(self.enter_args, self.calling_args) }
-        for i, arg in enumerate(self.inputargs):
-            self.inputargs[i] = rename[arg]
-
-
 class TreeLoop(object):
     inputargs = None
     operations = None
@@ -791,12 +799,23 @@
     def check_consistency_of(inputargs, operations):
         for box in inputargs:
             assert isinstance(box, Box), "Loop.inputargs contains %r" % (box,)
-        seen = dict.fromkeys(inputargs)
+        seen = TreeLoop.seen_args(inputargs)
         assert len(seen) == len(inputargs), (
                "duplicate Box in the Loop.inputargs")
         TreeLoop.check_consistency_of_branch(operations, seen)
 
     @staticmethod
+    def seen_args(inputargs):
+        seen = {}
+        for arg in inputargs:
+            if isinstance(arg, BoxVectorAccum):
+                seen[arg.scalar_var] = None
+                seen[arg] = None
+            else:
+                seen[arg] = None
+        return seen
+
+    @staticmethod
     def check_consistency_of_branch(operations, seen):
         "NOT_RPYTHON"
         for op in operations:
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -137,7 +137,9 @@
 
     def exits_early(self):
         if self.op.is_guard():
-            return isinstance(self.op.getdescr(), compile.ResumeAtLoopHeaderDescr)
+            descr = self.op.getdescr()
+            return isinstance(descr, compile.ResumeAtLoopHeaderDescr) or \
+                   isinstance(descr, compile.CompileLoopVersionDescr)
         return False
 
     def is_guard_early_exit(self):
@@ -292,7 +294,10 @@
         return None
 
     def __repr__(self):
-        return "Node(opidx: %d)" % self.opidx
+        pack = ''
+        if self.pack:
+            pack = "p: %d" % self.pack.opcount()
+        return "Node(%s,%s i: %d)" % (self.op.getopname(), pack, self.opidx)
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -625,7 +630,8 @@
         self.guard_argument_protection(guard_node, tracker)
         #
         descr = guard_op.getdescr()
-        if isinstance(descr, compile.ResumeAtLoopHeaderDescr):
+        if isinstance(descr, compile.ResumeAtLoopHeaderDescr) or \
+           isinstance(descr, compile.CompileLoopVersionDescr):
             return
         # handle fail args
         if guard_op.getfailargs():
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -36,6 +36,8 @@
             imp.import_value(value)
 
     def emit_operation(self, op):
+        if op.getopnum() == rop.GUARD_EARLY_EXIT:
+            return
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
         if self.emitting_dissabled:
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -168,6 +168,8 @@
     def emit_operation(self, op):
         if op.getopnum() == rop.DEBUG_MERGE_POINT:
             return
+        if op.getopnum() == rop.GUARD_EARLY_EXIT:
+            return
         self._last_emitted_op = op
         self._newoperations.append(op)
 
@@ -196,7 +198,7 @@
 
         self.emit_unrolled_operation(label_op)
 
-        self.orig_loop_version.calling_args = label_op.getarglist()
+        self.orig_loop_version.parent_trace_label_args = label_op.getarglist()[:]
 
         renamer = Renamer()
         oi = 0
@@ -583,9 +585,9 @@
                         label_node.edge_to(last_but_one, label='pullup')
                 # only the last guard needs a connection
                 guard_node.edge_to(ee_guard_node, label='pullup-last-guard')
-                self.relax_guard_to(guard_node, ee_guard_node)
+                self.relax_guard_to(guard_node, ee_guard_node, label_node)
 
-    def relax_guard_to(self, guard_node, other_node):
+    def relax_guard_to(self, guard_node, other_node, label_node):
         """ Relaxes a guard operation to an earlier guard. """
         # clone this operation object. if the vectorizer is
         # not able to relax guards, it won't leave behind a modified operation
@@ -599,12 +601,14 @@
         descr = CompileLoopVersionDescr()
         if olddescr:
             descr.copy_all_attributes_from(olddescr)
-        self.orig_loop_version.inputargs = op.getfailargs()
-        self.orig_loop_version.adddescr(descr)
         #
         tgt_op.setdescr(descr)
         tgt_op.rd_snapshot = op.rd_snapshot
         tgt_op.setfailargs(op.getfailargs())
+        if tgt_op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
+            self.orig_loop_version.adddescr(tgt_op, descr)
+            tgt_op.setfailargs(label_node.getoperation().getarglist()[:])
+            tgt_op.rd_snapshot = None
 
 
 class CostModel(object):
@@ -849,6 +853,7 @@
         for pack in self.packs:
             if not pack.is_accumulating():
                 continue
+            import pdb; pdb.set_trace()
             accum = pack.accum
             # create a new vector box for the parameters
             box = pack.input_type.new_vector_box()
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -63,7 +63,7 @@
     @py.test.mark.parametrize('i',[1,2,3,8,17,128,130,500,501,502,1300])
     def test_vectorize_array_get_set(self,i):
         myjitdriver = JitDriver(greens = [],
-                                reds = ['i','d','va','vb','vc'],
+                                reds = 'auto',
                                 vectorize=True)
         T = lltype.Array(rffi.INT, hints={'nolength': True})
         def f(d):
@@ -75,8 +75,7 @@
                 va[j] = rffi.r_int(j)
                 vb[j] = rffi.r_int(j)
             while i < d:
-                myjitdriver.can_enter_jit(i=i, d=d, va=va, vb=vb, vc=vc)
-                myjitdriver.jit_merge_point(i=i, d=d, va=va, vb=vb, vc=vc)
+                myjitdriver.jit_merge_point()
 
                 a = va[i]
                 b = vb[i]

From noreply at buildbot.pypy.org  Mon Jul  6 13:22:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 13:22:55 +0200 (CEST)
Subject: [pypy-commit] cffi default: Found a simpler and more efficient way
 to implement any ffi.gc().
Message-ID: <20150706112255.1CE691C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2212:68008f1b02e3
Date: 2015-07-06 13:20 +0200
http://bitbucket.org/cffi/cffi/changeset/68008f1b02e3/

Log:	Found a simpler and more efficient way to implement any ffi.gc().

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -188,11 +188,13 @@
 static PyTypeObject CData_Type;
 static PyTypeObject CDataOwning_Type;
 static PyTypeObject CDataOwningGC_Type;
+static PyTypeObject CDataGCP_Type;
 
 #define CTypeDescr_Check(ob)  (Py_TYPE(ob) == &CTypeDescr_Type)
 #define CData_Check(ob)       (Py_TYPE(ob) == &CData_Type ||            \
                                Py_TYPE(ob) == &CDataOwning_Type ||      \
-                               Py_TYPE(ob) == &CDataOwningGC_Type)
+                               Py_TYPE(ob) == &CDataOwningGC_Type ||    \
+                               Py_TYPE(ob) == &CDataGCP_Type)
 #define CDataOwn_Check(ob)    (Py_TYPE(ob) == &CDataOwning_Type ||      \
                                Py_TYPE(ob) == &CDataOwningGC_Type)
 
@@ -235,6 +237,11 @@
 } CDataObject_owngc_frombuf;
 
 typedef struct {
+    CDataObject head;
+    PyObject *origobj, *destructor;
+} CDataObject_gcp;
+
+typedef struct {
     ffi_cif cif;
     /* the following information is used when doing the call:
        - a buffer of size 'exchange_size' is malloced
@@ -1625,6 +1632,35 @@
     return 0;
 }
 
+/* forward */
+static void _my_PyErr_WriteUnraisable(char *objdescr, PyObject *obj,
+                                      char *extra_error_line);
+
+static void cdatagcp_dealloc(CDataObject_gcp *cd)
+{
+    PyObject *result;
+    PyObject *destructor = cd->destructor;
+    PyObject *origobj = cd->origobj;
+    cdata_dealloc((CDataObject *)cd);
+
+    result = PyObject_CallFunctionObjArgs(destructor, origobj, NULL);
+    if (result != NULL) {
+        Py_DECREF(result);
+    }
+    else {
+        _my_PyErr_WriteUnraisable("From callback for ffi.gc ", origobj, NULL);
+    }
+    Py_DECREF(destructor);
+    Py_DECREF(origobj);
+}
+
+static int cdatagcp_traverse(CDataObject_gcp *cd, visitproc visit, void *arg)
+{
+    Py_VISIT(cd->destructor);
+    Py_VISIT(cd->origobj);
+    return 0;
+}
+
 static PyObject *cdata_float(CDataObject *cd);  /*forward*/
 
 static PyObject *convert_cdata_to_enum_string(CDataObject *cd, int both)
@@ -2729,6 +2765,41 @@
     &CDataOwning_Type,                          /* tp_base */
 };
 
+static PyTypeObject CDataGCP_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "_cffi_backend.CDataGCP",
+    sizeof(CDataObject_gcp),
+    0,
+    (destructor)cdatagcp_dealloc,               /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,                                          /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    0,                                          /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES  /* tp_flags */
+                       | Py_TPFLAGS_HAVE_GC,
+    0,                                          /* tp_doc */
+    (traverseproc)cdatagcp_traverse,            /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+    0,                                          /* tp_getset */
+    &CData_Type,                                /* tp_base */
+};
+
 /************************************************************/
 
 typedef struct {
@@ -4709,7 +4780,8 @@
     return convert_from_object(result, ctype, pyobj);
 }
 
-static void _my_PyErr_WriteUnraisable(PyObject *obj, char *extra_error_line)
+static void _my_PyErr_WriteUnraisable(char *objdescr, PyObject *obj,
+                                      char *extra_error_line)
 {
     /* like PyErr_WriteUnraisable(), but write a full traceback */
     PyObject *f, *t, *v, *tb;
@@ -4726,7 +4798,7 @@
     f = PySys_GetObject("stderr");
     if (f != NULL) {
         if (obj != NULL) {
-            PyFile_WriteString("From cffi callback ", f);
+            PyFile_WriteString(objdescr, f);
             PyFile_WriteObject(obj, f, 0);
             PyFile_WriteString(":\n", f);
         }
@@ -4799,7 +4871,8 @@
     }
     onerror_cb = PyTuple_GET_ITEM(cb_args, 3);
     if (onerror_cb == Py_None) {
-        _my_PyErr_WriteUnraisable(py_ob, extra_error_line);
+        _my_PyErr_WriteUnraisable("From cffi callback ", py_ob,
+                                  extra_error_line);
     }
     else {
         PyObject *exc1, *val1, *tb1, *res1, *exc2, *val2, *tb2;
@@ -4824,11 +4897,12 @@
             /* double exception! print a double-traceback... */
             PyErr_Fetch(&exc2, &val2, &tb2);
             PyErr_Restore(exc1, val1, tb1);
-            _my_PyErr_WriteUnraisable(py_ob, extra_error_line);
+            _my_PyErr_WriteUnraisable("From cffi callback ", py_ob,
+                                      extra_error_line);
             PyErr_Restore(exc2, val2, tb2);
             extra_error_line = ("\nDuring the call to 'onerror', "
                                 "another exception occurred:\n\n");
-            _my_PyErr_WriteUnraisable(NULL, extra_error_line);
+            _my_PyErr_WriteUnraisable(NULL, NULL, extra_error_line);
         }
     }
     goto done;
@@ -5554,6 +5628,34 @@
                            (PyObject *)&CTypeDescr_Type);
 }
 
+static PyObject *b_gcp(PyObject *self, PyObject *args, PyObject *kwds)
+{
+    CDataObject_gcp *cd;
+    CDataObject *origobj;
+    PyObject *destructor;
+    static char *keywords[] = {"cdata", "destructor", NULL};
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O:gc", keywords,
+                                     &CData_Type, &origobj, &destructor))
+        return NULL;
+
+    cd = PyObject_GC_New(CDataObject_gcp, &CDataGCP_Type);
+    if (cd == NULL)
+        return NULL;
+
+    Py_INCREF(destructor);
+    Py_INCREF(origobj);
+    Py_INCREF(origobj->c_type);
+    cd->head.c_data = origobj->c_data;
+    cd->head.c_type = origobj->c_type;
+    cd->head.c_weakreflist = NULL;
+    cd->origobj = (PyObject *)origobj;
+    cd->destructor = destructor;
+
+    PyObject_GC_Track(cd);
+    return (PyObject *)cd;
+}
+
 /************************************************************/
 
 static char _testfunc0(char a, char b)
@@ -5859,6 +5961,7 @@
     {"newp_handle", b_newp_handle, METH_VARARGS},
     {"from_handle", b_from_handle, METH_O},
     {"from_buffer", b_from_buffer, METH_VARARGS},
+    {"gcp", (PyCFunction)b_gcp, METH_VARARGS | METH_KEYWORDS},
 #ifdef MS_WIN32
     {"getwinerror", (PyCFunction)b_getwinerror, METH_VARARGS | METH_KEYWORDS},
 #endif
@@ -6096,6 +6199,8 @@
         INITERROR;
     if (PyType_Ready(&CDataOwningGC_Type) < 0)
         INITERROR;
+    if (PyType_Ready(&CDataGCP_Type) < 0)
+        INITERROR;
     if (PyType_Ready(&CDataIter_Type) < 0)
         INITERROR;
     if (PyType_Ready(&MiniBuffer_Type) < 0)
diff --git a/c/cffi1_module.c b/c/cffi1_module.c
--- a/c/cffi1_module.c
+++ b/c/cffi1_module.c
@@ -13,7 +13,6 @@
 
 #include "ffi_obj.c"
 #include "cglob.c"
-#include "cgc.c"
 #include "lib_obj.c"
 #include "cdlopen.c"
 
diff --git a/c/cgc.c b/c/cgc.c
deleted file mode 100644
--- a/c/cgc.c
+++ /dev/null
@@ -1,122 +0,0 @@
-
-/* translated to C from cffi/gc_weakref.py */
-
-
-static PyObject *gc_wref_remove(PyObject *ffi_wref_tup, PyObject *key)
-{
-    FFIObject *ffi;
-    PyObject *indexobj, *destructor, *cdata, *freelist, *result;
-    Py_ssize_t index;
-
-    /* here, tup is a 4-tuple (ffi, destructor, cdata, index) */
-    if (!PyTuple_Check(ffi_wref_tup))
-        goto oops;    /* should never occur */
-
-    ffi = (FFIObject *)PyTuple_GET_ITEM(ffi_wref_tup, 0);
-    destructor = PyTuple_GET_ITEM(ffi_wref_tup, 1);
-    cdata = PyTuple_GET_ITEM(ffi_wref_tup, 2);
-    indexobj = PyTuple_GET_ITEM(ffi_wref_tup, 3);
-
-    index = PyInt_AsSsize_t(indexobj);
-    if (index < 0)
-        goto oops;    /* should never occur */
-
-    /* assert gc_wrefs[index] is key */
-    if (PyList_GET_ITEM(ffi->gc_wrefs, index) != key)
-        goto oops;    /* should never occur */
-
-    /* gc_wrefs[index] = freelist */
-    /* transfer ownership of 'freelist' to 'gc_wrefs[index]' */
-    freelist = ffi->gc_wrefs_freelist;
-    PyList_SET_ITEM(ffi->gc_wrefs, index, freelist);
-
-    /* freelist = index */
-    ffi->gc_wrefs_freelist = indexobj;
-    Py_INCREF(indexobj);
-
-    /* destructor(cdata) */
-    result = PyObject_CallFunctionObjArgs(destructor, cdata, NULL);
-
-    Py_DECREF(key);    /* free the reference that was in 'gc_wrefs[index]' */
-    return result;
-
- oops:
-    PyErr_SetString(PyExc_SystemError, "cgc: internal inconsistency");
-    /* random leaks may follow */
-    return NULL;
-}
-
-static PyMethodDef remove_callback = {
-    "gc_wref_remove", (PyCFunction)gc_wref_remove, METH_O
-};
-
-static PyObject *gc_weakrefs_build(FFIObject *ffi, CDataObject *cdata,
-                                   PyObject *destructor)
-{
-    PyObject *new_cdata, *ref = NULL, *tup = NULL, *remove_fn = NULL;
-    Py_ssize_t index;
-    PyObject *datalist;
-
-    if (ffi->gc_wrefs == NULL) {
-        /* initialize */
-        datalist = PyList_New(0);
-        if (datalist == NULL)
-            return NULL;
-        ffi->gc_wrefs = datalist;
-        assert(ffi->gc_wrefs_freelist == NULL);
-        ffi->gc_wrefs_freelist = Py_None;
-        Py_INCREF(Py_None);
-    }
-
-    /* new_cdata = self.ffi.cast(typeof(cdata), cdata) */
-    new_cdata = do_cast(cdata->c_type, (PyObject *)cdata);
-    if (new_cdata == NULL)
-        goto error;
-
-    /* if freelist is None: */
-    datalist = ffi->gc_wrefs;
-    if (ffi->gc_wrefs_freelist == Py_None) {
-        /* index = len(gc_wrefs) */
-        index = PyList_GET_SIZE(datalist);
-        /* gc_wrefs.append(None) */
-        if (PyList_Append(datalist, Py_None) < 0)
-            goto error;
-        tup = Py_BuildValue("OOOn", ffi, destructor, cdata, index);
-    }
-    else {
-        /* index = freelist */
-        index = PyInt_AsSsize_t(ffi->gc_wrefs_freelist);
-        if (index < 0)
-            goto error;   /* should not occur */
-        tup = PyTuple_Pack(4, ffi, destructor, cdata, ffi->gc_wrefs_freelist);
-    }
-    if (tup == NULL)
-        goto error;
-
-    remove_fn = PyCFunction_New(&remove_callback, tup);
-    if (remove_fn == NULL)
-        goto error;
-
-    ref = PyWeakref_NewRef(new_cdata, remove_fn);
-    if (ref == NULL)
-        goto error;
-
-    /* freelist = gc_wrefs[index] (which is None if we just did append(None)) */
-    /* transfer ownership of 'datalist[index]' into gc_wrefs_freelist */
-    Py_DECREF(ffi->gc_wrefs_freelist);
-    ffi->gc_wrefs_freelist = PyList_GET_ITEM(datalist, index);
-    /* gc_wrefs[index] = ref */
-    /* transfer ownership of 'ref' into 'datalist[index]' */
-    PyList_SET_ITEM(datalist, index, ref);
-    Py_DECREF(remove_fn);
-    Py_DECREF(tup);
-
-    return new_cdata;
-
- error:
-    Py_XDECREF(new_cdata);
-    Py_XDECREF(ref);
-    Py_XDECREF(tup);
-    Py_XDECREF(remove_fn);
-    return NULL;
-}
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -664,21 +664,8 @@
 "Later, when this new cdata object is garbage-collected,\n"
 "'destructor(old_cdata_object)' will be called.");
 
-static PyObject *gc_weakrefs_build(FFIObject *ffi, CDataObject *cd,
-                                   PyObject *destructor);   /* forward */
-
-static PyObject *ffi_gc(FFIObject *self, PyObject *args, PyObject *kwds)
-{
-    CDataObject *cd;
-    PyObject *destructor;
-    static char *keywords[] = {"cdata", "destructor", NULL};
-
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O:gc", keywords,
-                                     &CData_Type, &cd, &destructor))
-        return NULL;
-
-    return gc_weakrefs_build(self, cd, destructor);
-}
+#define ffi_gc  b_gcp     /* ffi_gc() => b_gcp()
+                             from _cffi_backend.c */
 
 PyDoc_STRVAR(ffi_callback_doc,
 "Return a callback object or a decorator making such a callback object.\n"
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -72,7 +72,6 @@
         self._cdefsources = []
         self._included_ffis = []
         self._windows_unicode = None
-        self._gcp = None
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
         for name in backend.__dict__:
@@ -329,14 +328,13 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
-        if self._gcp is not None:
-            return self._gcp(cdata, destructor)
-        if hasattr(self._backend, 'FFI'):
-            compiled_ffi = self._backend.FFI()
-            self._gcp = compiled_ffi.gc
-            return self._gcp(cdata, destructor)
+        try:
+            gcp = self._backend.gcp
+        except AttributeError:
+            pass
+        else:
+            return gcp(cdata, destructor)
         #
-        # the rest is for the ctypes backend only
         with self._lock:
             try:
                 gc_weakrefs = self.gc_weakrefs

From noreply at buildbot.pypy.org  Mon Jul  6 13:24:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 13:24:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Found out that ffi.gc() can be
 implemented without this mess of weakrefs.
Message-ID: <20150706112432.3E3711C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78455:7b0ceb5a2ce0
Date: 2015-07-06 12:53 +0200
http://bitbucket.org/pypy/pypy/changeset/7b0ceb5a2ce0/

Log:	Found out that ffi.gc() can be implemented without this mess of
	weakrefs.

diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -37,6 +37,7 @@
         'from_handle': 'handle.from_handle',
         '_get_types': 'func._get_types',
         'from_buffer': 'func.from_buffer',
+        'gcp': 'func.gcp',
 
         'string': 'func.string',
         'buffer': 'cbuffer.buffer',
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -363,6 +363,10 @@
     def _sizeof(self):
         return self.ctype.size
 
+    def with_gc(self, w_destructor):
+        with self as ptr:
+            return W_CDataGCP(self.space, ptr, self.ctype, self, w_destructor)
+
 
 class W_CDataMem(W_CData):
     """This is the base class used for cdata objects that own and free
@@ -483,6 +487,20 @@
             self.length, self.space.type(self.w_keepalive).name)
 
 
+class W_CDataGCP(W_CData):
+    """For ffi.gc()."""
+    _attrs_ = ['w_original_cdata', 'w_destructor']
+    _immutable_fields_ = ['w_original_cdata', 'w_destructor']
+
+    def __init__(self, space, cdata, ctype, w_original_cdata, w_destructor):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.w_original_cdata = w_original_cdata
+        self.w_destructor = w_destructor
+
+    def __del__(self):
+        self.space.call_function(self.w_destructor, self.w_original_cdata)
+
+
 W_CData.typedef = TypeDef(
     '_cffi_backend.CData',
     __module__ = '_cffi_backend',
diff --git a/pypy/module/_cffi_backend/cgc.py b/pypy/module/_cffi_backend/cgc.py
deleted file mode 100644
--- a/pypy/module/_cffi_backend/cgc.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from rpython.rlib import jit
-
-
- at jit.dont_look_inside
-def gc_weakrefs_build(ffi, w_cdata, w_destructor):
-    from pypy.module._weakref import interp__weakref
-
-    space = ffi.space
-    if ffi.w_gc_wref_remove is None:
-        ffi.gc_wref_dict = {}
-        ffi.w_gc_wref_remove = space.getattr(space.wrap(ffi),
-                                             space.wrap("__gc_wref_remove"))
-
-    w_new_cdata = w_cdata.ctype.cast(w_cdata)
-    assert w_new_cdata is not w_cdata
-
-    w_ref = interp__weakref.make_weakref_with_callback(
-        space,
-        space.gettypefor(interp__weakref.W_Weakref),
-        w_new_cdata,
-        ffi.w_gc_wref_remove)
-
-    ffi.gc_wref_dict[w_ref] = (w_destructor, w_cdata)
-    return w_new_cdata
-
-
-def gc_wref_remove(ffi, w_ref):
-    (w_destructor, w_cdata) = ffi.gc_wref_dict.pop(w_ref)
-    ffi.space.call_function(w_destructor, w_cdata)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -10,7 +10,7 @@
 from pypy.module._cffi_backend import parse_c_type, realize_c_type
 from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray
 from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle
-from pypy.module._cffi_backend import cbuffer, func, cgc, wrapper
+from pypy.module._cffi_backend import cbuffer, func, wrapper
 from pypy.module._cffi_backend import cffi_opcode
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend.cdataobj import W_CData
@@ -344,10 +344,7 @@
 Later, when this new cdata object is garbage-collected,
 'destructor(old_cdata_object)' will be called."""
         #
-        return cgc.gc_weakrefs_build(self, w_cdata, w_destructor)
-
-    def descr___gc_wref_remove(self, w_ref):
-        return cgc.gc_wref_remove(self, w_ref)
+        return w_cdata.with_gc(w_destructor)
 
 
     @unwrap_spec(replace_with=str)
@@ -586,7 +583,6 @@
                                      W_FFIObject.set_errno,
                                      doc=W_FFIObject.doc_errno,
                                      cls=W_FFIObject),
-        __gc_wref_remove = interp2app(W_FFIObject.descr___gc_wref_remove),
         addressof   = interp2app(W_FFIObject.descr_addressof),
         alignof     = interp2app(W_FFIObject.descr_alignof),
         buffer      = interp2app(W_FFIObject.descr_buffer),
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -105,3 +105,9 @@
                     "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
+
+# ____________________________________________________________
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData)
+def gcp(space, w_cdata, w_destructor):
+    return w_cdata.with_gc(w_destructor)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -265,9 +265,10 @@
             assert p1[0] == 123
             seen.append(1)
         ffi.gc(p, destructor=destructor)    # instantly forgotten
+        _cffi1_backend.gcp(p, destructor=destructor)
         for i in range(5):
             if seen:
                 break
             import gc
             gc.collect()
-        assert seen == [1]
+        assert seen == [1, 1]

From noreply at buildbot.pypy.org  Mon Jul  6 14:57:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  6 Jul 2015 14:57:19 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: a completely different approach
Message-ID: <20150706125719.145B61C1016@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78456:cbf2d99e2ae8
Date: 2015-07-06 14:57 +0200
http://bitbucket.org/pypy/pypy/changeset/cbf2d99e2ae8/

Log:	a completely different approach

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -24,6 +24,37 @@
     raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
 
 
+class LoopCompileData(object):
+    """ An object that accumulates all of the necessary info for
+    the optimization phase, but does not actually have any other state
+
+    This is the case of label() ops label()
+    """
+    def __init__(self, start_label, end_label, operations):
+        self.start_label = start_label
+        self.end_label = end_label
+        assert start_label.getopnum() == rop.LABEL
+        assert end_label.getopnum() == rop.LABEL
+        self.operations = operations
+
+    def forget_optimization_info(self):
+        for arg in self.start_label.getarglist():
+            arg.set_forwarded(None)
+        for op in self.operations:
+            op.set_forwarded(None)
+
+    def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
+        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
+
+        if unroll:
+            opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
+            return opt.optimize_preamble(self.start_label, self.end_label,
+                                         self.operations)
+            xxx
+        else:
+            xxx
+
+
 def show_procedures(metainterp_sd, procedure=None, error=None):
     # debugging
     if option and (option.view or option.viewloops):
@@ -104,10 +135,8 @@
 # ____________________________________________________________
 
 
-def compile_loop(metainterp, greenkey, start,
-                 inputargs, jumpargs,
-                 full_preamble_needed=True,
-                 try_disabling_unroll=False):
+def compile_loop(metainterp, greenkey, start, inputargs, jumpargs,
+                 full_preamble_needed=True, try_disabling_unroll=False):
     """Try to compile a new procedure by closing the current history back
     to the first operation.
     """
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -4,7 +4,6 @@
 from rpython.jit.metainterp.optimizeopt.virtualize import OptVirtualize
 from rpython.jit.metainterp.optimizeopt.heap import OptHeap
 from rpython.jit.metainterp.optimizeopt.vstring import OptString
-from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
 from rpython.jit.metainterp.optimizeopt.simplify import OptSimplify
 from rpython.jit.metainterp.optimizeopt.pure import OptPure
 from rpython.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
@@ -47,26 +46,17 @@
 
     return optimizations, unroll
 
-def optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts,
-                   inline_short_preamble=True, start_state=None,
-                   export_state=True):
+def optimize_trace(metainterp_sd, jitdriver_sd, compile_data):
     """Optimize loop.operations to remove internal overheadish operations.
     """
-
     debug_start("jit-optimize")
+    #inputargs = start_label.getarglist()
     try:
-        loop.logops = metainterp_sd.logger_noopt.log_loop(loop.inputargs,
-                                                          loop.operations)
-        optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
-        if unroll:
-            return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
-                                   optimizations,
-                                   inline_short_preamble, start_state,
-                                   export_state)
-        else:
-            optimizer = Optimizer(metainterp_sd, jitdriver_sd, loop,
-                                  optimizations)
-            optimizer.propagate_all_forward()
+        #logops = metainterp_sd.logger_noopt.log_loop(inputargs, operations)
+        optimizations, unroll = build_opt_chain(metainterp_sd,
+                                                compile_data.enable_opts)
+        return compile_data.optimize(metainterp_sd, jitdriver_sd,
+                                     optimizations, unroll)
     finally:
         debug_stop("jit-optimize")
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -211,14 +211,10 @@
 
 class Optimizer(Optimization):
 
-    exporting_state = False
-    emitting_dissabled = False
-
-    def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations=None):
+    def __init__(self, metainterp_sd, jitdriver_sd, optimizations=None):
         self.metainterp_sd = metainterp_sd
         self.jitdriver_sd = jitdriver_sd
         self.cpu = metainterp_sd.cpu
-        self.loop = loop
         self.logops = LogOperations(metainterp_sd, False)
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.interned_ints = {}
@@ -234,14 +230,15 @@
         self.optheap = None
         self.optearlyforce = None
         self.optunroll = None
-        # the following two fields is the data kept for unrolling,
-        # those are the operations that can go to the short_preamble
-        if loop is not None:
-            self.call_pure_results = loop.call_pure_results
 
         self.set_optimizations(optimizations)
         self.setup()
 
+    def init_inparg_dict_from(self, lst):
+        self.inparg_dict = {}
+        for box in lst:
+            self.inparg_dict[box] = None
+
     def set_optimizations(self, optimizations):
         if optimizations:
             self.first_optimization = optimizations[0]
@@ -442,18 +439,13 @@
         else:
             return CONST_0
 
-    def propagate_all_forward(self, clear=True, create_inp_args=True):
-        if clear:
-            self.clear_newoperations()
-        if create_inp_args:
-            self.inparg_dict = {}
-            for op in self.loop.inputargs:
-                self.inparg_dict[op] = None
-        for op in self.loop.operations:
+    def propagate_all_forward(self, inputargs, ops, create_inp_args=True):
+        self.init_inparg_dict_from(inputargs)
+        for op in ops:
             self._really_emitted_operation = None
             self.first_optimization.propagate_forward(op)
-        self.loop.operations = self.get_newoperations()
-        self.loop.quasi_immutable_deps = self.quasi_immutable_deps
+        #self.loop.operations = self.get_newoperations()
+        #self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -3,10 +3,11 @@
 """
 
 from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest,\
-     LLtypeMixin, FakeMetaInterpStaticData
-from rpython.jit.metainterp.history import (TreeLoop, AbstractDescr, ConstInt,
+     LLtypeMixin
+from rpython.jit.metainterp.history import (TreeLoop, ConstInt,
                                             JitCellToken, TargetToken)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.compile import LoopCompileData
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN
 
@@ -18,7 +19,6 @@
     
     def optimize(self, ops):
         loop = self.parse(ops, postprocess=self.postprocess)
-        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
         self.add_guard_future_condition(loop)
         operations =  loop.operations
         jumpop = operations[-1]
@@ -34,11 +34,10 @@
         token = JitCellToken()
         start_label = ResOperation(rop.LABEL, inputargs, descr=TargetToken(token))
         stop_label = ResOperation(rop.LABEL, jump_args, descr=token)
-        preamble.operations = [start_label] + operations + [stop_label]
-        start_state = self._do_optimize_loop(preamble, None,
-                                             export_state=True)
-        vs = preamble.operations[-1].getdescr().virtual_state
-        return start_state, vs, loop, preamble
+        compile_data = LoopCompileData(start_label, stop_label, operations)
+        start_state, newops = self._do_optimize_loop(compile_data)
+        preamble.operations = newops
+        return start_state, loop, preamble
 
 class TestUnroll(BaseTestUnroll):
     def test_simple(self):
@@ -48,7 +47,8 @@
         guard_value(i1, 1) []
         jump(i1)
         """
-        es, vs, loop, preamble = self.optimize(loop)
+        es, loop, preamble = self.optimize(loop)
+        vs = es.virtual_state
         assert isinstance(vs.state[0], NotVirtualStateInfo)
         # the virtual state is constant, so we don't need to have it in
         # inputargs
@@ -65,8 +65,9 @@
         i1 = int_add(i0, 1)
         jump(i0)
         """
-        es, vs, loop, preamble = self.optimize(loop)
+        es, loop, preamble = self.optimize(loop)
+        vs = es.virtual_state
         assert isinstance(vs.state[0], NotVirtualStateInfo)
         assert vs.state[0].level == LEVEL_UNKNOWN
-        op = preamble.operations[1]
+        op = preamble.operations[0]
         assert es.short_boxes == {op: op}
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -405,15 +405,11 @@
         assert equaloplists(optimized.operations,
                             expected.operations, False, remap, text_right)
 
-    def _do_optimize_loop(self, loop, call_pure_results, start_state=None,
-                          export_state=False):
+    def _do_optimize_loop(self, compile_data, call_pure_results=None):
         from rpython.jit.metainterp.optimizeopt import optimize_trace
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
-        self.loop = loop
-        operations = loop.operations
-        inputargs = loop.inputargs
-        loop.call_pure_results = args_dict()
+        # XXX
         if call_pure_results is not None:
             for k, v in call_pure_results.items():
                 loop.call_pure_results[list(k)] = v
@@ -423,17 +419,14 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        state = optimize_trace(metainterp_sd, None, loop,
-                              self.enable_opts,
-                              start_state=start_state,
-                              export_state=export_state)
-        compile.forget_optimization_info(operations)
-        compile.forget_optimization_info(inputargs)
+        compile_data.enable_opts = self.enable_opts
+        state = optimize_trace(metainterp_sd, None, compile_data)
+        compile_data.forget_optimization_info()
         return state
 
     def unroll_and_optimize(self, loop, call_pure_results=None):
+        xxx
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        logops = LogOperations(metainterp_sd, False)
         self.add_guard_future_condition(loop)
         operations =  loop.operations
         jumpop = operations[-1]
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -13,15 +13,6 @@
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
 
 
-# FIXME: Introduce some VirtualOptimizer super class instead
-
-def optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
-                    inline_short_preamble=True, start_state=None,
-                    export_state=True):
-    opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, loop, optimizations)
-    opt.inline_short_preamble = inline_short_preamble
-    return opt.propagate_all_forward(start_state, export_state)
-
 
 class PreambleOp(AbstractResOp):
     def __init__(self, op, info):
@@ -36,18 +27,6 @@
 
 
 class UnrollableOptimizer(Optimizer):
-    def setup(self):
-        self.importable_values = {}
-        self.emitting_dissabled = False
-        self.emitted_guards = 0
-
-    def emit_operation(self, op):
-        if self.emitting_dissabled:
-            return
-        if op.is_guard():
-            self.emitted_guards += 1 # FIXME: can we use counter in self._emit_operation?
-        self._emit_operation(op)
-
     def force_op_from_preamble(self, preamble_op):
         op = preamble_op.op
         self.optunroll.short.append(op)
@@ -63,52 +42,27 @@
 
     inline_short_preamble = True
 
-    def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations):
+    def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
         self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
-                                             loop, optimizations)
+                                             optimizations)
         self.optimizer.optunroll = self
-        self.boxes_created_this_iteration = None
 
     def get_virtual_state(self, args):
         modifier = VirtualStateConstructor(self.optimizer)
         return modifier.get_virtual_state(args)
 
-    def propagate_all_forward(self, starting_state, export_state=True):
-        self.optimizer.exporting_state = export_state
-        loop = self.optimizer.loop
-        self.optimizer.clear_newoperations()
-        for op in loop.operations:
-            assert op.get_forwarded() is None
-        for op in loop.inputargs:
-            assert op.get_forwarded() is None
-
-        start_label = loop.operations[0]
-        if start_label.getopnum() == rop.LABEL:
-            loop.operations = loop.operations[1:]
-            # We need to emit the label op before import_state() as emitting it
-            # will clear heap caches
-            self.optimizer.send_extra_operation(start_label)
-        else:
-            start_label = None
-
-        patchguardop = None
-        if len(loop.operations) > 1:
-            patchguardop = loop.operations[-2]
-            if patchguardop.getopnum() != rop.GUARD_FUTURE_CONDITION:
-                patchguardop = None
-
-        jumpop = loop.operations[-1]
-        if jumpop.getopnum() == rop.JUMP or jumpop.getopnum() == rop.LABEL:
-            loop.operations = loop.operations[:-1]
-        else:
-            jumpop = None
-
-        self.import_state(start_label, starting_state)
-        self.optimizer.inparg_dict = {}
-        for box in start_label.getarglist():
-            self.optimizer.inparg_dict[box] = None
-        self.optimizer.propagate_all_forward(clear=False, create_inp_args=False)
-
+    def _check_no_forwarding(self, lsts):
+        for lst in lsts:
+            for op in lst:
+                assert op.get_forwarded() is None
+        assert not self.optimizer._newoperations
+    
+    def optimize_preamble(self, start_label, end_label, ops):
+        self._check_no_forwarding([[start_label, end_label], ops])
+        self.optimizer.propagate_all_forward(start_label.getarglist(), ops)
+        exported_state = self.export_state(start_label, end_label)
+        return exported_state, self.optimizer._newoperations
+        # WTF is the rest of this function
         if not jumpop:
             return
 
@@ -183,7 +137,7 @@
                     self.optimizer.send_extra_operation(stop_label)
                     loop.operations = self.optimizer.get_newoperations()
                     return None
-            final_state = self.export_state(stop_label)
+            final_state = self.export_state(start_label, stop_label)
         else:
             final_state = None
         loop.operations.append(stop_label)
@@ -200,19 +154,16 @@
         return stop_target.targeting_jitcell_token is start_target.targeting_jitcell_token
 
 
-    def export_state(self, targetop):
-        original_jump_args = targetop.getarglist()
-        label_op = self.optimizer.loop.operations[0]
-        jump_args = [self.get_box_replacement(a) for a in original_jump_args]
-        virtual_state = self.get_virtual_state(jump_args)
-        target_token = targetop.getdescr()
-        assert isinstance(target_token, TargetToken)
-        target_token.virtual_state = virtual_state
+    def export_state(self, start_label, end_label):
+        original_label_args = end_label.getarglist()
+        end_args = [self.get_box_replacement(a) for a in original_label_args]
+        virtual_state = self.get_virtual_state(end_args)
         sb = ShortBoxes()
-        sb.create_short_boxes(self.optimizer, jump_args)
-        inparg_mapping = [(label_op.getarg(i), jump_args[i])
-                          for i in range(len(jump_args))]
-        return ExportedState(inparg_mapping, [], sb.short_boxes)
+        sb.create_short_boxes(self.optimizer, end_args)
+        inparg_mapping = [(start_label.getarg(i), end_args[i])
+                          for i in range(len(end_args)) if
+                          start_label.getarg(i) is not end_args[i]]
+        return ExportedState(inparg_mapping, virtual_state, [], sb.short_boxes)
 
 
         inputargs = virtual_state.make_inputargs(jump_args, self.optimizer)
@@ -297,6 +248,8 @@
         # think about it, it seems to be just for consts
         #for source, target in exported_state.inputarg_setup_ops:
         #    source.set_forwarded(target)
+        for source, target in exported_state.inputarg_mapping:
+            source.set_forwarded(target)
 
         for op, preamble_op in exported_state.short_boxes.iteritems():
             if preamble_op.is_always_pure():
@@ -709,7 +662,9 @@
     * short boxes - a mapping op -> preamble_op
     """
     
-    def __init__(self, inputarg_mapping, exported_infos, short_boxes):
+    def __init__(self, inputarg_mapping, virtual_state, exported_infos,
+                 short_boxes):
         self.inputarg_mapping = inputarg_mapping
+        self.virtual_state = virtual_state
         self.exported_infos = exported_infos
         self.short_boxes = short_boxes

From noreply at buildbot.pypy.org  Mon Jul  6 15:24:28 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  6 Jul 2015 15:24:28 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: port test_optimizebasic to the
	new style
Message-ID: <20150706132428.4C1FC1C1158@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78457:b16656d315c9
Date: 2015-07-06 15:24 +0200
http://bitbucket.org/pypy/pypy/changeset/b16656d315c9/

Log:	port test_optimizebasic to the new style

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -23,25 +23,27 @@
     from rpython.jit.metainterp.pyjitpl import SwitchToBlackhole
     raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
 
+class CompileData(object):
+    def forget_optimization_info(self):
+        for arg in self.start_label.getarglist():
+            arg.set_forwarded(None)
+        for op in self.operations:
+            op.set_forwarded(None)
 
-class LoopCompileData(object):
+class LoopCompileData(CompileData):
     """ An object that accumulates all of the necessary info for
     the optimization phase, but does not actually have any other state
 
     This is the case of label() ops label()
     """
-    def __init__(self, start_label, end_label, operations):
+    def __init__(self, start_label, end_label, operations,
+                 call_pure_results=None):
         self.start_label = start_label
         self.end_label = end_label
         assert start_label.getopnum() == rop.LABEL
         assert end_label.getopnum() == rop.LABEL
         self.operations = operations
-
-    def forget_optimization_info(self):
-        for arg in self.start_label.getarglist():
-            arg.set_forwarded(None)
-        for op in self.operations:
-            op.set_forwarded(None)
+        self.call_pure_results = call_pure_results
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
@@ -54,6 +56,22 @@
         else:
             xxx
 
+class SimpleCompileData(CompileData):
+    """ This represents label() ops jump with no extra info associated with
+    the label
+    """
+    def __init__(self, start_label, operations, call_pure_results=None):
+        self.start_label = start_label
+        self.operations = operations
+        self.call_pure_results = call_pure_results
+
+    def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
+        from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
+
+        opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations)
+        return opt.propagate_all_forward(self.start_label.getarglist(),
+                                         self.operations,
+                                         self.call_pure_results)
 
 def show_procedures(metainterp_sd, procedure=None, error=None):
     # debugging
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -439,8 +439,9 @@
         else:
             return CONST_0
 
-    def propagate_all_forward(self, inputargs, ops, create_inp_args=True):
+    def propagate_all_forward(self, inputargs, ops, call_pure_results=None):
         self.init_inparg_dict_from(inputargs)
+        self.call_pure_results = call_pure_results
         for op in ops:
             self._really_emitted_operation = None
             self.first_optimization.propagate_forward(op)
@@ -448,6 +449,7 @@
         #self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
+        return None, self._newoperations
 
     def send_extra_operation(self, op):
         self.first_optimization.propagate_forward(op)
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -151,9 +151,9 @@
         opnum = OpHelpers.call_for_descr(op.getdescr())
         newop = self.optimizer.replace_op_with(op, opnum)
         self.emit_operation(newop)
-        if self.optimizer.emitting_dissabled:
-            self.extra_call_pure.append(op) # XXX
-        else:
+        #if self.optimizer.emitting_dissabled:
+        #    self.extra_call_pure.append(op) # XXX
+        #else:
             # don't move call_pure_with_exception in the short preamble...
             # issue #2015
 
@@ -162,9 +162,9 @@
             # fix together with unroll
             #effectinfo = op.getdescr().get_extra_info()
             #if not effectinfo.check_can_raise(ignore_memoryerror=True):
-            if 1:
-                self.call_pure_positions.append(
-                    len(self.optimizer._newoperations) - 1)
+        #    if 1:
+        self.call_pure_positions.append(
+            len(self.optimizer._newoperations) - 1)
 
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
     optimize_CALL_PURE_F = optimize_CALL_PURE_I
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -45,29 +45,29 @@
     def optimize_RECORD_KNOWN_CLASS(self, op):
         pass
 
-    def optimize_LABEL(self, op):
-        if not self.unroll:
-            descr = op.getdescr()
-            if isinstance(descr, JitCellToken):
-                return self.optimize_JUMP(op.copy_and_change(rop.JUMP))
-            self.last_label_descr = op.getdescr()
-        self.emit_operation(op)
+    # def optimize_LABEL(self, op):
+    #     if not self.unroll:
+    #         descr = op.getdescr()
+    #         if isinstance(descr, JitCellToken):
+    #             return self.optimize_JUMP(op.copy_and_change(rop.JUMP))
+    #         self.last_label_descr = op.getdescr()
+    #     self.emit_operation(op)
 
-    def optimize_JUMP(self, op):
-        if not self.unroll:
-            op = op.copy_and_change(op.getopnum())
-            descr = op.getdescr()
-            assert isinstance(descr, JitCellToken)
-            if not descr.target_tokens:
-                assert self.last_label_descr is not None
-                target_token = self.last_label_descr
-                assert isinstance(target_token, TargetToken)
-                assert target_token.targeting_jitcell_token is descr
-                op.setdescr(self.last_label_descr)
-            else:
-                assert len(descr.target_tokens) == 1
-                op.setdescr(descr.target_tokens[0])
-        self.emit_operation(op)
+    # def optimize_JUMP(self, op):
+    #     if not self.unroll:
+    #         op = op.copy_and_change(op.getopnum())
+    #         descr = op.getdescr()
+    #         assert isinstance(descr, JitCellToken)
+    #         if not descr.target_tokens:
+    #             assert self.last_label_descr is not None
+    #             target_token = self.last_label_descr
+    #             assert isinstance(target_token, TargetToken)
+    #             assert target_token.targeting_jitcell_token is descr
+    #             op.setdescr(self.last_label_descr)
+    #         else:
+    #             assert len(descr.target_tokens) == 1
+    #             op.setdescr(descr.target_tokens[0])
+    #     self.emit_operation(op)
 
     def optimize_GUARD_FUTURE_CONDITION(self, op):
         pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -73,17 +73,28 @@
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
+        from rpython.jit.metainterp.optimizeopt.util import args_dict
+
                 
         loop = self.parse(ops, postprocess=self.postprocess)
         token = JitCellToken()
-        loop.operations = [ResOperation(rop.LABEL, loop.inputargs, descr=TargetToken(token))] + \
-                          loop.operations
+        label_op = ResOperation(rop.LABEL, loop.inputargs,
+                                descr=TargetToken(token))
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
         exp = parse(optops, namespace=self.namespace.copy())
         expected = convert_old_style_to_targets(exp, jump=True)
-        self._do_optimize_loop(loop, call_pure_results, export_state=False)
+        if call_pure_results is not None:
+            d = call_pure_results
+            call_pure_results = args_dict()
+            for k, v in d.items():
+                call_pure_results[list(k)] = v
+        compile_data = compile.SimpleCompileData(label_op, loop.operations,
+                                                 call_pure_results)
+        _, ops = self._do_optimize_loop(compile_data)
+        loop.operations = [label_op] + ops
         #print '\n'.join([str(o) for o in loop.operations])
+        self.loop = loop
         self.assert_equal(loop, expected)
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -407,12 +407,6 @@
 
     def _do_optimize_loop(self, compile_data, call_pure_results=None):
         from rpython.jit.metainterp.optimizeopt import optimize_trace
-        from rpython.jit.metainterp.optimizeopt.util import args_dict
-
-        # XXX
-        if call_pure_results is not None:
-            for k, v in call_pure_results.items():
-                loop.call_pure_results[list(k)] = v
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
         if hasattr(self, 'vrefinfo'):
             metainterp_sd.virtualref_info = self.vrefinfo

From noreply at buildbot.pypy.org  Mon Jul  6 16:00:01 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  6 Jul 2015 16:00:01 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: port test_optimizeopt to the
	new style
Message-ID: <20150706140001.7B3201C06AD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78458:775e4245077a
Date: 2015-07-06 15:59 +0200
http://bitbucket.org/pypy/pypy/changeset/775e4245077a/

Log:	port test_optimizeopt to the new style

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -52,7 +52,6 @@
             opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
             return opt.optimize_preamble(self.start_label, self.end_label,
                                          self.operations)
-            xxx
         else:
             xxx
 
@@ -68,11 +67,32 @@
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
 
+        assert not unroll
         opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.propagate_all_forward(self.start_label.getarglist(),
                                          self.operations,
                                          self.call_pure_results)
 
+class UnrolledLoopData(CompileData):
+    """ This represents label() ops jump with extra info that's from the
+    run of LoopCompileData
+    """
+    def __init__(self, start_label, end_jump, operations, state,
+                 call_pure_results=None):
+        self.start_label = start_label
+        self.end_jump = end_jump
+        self.operations = operations
+        self.state = state
+        self.call_pure_results = call_pure_results
+
+    def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
+        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
+
+        assert unroll # we should not be here if it's disabled
+        opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
+        return opt.optimize_peeled_loop(self.start_label, self.end_jump,
+                                        self.operations, self.state)
+
 def show_procedures(metainterp_sd, procedure=None, error=None):
     # debugging
     if option and (option.view or option.viewloops):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -73,9 +73,6 @@
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
-        from rpython.jit.metainterp.optimizeopt.util import args_dict
-
-                
         loop = self.parse(ops, postprocess=self.postprocess)
         token = JitCellToken()
         label_op = ResOperation(rop.LABEL, loop.inputargs,
@@ -84,11 +81,7 @@
             loop.operations[-1].setdescr(token)
         exp = parse(optops, namespace=self.namespace.copy())
         expected = convert_old_style_to_targets(exp, jump=True)
-        if call_pure_results is not None:
-            d = call_pure_results
-            call_pure_results = args_dict()
-            for k, v in d.items():
-                call_pure_results[list(k)] = v
+        call_pure_results = self._convert_call_pure_results(call_pure_results)
         compile_data = compile.SimpleCompileData(label_op, loop.operations,
                                                  call_pure_results)
         _, ops = self._do_optimize_loop(compile_data)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -418,8 +418,36 @@
         compile_data.forget_optimization_info()
         return state
 
+    def _convert_call_pure_results(self, d):
+        from rpython.jit.metainterp.optimizeopt.util import args_dict
+
+        if d is None:
+            return
+        call_pure_results = args_dict()
+        for k, v in d.items():
+            call_pure_results[list(k)] = v
+        return call_pure_results
+
     def unroll_and_optimize(self, loop, call_pure_results=None):
-        xxx
+        jump_op = loop.operations[-1]
+        assert jump_op.getopnum() == rop.JUMP
+        ops = loop.operations[:-1]
+        start_label = ResOperation(rop.LABEL, loop.inputargs)
+        end_label = jump_op.copy_and_change(opnum=rop.LABEL)
+        preamble_data = compile.LoopCompileData(start_label, end_label, ops)
+        start_state, preamble_ops = self._do_optimize_loop(preamble_data,
+                                                  call_pure_results)
+        preamble_data.forget_optimization_info()
+        loop_data = compile.UnrolledLoopData(end_label, jump_op,
+                                             ops + [jump_op], start_state)
+        _, ops = self._do_optimize_loop(loop_data, call_pure_results)
+        preamble = TreeLoop('preamble')
+        preamble.inputargs = start_label.getarglist()
+        preamble.operations = [start_label] + preamble_ops
+        loop.operations = [end_label] + ops
+        return preamble
+
+    def foo(self):
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
         self.add_guard_future_condition(loop)
         operations =  loop.operations
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -62,6 +62,14 @@
         self.optimizer.propagate_all_forward(start_label.getarglist(), ops)
         exported_state = self.export_state(start_label, end_label)
         return exported_state, self.optimizer._newoperations
+
+    def optimize_peeled_loop(self, start_label, end_jump, ops, state):
+        self._check_no_forwarding([[start_label, end_jump], ops])
+        self.import_state(start_label, state)
+        self.optimizer.propagate_all_forward(start_label.getarglist(), ops)
+        return None, self.optimizer._newoperations
+
+    def random_garbage(self):
         # WTF is the rest of this function
         if not jumpop:
             return
@@ -211,12 +219,9 @@
         return ExportedState([], [])
 
     def import_state(self, targetop, exported_state):
-        if not targetop: # Trace did not start with a label
-            self.inputargs = self.optimizer.loop.inputargs
-            self.short = None
-            self.initial_virtual_state = None
-            return
-
+        for source, target in exported_state.inputarg_mapping:
+            xxx
+        return
         self.inputargs = targetop.getarglist()
         target_token = targetop.getdescr()
         assert isinstance(target_token, TargetToken)

From noreply at buildbot.pypy.org  Mon Jul  6 16:01:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 16:01:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Trying probably in vain to make this
	even clearer
Message-ID: <20150706140145.8DC801C06AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78459:4cd22fb52ace
Date: 2015-07-06 16:01 +0200
http://bitbucket.org/pypy/pypy/changeset/4cd22fb52ace/

Log:	Trying probably in vain to make this even clearer

diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -103,12 +103,15 @@
 Howto
 -----
 
-In pypy/goal::
+Grab a copy of the pypy repository_.  In the directory pypy/goal, run::
 
    ../../rpython/bin/rpython -O2 --sandbox targetpypystandalone.py
 
 If you don't have a regular PyPy installed, you should, because it's
-faster to translate, but you can also run ``python translate.py`` instead.
+faster to translate; but you can also run the same line with ``python``
+in front.
+
+.. _repository: https://bitbucket.org/pypy/pypy
 
 
 To run it, use the tools in the pypy/sandbox directory::

From noreply at buildbot.pypy.org  Mon Jul  6 16:05:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 16:05:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add yet another warning here
Message-ID: <20150706140528.E944E1C14DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78460:ff894baa20f6
Date: 2015-07-06 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/ff894baa20f6/

Log:	Add yet another warning here

diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -139,8 +139,6 @@
 Not all operations are supported; e.g. if you type os.readlink('...'),
 the controller crashes with an exception and the subprocess is killed.
 Other operations make the subprocess die directly with a "Fatal RPython
-error".  None of this is a security hole; it just means that if you try
-to run some random program, it risks getting killed depending on the
-Python built-in functions it tries to call.  This is a matter of the
-sandboxing layer being incomplete so far, but it should not really be
-a problem in practice.
+error".  None of this is a security hole.  More importantly, *most other
+built-in modules are not enabled.  Please read all the warnings in this
+page before complaining about this.  Contributions welcome.*

From noreply at buildbot.pypy.org  Mon Jul  6 17:15:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 17:15:29 +0200 (CEST)
Subject: [pypy-commit] cffi default: Port the new_allocator() work from pypy
Message-ID: <20150706151529.CE1801C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2213:307f969f209f
Date: 2015-07-06 17:16 +0200
http://bitbucket.org/cffi/cffi/changeset/307f969f209f/

Log:	Port the new_allocator() work from pypy

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -238,7 +238,9 @@
 
 typedef struct {
     CDataObject head;
-    PyObject *origobj, *destructor;
+    Py_ssize_t length;     /* same as CDataObject_own_length up to here */
+    PyObject *origobj;
+    PyObject *destructor;
 } CDataObject_gcp;
 
 typedef struct {
@@ -281,6 +283,8 @@
 # include "wchar_helper.h"
 #endif
 
+typedef PyObject *const cffi_allocator_t[3];
+static cffi_allocator_t default_allocator = { NULL, NULL, NULL };
 static PyObject *FFIError;
 
 /************************************************************/
@@ -1643,14 +1647,17 @@
     PyObject *origobj = cd->origobj;
     cdata_dealloc((CDataObject *)cd);
 
-    result = PyObject_CallFunctionObjArgs(destructor, origobj, NULL);
-    if (result != NULL) {
-        Py_DECREF(result);
-    }
-    else {
-        _my_PyErr_WriteUnraisable("From callback for ffi.gc ", origobj, NULL);
-    }
-    Py_DECREF(destructor);
+    if (destructor != NULL) {
+        result = PyObject_CallFunctionObjArgs(destructor, origobj, NULL);
+        if (result != NULL) {
+            Py_DECREF(result);
+        }
+        else {
+            _my_PyErr_WriteUnraisable("From callback for ffi.gc ",
+                                      origobj, NULL);
+        }
+        Py_DECREF(destructor);
+    }
     Py_DECREF(origobj);
 }
 
@@ -2918,7 +2925,79 @@
     return (PyObject *)cd;
 }
 
-static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init)
+static CDataObject *allocate_gcp_object(CDataObject *origobj,
+                                        CTypeDescrObject *ct,
+                                        PyObject *destructor)
+{
+    CDataObject_gcp *cd = PyObject_GC_New(CDataObject_gcp, &CDataGCP_Type);
+    if (cd == NULL)
+        return NULL;
+
+    Py_XINCREF(destructor);
+    Py_INCREF(origobj);
+    Py_INCREF(ct);
+    cd->head.c_data = origobj->c_data;
+    cd->head.c_type = ct;
+    cd->head.c_weakreflist = NULL;
+    cd->origobj = (PyObject *)origobj;
+    cd->destructor = destructor;
+
+    PyObject_GC_Track(cd);
+    return (CDataObject *)cd;
+}
+
+static CDataObject *allocate_with_allocator(Py_ssize_t basesize,
+                                            Py_ssize_t datasize,
+                                            CTypeDescrObject *ct,
+                                            cffi_allocator_t allocator)
+{
+    CDataObject *cd;
+    PyObject *my_alloc = allocator[0];
+    PyObject *my_free = allocator[1];
+    PyObject *dont_clear_after_alloc = allocator[2];
+
+    if (my_alloc == NULL) {   /* alloc */
+        cd = allocate_owning_object(basesize + datasize, ct);
+        if (cd == NULL)
+            return NULL;
+        cd->c_data = ((char *)cd) + basesize;
+    }
+    else {
+        PyObject *res = PyObject_CallFunction(my_alloc, "n", datasize);
+        if (res == NULL)
+            return NULL;
+
+        if (!CData_Check(res)) {
+            PyErr_Format(PyExc_TypeError,
+                         "alloc() must return a cdata object (got %.200s)",
+                         Py_TYPE(res)->tp_name);
+            Py_DECREF(res);
+            return NULL;
+        }
+        cd = (CDataObject *)res;
+        if (!(cd->c_type->ct_flags & (CT_POINTER|CT_ARRAY))) {
+            PyErr_Format(PyExc_TypeError,
+                         "alloc() must return a cdata pointer, not '%s'",
+                         cd->c_type->ct_name);
+            Py_DECREF(res);
+            return NULL;
+        }
+        if (!cd->c_data) {
+            PyErr_SetString(PyExc_MemoryError, "alloc() returned NULL");
+            Py_DECREF(res);
+            return NULL;
+        }
+
+        cd = allocate_gcp_object(cd, ct, my_free);
+        Py_DECREF(res);
+    }
+    if (dont_clear_after_alloc == NULL)
+        memset(cd->c_data, 0, datasize);
+    return cd;
+}
+
+static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init,
+                             cffi_allocator_t allocator)
 {
     CTypeDescrObject *ctitem;
     CDataObject *cd;
@@ -2982,7 +3061,8 @@
            having a strong reference to it */
         CDataObject *cds;
 
-        cds = allocate_owning_object(dataoffset + datasize, ct->ct_itemdescr);
+        cds = allocate_with_allocator(dataoffset, datasize, ct->ct_itemdescr,
+                                      allocator);
         if (cds == NULL)
             return NULL;
 
@@ -2995,19 +3075,17 @@
         ((CDataObject_own_structptr *)cd)->structobj = (PyObject *)cds;
         assert(explicitlength < 0);
 
-        cds->c_data = cd->c_data = ((char *)cds) + dataoffset;
+        cd->c_data = cds->c_data;
     }
     else {
-        cd = allocate_owning_object(dataoffset + datasize, ct);
+        cd = allocate_with_allocator(dataoffset, datasize, ct, allocator);
         if (cd == NULL)
             return NULL;
 
-        cd->c_data = ((char *)cd) + dataoffset;
         if (explicitlength >= 0)
             ((CDataObject_own_length*)cd)->length = explicitlength;
     }
 
-    memset(cd->c_data, 0, datasize);
     if (init != Py_None) {
         if (convert_from_object(cd->c_data,
               (ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct, init) < 0) {
@@ -3024,7 +3102,7 @@
     PyObject *init = Py_None;
     if (!PyArg_ParseTuple(args, "O!|O:newp", &CTypeDescr_Type, &ct, &init))
         return NULL;
-    return direct_newp(ct, init);
+    return direct_newp(ct, init, default_allocator);
 }
 
 static int
@@ -5630,7 +5708,7 @@
 
 static PyObject *b_gcp(PyObject *self, PyObject *args, PyObject *kwds)
 {
-    CDataObject_gcp *cd;
+    CDataObject *cd;
     CDataObject *origobj;
     PyObject *destructor;
     static char *keywords[] = {"cdata", "destructor", NULL};
@@ -5639,20 +5717,7 @@
                                      &CData_Type, &origobj, &destructor))
         return NULL;
 
-    cd = PyObject_GC_New(CDataObject_gcp, &CDataGCP_Type);
-    if (cd == NULL)
-        return NULL;
-
-    Py_INCREF(destructor);
-    Py_INCREF(origobj);
-    Py_INCREF(origobj->c_type);
-    cd->head.c_data = origobj->c_data;
-    cd->head.c_type = origobj->c_type;
-    cd->head.c_weakreflist = NULL;
-    cd->origobj = (PyObject *)origobj;
-    cd->destructor = destructor;
-
-    PyObject_GC_Track(cd);
+    cd = allocate_gcp_object(origobj, origobj->c_type, destructor);
     return (PyObject *)cd;
 }
 
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -331,7 +331,8 @@
 "used for a longer time.  Be careful about that when copying the\n"
 "pointer to the memory somewhere else, e.g. into another structure.");
 
-static PyObject *ffi_new(FFIObject *self, PyObject *args, PyObject *kwds)
+static PyObject *_ffi_new(FFIObject *self, PyObject *args, PyObject *kwds,
+                          cffi_allocator_t allocator)
 {
     CTypeDescrObject *ct;
     PyObject *arg, *init = Py_None;
@@ -344,7 +345,70 @@
     if (ct == NULL)
         return NULL;
 
-    return direct_newp(ct, init);
+    return direct_newp(ct, init, allocator);
+}
+
+static PyObject *ffi_new(FFIObject *self, PyObject *args, PyObject *kwds)
+{
+    return _ffi_new(self, args, kwds, default_allocator);
+}
+
+static PyObject *_ffi_new_with_allocator(PyObject *allocator, PyObject *args,
+                                         PyObject *kwds)
+{
+    return _ffi_new((FFIObject *)PyTuple_GET_ITEM(allocator, 0),
+                    args, kwds,
+                    &PyTuple_GET_ITEM(allocator, 1));
+}
+
+PyDoc_STRVAR(ffi_new_allocator_doc, "XXX");
+
+static PyObject *ffi_new_allocator(FFIObject *self, PyObject *args,
+                                   PyObject *kwds)
+{
+    PyObject *allocator, *result;
+    PyObject *my_alloc = Py_None, *my_free = Py_None;
+    int should_clear_after_alloc = 1;
+    static char *keywords[] = {"alloc", "free", "should_clear_after_alloc",
+                               NULL};
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi:new_allocator", keywords,
+                                     &my_alloc, &my_free,
+                                     &should_clear_after_alloc))
+        return NULL;
+
+    if (my_alloc == Py_None && my_free != Py_None) {
+        PyErr_SetString(PyExc_TypeError, "cannot pass 'free' without 'alloc'");
+        return NULL;
+    }
+
+    allocator = PyTuple_New(4);
+    if (allocator == NULL)
+        return NULL;
+
+    Py_INCREF(self);
+    PyTuple_SET_ITEM(allocator, 0, (PyObject *)self);
+
+    if (my_alloc != Py_None) {
+        Py_INCREF(my_alloc);
+        PyTuple_SET_ITEM(allocator, 1, my_alloc);
+    }
+    if (my_free != Py_None) {
+        Py_INCREF(my_free);
+        PyTuple_SET_ITEM(allocator, 2, my_free);
+    }
+    if (!should_clear_after_alloc) {
+        Py_INCREF(Py_True);
+        PyTuple_SET_ITEM(allocator, 3, Py_True);  /* dont_clear_after_alloc */
+    }
+
+    {
+        static PyMethodDef md = {"allocator",
+                                 (PyCFunction)_ffi_new_with_allocator,
+                                 METH_VARARGS | METH_KEYWORDS};
+        result = PyCFunction_New(&md, allocator);
+    }
+    Py_DECREF(allocator);
+    return result;
 }
 
 PyDoc_STRVAR(ffi_cast_doc,
@@ -805,6 +869,7 @@
 #endif
  {"integer_const",(PyCFunction)ffi_int_const,METH_VKW,     ffi_int_const_doc},
  {"new",        (PyCFunction)ffi_new,        METH_VKW,     ffi_new_doc},
+{"new_allocator",(PyCFunction)ffi_new_allocator,METH_VKW,ffi_new_allocator_doc},
  {"new_handle", (PyCFunction)ffi_new_handle, METH_O,       ffi_new_handle_doc},
  {"offsetof",   (PyCFunction)ffi_offsetof,   METH_VARARGS, ffi_offsetof_doc},
  {"sizeof",     (PyCFunction)ffi_sizeof,     METH_O,       ffi_sizeof_doc},
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -236,6 +236,30 @@
             cdecl = self._typeof(cdecl)
         return self._backend.newp(cdecl, init)
 
+    def new_allocator(self, alloc=None, free=None,
+                      should_clear_after_alloc=True):
+        """Return a new allocator, i.e. a function that behaves like ffi.new()
+        but uses the provided low-level 'alloc' and 'free' functions.
+
+        'alloc' is called with the size as argument.  If it returns NULL, a
+        MemoryError is raised.  'free' is called with the result of 'alloc'
+        as argument.  Both can be either Python function or directly C
+        functions.  If 'free' is None, then no free function is called.
+        If both 'alloc' and 'free' are None, the default is used.
+
+        If 'should_clear_after_alloc' is set to False, then the memory
+        returned by 'alloc' is assumed to be already cleared (or you are
+        fine with garbage); otherwise CFFI will clear it.
+        """
+        compiled_ffi = self._backend.FFI()
+        allocator = compiled_ffi.new_allocator(alloc, free,
+                                               should_clear_after_alloc)
+        def allocate(cdecl, init=None):
+            if isinstance(cdecl, basestring):
+                cdecl = self._typeof(cdecl)
+            return allocator(cdecl, init)
+        return allocate
+
     def cast(self, cdecl, source):
         """Similar to a C cast: returns an instance of the named C
         type initialized with the given 'source'.  The source is
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -57,6 +57,77 @@
         assert tb.tb_frame.f_code.co_name == 'cb'
         assert tb.tb_frame.f_locals['n'] == 234
 
+    def test_ffi_new_allocator_2(self):
+        ffi = FFI(backend=self.Backend())
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        def myfree(raw):
+            seen.append(raw)
+        alloc1 = ffi.new_allocator(myalloc, myfree)
+        alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+                                   should_clear_after_alloc=False)
+        p1 = alloc1("int[10]")
+        p2 = alloc2("int[]", 10)
+        assert seen == [40, 40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert ffi.typeof(p2) == ffi.typeof("int[]")
+        assert ffi.sizeof(p2) == 40
+        assert p1[5] == 0
+        assert p2[6] == ord('X') * 0x01010101
+        raw1 = ffi.cast("char *", p1)
+        raw2 = ffi.cast("char *", p2)
+        del p1, p2
+        retries = 0
+        while len(seen) != 4:
+            retries += 1
+            assert retries <= 5
+            import gc; gc.collect()
+        assert seen == [40, 40, raw1, raw2]
+        assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+        assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+    def test_ffi_new_allocator_3(self):
+        ffi = FFI(backend=self.Backend())
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        alloc1 = ffi.new_allocator(myalloc)    # no 'free'
+        p1 = alloc1("int[10]")
+        assert seen == [40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert p1[5] == 0
+
+    def test_ffi_new_allocator_4(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(TypeError, ffi.new_allocator, free=lambda x: None)
+        #
+        def myalloc2(size):
+            raise LookupError
+        alloc2 = ffi.new_allocator(myalloc2)
+        py.test.raises(LookupError, alloc2, "int[5]")
+        #
+        def myalloc3(size):
+            return 42
+        alloc3 = ffi.new_allocator(myalloc3)
+        e = py.test.raises(TypeError, alloc3, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata object (got int)"
+        #
+        def myalloc4(size):
+            return ffi.cast("int", 42)
+        alloc4 = ffi.new_allocator(myalloc4)
+        e = py.test.raises(TypeError, alloc4, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+        #
+        def myalloc5(size):
+            return ffi.NULL
+        alloc5 = ffi.new_allocator(myalloc5)
+        py.test.raises(MemoryError, alloc5, "int[5]")
+
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -224,3 +224,95 @@
     n = (1 << 29) + 42
     code, message = ffi.getwinerror(code=n)
     assert code == n
+
+def test_ffi_new_allocator_1():
+    ffi = _cffi1_backend.FFI()
+    alloc1 = ffi.new_allocator()
+    alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
+    for retry in range(100):
+        p1 = alloc1("int[10]")
+        p2 = alloc2("int[10]")
+        combination = 0
+        for i in range(10):
+            assert p1[i] == 0
+            combination |= p2[i]
+            p1[i] = -42
+            p2[i] = -43
+        if combination != 0:
+            break
+        del p1, p2
+        import gc; gc.collect()
+    else:
+        raise AssertionError("cannot seem to get an int[10] not "
+                             "completely cleared")
+
+def test_ffi_new_allocator_2():
+    ffi = _cffi1_backend.FFI()
+    seen = []
+    def myalloc(size):
+        seen.append(size)
+        return ffi.new("char[]", "X" * size)
+    def myfree(raw):
+        seen.append(raw)
+    alloc1 = ffi.new_allocator(myalloc, myfree)
+    alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+                               should_clear_after_alloc=False)
+    p1 = alloc1("int[10]")
+    p2 = alloc2("int[]", 10)
+    assert seen == [40, 40]
+    assert ffi.typeof(p1) == ffi.typeof("int[10]")
+    assert ffi.sizeof(p1) == 40
+    assert ffi.typeof(p2) == ffi.typeof("int[]")
+    assert ffi.sizeof(p2) == 40
+    assert p1[5] == 0
+    assert p2[6] == ord('X') * 0x01010101
+    raw1 = ffi.cast("char *", p1)
+    raw2 = ffi.cast("char *", p2)
+    del p1, p2
+    retries = 0
+    while len(seen) != 4:
+        retries += 1
+        assert retries <= 5
+        import gc; gc.collect()
+    assert seen == [40, 40, raw1, raw2]
+    assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+    assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+def test_ffi_new_allocator_3():
+    ffi = _cffi1_backend.FFI()
+    seen = []
+    def myalloc(size):
+        seen.append(size)
+        return ffi.new("char[]", "X" * size)
+    alloc1 = ffi.new_allocator(myalloc)    # no 'free'
+    p1 = alloc1("int[10]")
+    assert seen == [40]
+    assert ffi.typeof(p1) == ffi.typeof("int[10]")
+    assert ffi.sizeof(p1) == 40
+    assert p1[5] == 0
+
+def test_ffi_new_allocator_4():
+    ffi = _cffi1_backend.FFI()
+    py.test.raises(TypeError, ffi.new_allocator, free=lambda x: None)
+    #
+    def myalloc2(size):
+        raise LookupError
+    alloc2 = ffi.new_allocator(myalloc2)
+    py.test.raises(LookupError, alloc2, "int[5]")
+    #
+    def myalloc3(size):
+        return 42
+    alloc3 = ffi.new_allocator(myalloc3)
+    e = py.test.raises(TypeError, alloc3, "int[5]")
+    assert str(e.value) == "alloc() must return a cdata object (got int)"
+    #
+    def myalloc4(size):
+        return ffi.cast("int", 42)
+    alloc4 = ffi.new_allocator(myalloc4)
+    e = py.test.raises(TypeError, alloc4, "int[5]")
+    assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+    #
+    def myalloc5(size):
+        return ffi.NULL
+    alloc5 = ffi.new_allocator(myalloc5)
+    py.test.raises(MemoryError, alloc5, "int[5]")

From noreply at buildbot.pypy.org  Mon Jul  6 17:15:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 17:15:42 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: Tweak to the docstring
Message-ID: <20150706151542.33F681C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78461:0dfa553a36e2
Date: 2015-07-06 16:14 +0200
http://bitbucket.org/pypy/pypy/changeset/0dfa553a36e2/

Log:	Tweak to the docstring

diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -427,15 +427,15 @@
     def descr_new_allocator(self, w_alloc, w_free,
                             should_clear_after_alloc=1):
         """\
-Return a new allocator.
+Return a new allocator, i.e. a function that behaves like ffi.new()
+but uses the provided low-level 'alloc' and 'free' functions.
 
 'alloc' is called with the size as argument.  If it returns NULL, a
 MemoryError is raised.  'free' is called with the result of 'alloc'
 as argument.  Both can be either Python function or directly C
-functions.  If 'free' is explicitly set to None, then no free function
-is called.
+functions.  If 'free' is None, then no free function is called.
+If both 'alloc' and 'free' are None, the default is used.
 
-If both 'alloc' and 'free' are None, the default is used.
 If 'should_clear_after_alloc' is set to False, then the memory
 returned by 'alloc' is assumed to be already cleared (or you are
 fine with garbage); otherwise CFFI will clear it.

From noreply at buildbot.pypy.org  Mon Jul  6 17:15:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 17:15:43 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: hg merge default
Message-ID: <20150706151543.5EDCF1C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78462:4461915472b8
Date: 2015-07-06 16:15 +0200
http://bitbucket.org/pypy/pypy/changeset/4461915472b8/

Log:	hg merge default

diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -103,12 +103,15 @@
 Howto
 -----
 
-In pypy/goal::
+Grab a copy of the pypy repository_.  In the directory pypy/goal, run::
 
    ../../rpython/bin/rpython -O2 --sandbox targetpypystandalone.py
 
 If you don't have a regular PyPy installed, you should, because it's
-faster to translate, but you can also run ``python translate.py`` instead.
+faster to translate; but you can also run the same line with ``python``
+in front.
+
+.. _repository: https://bitbucket.org/pypy/pypy
 
 
 To run it, use the tools in the pypy/sandbox directory::
@@ -136,8 +139,6 @@
 Not all operations are supported; e.g. if you type os.readlink('...'),
 the controller crashes with an exception and the subprocess is killed.
 Other operations make the subprocess die directly with a "Fatal RPython
-error".  None of this is a security hole; it just means that if you try
-to run some random program, it risks getting killed depending on the
-Python built-in functions it tries to call.  This is a matter of the
-sandboxing layer being incomplete so far, but it should not really be
-a problem in practice.
+error".  None of this is a security hole.  More importantly, *most other
+built-in modules are not enabled.  Please read all the warnings in this
+page before complaining about this.  Contributions welcome.*
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -363,6 +363,10 @@
     def _sizeof(self):
         return self.ctype.size
 
+    def with_gc(self, w_destructor):
+        with self as ptr:
+            return W_CDataGCP(self.space, ptr, self.ctype, self, w_destructor)
+
 
 class W_CDataMem(W_CData):
     """This is used only by the results of cffi.cast('int', x)
@@ -504,6 +508,20 @@
             self.length, self.space.type(self.w_keepalive).name)
 
 
+class W_CDataGCP(W_CData):
+    """For ffi.gc()."""
+    _attrs_ = ['w_original_cdata', 'w_destructor']
+    _immutable_fields_ = ['w_original_cdata', 'w_destructor']
+
+    def __init__(self, space, cdata, ctype, w_original_cdata, w_destructor):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.w_original_cdata = w_original_cdata
+        self.w_destructor = w_destructor
+
+    def __del__(self):
+        self.space.call_function(self.w_destructor, self.w_original_cdata)
+
+
 W_CData.typedef = TypeDef(
     '_cffi_backend.CData',
     __module__ = '_cffi_backend',
diff --git a/pypy/module/_cffi_backend/cgc.py b/pypy/module/_cffi_backend/cgc.py
deleted file mode 100644
--- a/pypy/module/_cffi_backend/cgc.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from rpython.rlib import jit
-
-
- at jit.dont_look_inside
-def gc_weakrefs_build(ffi, w_cdata, w_destructor):
-    from pypy.module._weakref import interp__weakref
-
-    space = ffi.space
-    if ffi.w_gc_wref_remove is None:
-        ffi.gc_wref_dict = {}
-        ffi.w_gc_wref_remove = space.getattr(space.wrap(ffi),
-                                             space.wrap("__gc_wref_remove"))
-
-    w_new_cdata = w_cdata.ctype.cast(w_cdata)
-    assert w_new_cdata is not w_cdata
-
-    w_ref = interp__weakref.make_weakref_with_callback(
-        space,
-        space.gettypefor(interp__weakref.W_Weakref),
-        w_new_cdata,
-        ffi.w_gc_wref_remove)
-
-    ffi.gc_wref_dict[w_ref] = (w_destructor, w_cdata)
-    return w_new_cdata
-
-
-def gc_wref_remove(ffi, w_ref):
-    (w_destructor, w_cdata) = ffi.gc_wref_dict.pop(w_ref)
-    ffi.space.call_function(w_destructor, w_cdata)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -10,7 +10,7 @@
 from pypy.module._cffi_backend import parse_c_type, realize_c_type
 from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray
 from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle
-from pypy.module._cffi_backend import cbuffer, func, cgc, wrapper
+from pypy.module._cffi_backend import cbuffer, func, wrapper
 from pypy.module._cffi_backend import cffi_opcode, allocator
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend.cdataobj import W_CData
@@ -348,10 +348,7 @@
 Later, when this new cdata object is garbage-collected,
 'destructor(old_cdata_object)' will be called."""
         #
-        return cgc.gc_weakrefs_build(self, w_cdata, w_destructor)
-
-    def descr___gc_wref_remove(self, w_ref):
-        return cgc.gc_wref_remove(self, w_ref)
+        return w_cdata.with_gc(w_destructor)
 
 
     @unwrap_spec(replace_with=str)
@@ -614,7 +611,6 @@
                                      W_FFIObject.set_errno,
                                      doc=W_FFIObject.doc_errno,
                                      cls=W_FFIObject),
-        __gc_wref_remove = interp2app(W_FFIObject.descr___gc_wref_remove),
         addressof   = interp2app(W_FFIObject.descr_addressof),
         alignof     = interp2app(W_FFIObject.descr_alignof),
         buffer      = interp2app(W_FFIObject.descr_buffer),
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -105,3 +105,9 @@
                     "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
+
+# ____________________________________________________________
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData)
+def gcp(space, w_cdata, w_destructor):
+    return w_cdata.with_gc(w_destructor)
diff --git a/rpython/rlib/test/test_longlong2float.py b/rpython/rlib/test/test_longlong2float.py
--- a/rpython/rlib/test/test_longlong2float.py
+++ b/rpython/rlib/test/test_longlong2float.py
@@ -88,7 +88,7 @@
 
 def test_compiled_encode_nan():
     fn2 = compile(fn_encode_nan, [float, int])
-    ints = [-2**31, 2**31-1, 42]
+    ints = [int(-2**31), int(2**31-1), 42]
     for x in enum_floats():
         y = ints.pop()
         ints.insert(0, y)

From noreply at buildbot.pypy.org  Mon Jul  6 17:15:44 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 17:15:44 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: error fixes
Message-ID: <20150706151544.80A381C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78463:d926a809bca0
Date: 2015-07-06 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/d926a809bca0/

Log:	error fixes

diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -16,7 +16,7 @@
         self.should_clear_after_alloc = should_clear_after_alloc
 
     def allocate(self, space, datasize, ctype, length=-1):
-        from pypy.module._cffi_backend import cdataobj, ctypeptr, ffi_obj
+        from pypy.module._cffi_backend import cdataobj, ctypeptr
         if self.w_alloc is None:
             if self.should_clear_after_alloc:
                 ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
@@ -29,15 +29,18 @@
             w_raw_cdata = space.call_function(self.w_alloc,
                                               space.wrap(datasize))
             if not isinstance(w_raw_cdata, cdataobj.W_CData):
-                raise oefmt(ffi_obj.get_ffi_error(space),
-                            "expected cdata object from the call to %R, "
-                            "got '%T'", self.w_alloc, w_raw_cdata)
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata object (got %T)",
+                            w_raw_cdata)
             if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray):
-                raise oefmt(ffi_obj.get_ffi_error(space),
-                            "expected cdata pointer from the call to %R, "
-                            "got '%s'", self.w_alloc, w_raw_cdata.ctype.name)
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata pointer, not '%s'",
+                            w_raw_cdata.ctype.name)
             #
             ptr = w_raw_cdata.unsafe_escaping_ptr()
+            if not ptr:
+                raise oefmt(space.w_MemoryError, "alloc() returned NULL")
+            #
             if self.should_clear_after_alloc:
                 rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0,
                               rffi.cast(rffi.SIZE_T, datasize))
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -340,3 +340,30 @@
         assert ffi.typeof(p1) == ffi.typeof("int[10]")
         assert ffi.sizeof(p1) == 40
         assert p1[5] == 0
+
+    def test_ffi_new_allocator_4(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(TypeError, ffi.new_allocator, free=lambda x: None)
+        #
+        def myalloc2(size):
+            raise LookupError
+        alloc2 = ffi.new_allocator(myalloc2)
+        raises(LookupError, alloc2, "int[5]")
+        #
+        def myalloc3(size):
+            return 42
+        alloc3 = ffi.new_allocator(myalloc3)
+        e = raises(TypeError, alloc3, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata object (got int)"
+        #
+        def myalloc4(size):
+            return ffi.cast("int", 42)
+        alloc4 = ffi.new_allocator(myalloc4)
+        e = raises(TypeError, alloc4, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+        #
+        def myalloc5(size):
+            return ffi.NULL
+        alloc5 = ffi.new_allocator(myalloc5)
+        raises(MemoryError, alloc5, "int[5]")

From noreply at buildbot.pypy.org  Mon Jul  6 17:33:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 17:33:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: translation fix
Message-ID: <20150706153326.3ED7D1C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78464:6d3732f1b1eb
Date: 2015-07-06 17:33 +0200
http://bitbucket.org/pypy/pypy/changeset/6d3732f1b1eb/

Log:	translation fix

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -498,6 +498,12 @@
         self.w_destructor = w_destructor
 
     def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space, W_CDataGCP.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataGCP)
         self.space.call_function(self.w_destructor, self.w_original_cdata)
 
 

From noreply at buildbot.pypy.org  Mon Jul  6 17:49:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 17:49:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: typo
Message-ID: <20150706154903.479771C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78465:bd2249a3ea29
Date: 2015-07-06 17:49 +0200
http://bitbucket.org/pypy/pypy/changeset/bd2249a3ea29/

Log:	typo

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -413,7 +413,7 @@
 
 
 class W_CDataPtrToStructOrUnion(W_CData):
-    """This subclass is used for the pointer returned by new('struct foo').
+    """This subclass is used for the pointer returned by new('struct foo *').
     It has a strong reference to a W_CDataNewOwning that really owns the
     struct, which is the object returned by the app-level expression 'p[0]'.
     But it is not itself owning any memory, although its repr says so;

From noreply at buildbot.pypy.org  Mon Jul  6 18:35:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 18:35:40 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: translation fixes
Message-ID: <20150706163540.87E1B1C06AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78466:799b38e51757
Date: 2015-07-06 17:50 +0200
http://bitbucket.org/pypy/pypy/changeset/799b38e51757/

Log:	translation fixes

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -430,6 +430,13 @@
     _attrs_ = ['w_free']
 
     def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space,
+                                     W_CDataNewNonStdFree.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataNewNonStdFree)
         self.space.call_function(self.w_free, self.w_raw_cdata)
 
 
@@ -519,6 +526,12 @@
         self.w_destructor = w_destructor
 
     def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space, W_CDataGCP.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataGCP)
         self.space.call_function(self.w_destructor, self.w_original_cdata)
 
 

From noreply at buildbot.pypy.org  Mon Jul  6 18:35:41 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 18:35:41 +0200 (CEST)
Subject: [pypy-commit] pypy cffi-new-allocator: Close branch, ready to merge
Message-ID: <20150706163541.B757D1C1016@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-new-allocator
Changeset: r78467:761b6b81f1cc
Date: 2015-07-06 18:34 +0200
http://bitbucket.org/pypy/pypy/changeset/761b6b81f1cc/

Log:	Close branch, ready to merge


From noreply at buildbot.pypy.org  Mon Jul  6 18:35:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 18:35:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: hg merge cffi-new-allocator
Message-ID: <20150706163543.0C1B91C06AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78468:5fcc7ff9de54
Date: 2015-07-06 18:34 +0200
http://bitbucket.org/pypy/pypy/changeset/5fcc7ff9de54/

Log:	hg merge cffi-new-allocator

	Support ffi.new_allocator()

diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -37,7 +37,6 @@
         'from_handle': 'handle.from_handle',
         '_get_types': 'func._get_types',
         'from_buffer': 'func.from_buffer',
-        'gcp': 'func.gcp',
 
         'string': 'func.string',
         'buffer': 'cbuffer.buffer',
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -0,0 +1,86 @@
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+class W_Allocator(W_Root):
+    _immutable_ = True
+
+    def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc):
+        self.ffi = ffi    # may be None
+        self.w_alloc = w_alloc
+        self.w_free = w_free
+        self.should_clear_after_alloc = should_clear_after_alloc
+
+    def allocate(self, space, datasize, ctype, length=-1):
+        from pypy.module._cffi_backend import cdataobj, ctypeptr
+        if self.w_alloc is None:
+            if self.should_clear_after_alloc:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=True)
+            else:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=False)
+            return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+        else:
+            w_raw_cdata = space.call_function(self.w_alloc,
+                                              space.wrap(datasize))
+            if not isinstance(w_raw_cdata, cdataobj.W_CData):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata object (got %T)",
+                            w_raw_cdata)
+            if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata pointer, not '%s'",
+                            w_raw_cdata.ctype.name)
+            #
+            ptr = w_raw_cdata.unsafe_escaping_ptr()
+            if not ptr:
+                raise oefmt(space.w_MemoryError, "alloc() returned NULL")
+            #
+            if self.should_clear_after_alloc:
+                rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0,
+                              rffi.cast(rffi.SIZE_T, datasize))
+            #
+            if self.w_free is None:
+                # use this class which does not have a __del__, but still
+                # keeps alive w_raw_cdata
+                res = cdataobj.W_CDataNewNonStdNoFree(space, ptr, ctype, length)
+            else:
+                res = cdataobj.W_CDataNewNonStdFree(space, ptr, ctype, length)
+                res.w_free = self.w_free
+            res.w_raw_cdata = w_raw_cdata
+            return res
+
+    @unwrap_spec(w_init=WrappedDefault(None))
+    def descr_call(self, space, w_arg, w_init):
+        ffi = self.ffi
+        assert ffi is not None
+        w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING | ffi.ACCEPT_CTYPE)
+        return w_ctype.newp(w_init, self)
+
+
+W_Allocator.typedef = TypeDef(
+        'FFIAllocator',
+        __call__ = interp2app(W_Allocator.descr_call),
+        )
+W_Allocator.typedef.acceptable_as_base_class = False
+
+
+def new_allocator(ffi, w_alloc, w_free, should_clear_after_alloc):
+    space = ffi.space
+    if space.is_none(w_alloc):
+        w_alloc = None
+    if space.is_none(w_free):
+        w_free = None
+    if w_alloc is None and w_free is not None:
+        raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'")
+    alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc))
+    return space.wrap(alloc)
+
+
+default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True)
+nonzero_allocator = W_Allocator(None, None, None,should_clear_after_alloc=False)
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -369,14 +369,13 @@
 
 
 class W_CDataMem(W_CData):
-    """This is the base class used for cdata objects that own and free
-    their memory.  Used directly by the results of cffi.cast('int', x)
-    or other primitive explicitly-casted types.  It is further subclassed
-    by W_CDataNewOwning."""
+    """This is used only by the results of cffi.cast('int', x)
+    or other primitive explicitly-casted types."""
     _attrs_ = []
 
-    def __init__(self, space, size, ctype):
-        cdata = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw', zero=True)
+    def __init__(self, space, ctype):
+        cdata = lltype.malloc(rffi.CCHARP.TO, ctype.size, flavor='raw',
+                              zero=False)
         W_CData.__init__(self, space, cdata, ctype)
 
     @rgc.must_be_light_finalizer
@@ -384,34 +383,63 @@
         lltype.free(self._ptr, flavor='raw')
 
 
-class W_CDataNewOwning(W_CDataMem):
-    """This is the class used for the cata objects created by newp()."""
-    _attrs_ = []
+class W_CDataNewOwning(W_CData):
+    """This is the abstract base class used for cdata objects created
+    by newp().  They create and free their own memory according to an
+    allocator."""
+
+    # the 'length' is either >= 0 for arrays, or -1 for pointers.
+    _attrs_ = ['length']
+    _immutable_fields_ = ['length']
+
+    def __init__(self, space, cdata, ctype, length=-1):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.length = length
 
     def _repr_extra(self):
         return self._repr_extra_owning()
 
-
-class W_CDataNewOwningLength(W_CDataNewOwning):
-    """Subclass with an explicit length, for allocated instances of
-    the C type 'foo[]'."""
-    _attrs_ = ['length']
-    _immutable_fields_ = ['length']
-
-    def __init__(self, space, size, ctype, length):
-        W_CDataNewOwning.__init__(self, space, size, ctype)
-        self.length = length
-
     def _sizeof(self):
-        from pypy.module._cffi_backend import ctypearray
         ctype = self.ctype
-        assert isinstance(ctype, ctypearray.W_CTypeArray)
-        return self.length * ctype.ctitem.size
+        if self.length >= 0:
+            from pypy.module._cffi_backend import ctypearray
+            assert isinstance(ctype, ctypearray.W_CTypeArray)
+            return self.length * ctype.ctitem.size
+        else:
+            return ctype.size
 
     def get_array_length(self):
         return self.length
 
 
+class W_CDataNewStd(W_CDataNewOwning):
+    """Subclass using the standard allocator, lltype.malloc()/lltype.free()"""
+    _attrs_ = []
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        lltype.free(self._ptr, flavor='raw')
+
+
+class W_CDataNewNonStdNoFree(W_CDataNewOwning):
+    """Subclass using a non-standard allocator, no free()"""
+    _attrs_ = ['w_raw_cdata']
+
+class W_CDataNewNonStdFree(W_CDataNewNonStdNoFree):
+    """Subclass using a non-standard allocator, with a free()"""
+    _attrs_ = ['w_free']
+
+    def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space,
+                                     W_CDataNewNonStdFree.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataNewNonStdFree)
+        self.space.call_function(self.w_free, self.w_raw_cdata)
+
+
 class W_CDataPtrToStructOrUnion(W_CData):
     """This subclass is used for the pointer returned by new('struct foo *').
     It has a strong reference to a W_CDataNewOwning that really owns the
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -28,7 +28,7 @@
     def _alignof(self):
         return self.ctitem.alignof()
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         datasize = self.size
         #
@@ -40,12 +40,10 @@
             except OverflowError:
                 raise OperationError(space.w_OverflowError,
                     space.wrap("array size would overflow a ssize_t"))
-            #
-            cdata = cdataobj.W_CDataNewOwningLength(space, datasize,
-                                                    self, length)
+        else:
+            length = self.length
         #
-        else:
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+        cdata = allocator.allocate(space, datasize, self, length)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -55,7 +55,7 @@
     def pack_list_of_items(self, cdata, w_ob):
         return False
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         raise oefmt(space.w_TypeError,
                     "expected a pointer or array ctype, got '%s'", self.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -63,7 +63,7 @@
             value = self._cast_result(value)
         else:
             value = self._cast_generic(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         self.write_raw_integer_data(w_cdata, value)
         return w_cdata
 
@@ -353,7 +353,7 @@
             value = self.cast_unicode(w_ob)
         else:
             value = space.float_w(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         if not isinstance(self, W_CTypePrimitiveLongDouble):
             w_cdata.write_raw_float_data(value)
         else:
@@ -446,7 +446,7 @@
         return self.space.wrap(value)
 
     def convert_to_object(self, cdata):
-        w_cdata = cdataobj.W_CDataMem(self.space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(self.space, self)
         with w_cdata as ptr:
             self._copy_longdouble(cdata, ptr)
         return w_cdata
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -185,7 +185,7 @@
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
         space = self.space
         ctitem = self.ctitem
@@ -205,14 +205,14 @@
                     datasize = ctitem.convert_struct_from_object(
                         lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
             #
-            cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
+            cdatastruct = allocator.allocate(space, datasize, ctitem)
             ptr = cdatastruct.unsafe_escaping_ptr()
             cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
                                                        self, cdatastruct)
         else:
             if self.is_char_or_unichar_ptr_or_array():
                 datasize *= 2       # forcefully add a null character
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+            cdata = allocator.allocate(space, datasize, self)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -81,10 +81,9 @@
     def copy_and_convert_to_object(self, source):
         space = self.space
         self.check_complete()
-        ob = cdataobj.W_CDataNewOwning(space, self.size, self)
-        with ob as target:
-            misc._raw_memcopy(source, target, self.size)
-        return ob
+        ptr = lltype.malloc(rffi.CCHARP.TO, self.size, flavor='raw', zero=False)
+        misc._raw_memcopy(source, ptr, self.size)
+        return cdataobj.W_CDataNewStd(space, ptr, self)
 
     def typeoffsetof_field(self, fieldname, following):
         self.force_lazy_struct()
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -11,7 +11,7 @@
 from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray
 from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle
 from pypy.module._cffi_backend import cbuffer, func, wrapper
-from pypy.module._cffi_backend import cffi_opcode
+from pypy.module._cffi_backend import cffi_opcode, allocator
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend.cdataobj import W_CData
 
@@ -44,6 +44,10 @@
 
 
 class W_FFIObject(W_Root):
+    ACCEPT_STRING = ACCEPT_STRING
+    ACCEPT_CTYPE  = ACCEPT_CTYPE
+    ACCEPT_CDATA  = ACCEPT_CDATA
+
     w_gc_wref_remove = None
 
     @jit.dont_look_inside
@@ -411,7 +415,31 @@
 pointer to the memory somewhere else, e.g. into another structure."""
         #
         w_ctype = self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CTYPE)
-        return w_ctype.newp(w_init)
+        return w_ctype.newp(w_init, allocator.default_allocator)
+
+
+    @unwrap_spec(w_alloc=WrappedDefault(None),
+                 w_free=WrappedDefault(None),
+                 should_clear_after_alloc=int)
+    def descr_new_allocator(self, w_alloc, w_free,
+                            should_clear_after_alloc=1):
+        """\
+Return a new allocator, i.e. a function that behaves like ffi.new()
+but uses the provided low-level 'alloc' and 'free' functions.
+
+'alloc' is called with the size as argument.  If it returns NULL, a
+MemoryError is raised.  'free' is called with the result of 'alloc'
+as argument.  Both can be either Python function or directly C
+functions.  If 'free' is None, then no free function is called.
+If both 'alloc' and 'free' are None, the default is used.
+
+If 'should_clear_after_alloc' is set to False, then the memory
+returned by 'alloc' is assumed to be already cleared (or you are
+fine with garbage); otherwise CFFI will clear it.
+        """
+        #
+        return allocator.new_allocator(self, w_alloc, w_free,
+                                       should_clear_after_alloc)
 
 
     def descr_new_handle(self, w_arg):
@@ -596,6 +624,7 @@
         getctype    = interp2app(W_FFIObject.descr_getctype),
         integer_const = interp2app(W_FFIObject.descr_integer_const),
         new         = interp2app(W_FFIObject.descr_new),
+        new_allocator = interp2app(W_FFIObject.descr_new_allocator),
         new_handle  = interp2app(W_FFIObject.descr_new_handle),
         offsetof    = interp2app(W_FFIObject.descr_offsetof),
         sizeof      = interp2app(W_FFIObject.descr_sizeof),
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,13 +1,13 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.module._cffi_backend import ctypeobj, cdataobj
+from pypy.module._cffi_backend import ctypeobj, cdataobj, allocator
 
 
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_init=WrappedDefault(None))
 def newp(space, w_ctype, w_init):
-    return w_ctype.newp(w_init)
+    return w_ctype.newp(w_init, allocator.default_allocator)
 
 # ____________________________________________________________
 
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -265,10 +265,105 @@
             assert p1[0] == 123
             seen.append(1)
         ffi.gc(p, destructor=destructor)    # instantly forgotten
-        _cffi1_backend.gcp(p, destructor=destructor)
         for i in range(5):
             if seen:
                 break
             import gc
             gc.collect()
-        assert seen == [1, 1]
+        assert seen == [1]
+
+    def test_ffi_new_allocator_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        alloc1 = ffi.new_allocator()
+        alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
+        for retry in range(100):
+            p1 = alloc1("int[10]")
+            p2 = alloc2("int[10]")
+            combination = 0
+            for i in range(10):
+                assert p1[i] == 0
+                combination |= p2[i]
+                p1[i] = -42
+                p2[i] = -43
+            if combination != 0:
+                break
+            del p1, p2
+            import gc; gc.collect()
+        else:
+            raise AssertionError("cannot seem to get an int[10] not "
+                                 "completely cleared")
+
+    def test_ffi_new_allocator_2(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        def myfree(raw):
+            seen.append(raw)
+        alloc1 = ffi.new_allocator(myalloc, myfree)
+        alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+                                   should_clear_after_alloc=False)
+        p1 = alloc1("int[10]")
+        p2 = alloc2("int[]", 10)
+        assert seen == [40, 40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert ffi.typeof(p2) == ffi.typeof("int[]")
+        assert ffi.sizeof(p2) == 40
+        assert p1[5] == 0
+        assert p2[6] == ord('X') * 0x01010101
+        raw1 = ffi.cast("char *", p1)
+        raw2 = ffi.cast("char *", p2)
+        del p1, p2
+        retries = 0
+        while len(seen) != 4:
+            retries += 1
+            assert retries <= 5
+            import gc; gc.collect()
+        assert seen == [40, 40, raw1, raw2]
+        assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+        assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+    def test_ffi_new_allocator_3(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        alloc1 = ffi.new_allocator(myalloc)    # no 'free'
+        p1 = alloc1("int[10]")
+        assert seen == [40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert p1[5] == 0
+
+    def test_ffi_new_allocator_4(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(TypeError, ffi.new_allocator, free=lambda x: None)
+        #
+        def myalloc2(size):
+            raise LookupError
+        alloc2 = ffi.new_allocator(myalloc2)
+        raises(LookupError, alloc2, "int[5]")
+        #
+        def myalloc3(size):
+            return 42
+        alloc3 = ffi.new_allocator(myalloc3)
+        e = raises(TypeError, alloc3, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata object (got int)"
+        #
+        def myalloc4(size):
+            return ffi.cast("int", 42)
+        alloc4 = ffi.new_allocator(myalloc4)
+        e = raises(TypeError, alloc4, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+        #
+        def myalloc5(size):
+            return ffi.NULL
+        alloc5 = ffi.new_allocator(myalloc5)
+        raises(MemoryError, alloc5, "int[5]")
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -9,6 +9,7 @@
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
+from pypy.module._cffi_backend import allocator
 
 
 class W_FunctionWrapper(W_Root):
@@ -74,7 +75,8 @@
             # then internally allocate the struct and pass a pointer to it as
             # a first argument.
             if locs[0] == 'R':
-                w_result_cdata = ctype.fargs[0].newp(space.w_None)
+                w_result_cdata = ctype.fargs[0].newp(space.w_None,
+                                                    allocator.nonzero_allocator)
                 args_w = [w_result_cdata] + args_w
                 prepare_args(space, rawfunctype, args_w, 1)
                 #
@@ -116,7 +118,7 @@
             # the equivalent of ffi.new()
             if space.is_w(w_arg, space.w_None):
                 continue
-            w_arg = farg.newp(w_arg)
+            w_arg = farg.newp(w_arg, allocator.default_allocator)
         args_w[i] = w_arg
 
 
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1249,3 +1249,8 @@
             lltype.Void,
             releasegil=False
         )
+c_memset = llexternal("memset",
+            [VOIDP, lltype.Signed, SIZE_T],
+            lltype.Void,
+            releasegil=False
+        )

From noreply at buildbot.pypy.org  Mon Jul  6 18:35:44 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 18:35:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add merged branch here
Message-ID: <20150706163544.3265B1C06AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78469:1505ff62cfa8
Date: 2015-07-06 18:35 +0200
http://bitbucket.org/pypy/pypy/changeset/1505ff62cfa8/

Log:	Add merged branch here

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -32,3 +32,4 @@
 ``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
 
 .. branch: cffi-callback-onerror
+.. branch: cffi-new-allocator

From noreply at buildbot.pypy.org  Mon Jul  6 18:36:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 18:36:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: import cffi/307f969f209f
Message-ID: <20150706163639.33E581C06AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78470:98ab85a7127c
Date: 2015-07-06 18:36 +0200
http://bitbucket.org/pypy/pypy/changeset/98ab85a7127c/

Log:	import cffi/307f969f209f

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -236,6 +236,30 @@
             cdecl = self._typeof(cdecl)
         return self._backend.newp(cdecl, init)
 
+    def new_allocator(self, alloc=None, free=None,
+                      should_clear_after_alloc=True):
+        """Return a new allocator, i.e. a function that behaves like ffi.new()
+        but uses the provided low-level 'alloc' and 'free' functions.
+
+        'alloc' is called with the size as argument.  If it returns NULL, a
+        MemoryError is raised.  'free' is called with the result of 'alloc'
+        as argument.  Both can be either Python function or directly C
+        functions.  If 'free' is None, then no free function is called.
+        If both 'alloc' and 'free' are None, the default is used.
+
+        If 'should_clear_after_alloc' is set to False, then the memory
+        returned by 'alloc' is assumed to be already cleared (or you are
+        fine with garbage); otherwise CFFI will clear it.
+        """
+        compiled_ffi = self._backend.FFI()
+        allocator = compiled_ffi.new_allocator(alloc, free,
+                                               should_clear_after_alloc)
+        def allocate(cdecl, init=None):
+            if isinstance(cdecl, basestring):
+                cdecl = self._typeof(cdecl)
+            return allocator(cdecl, init)
+        return allocate
+
     def cast(self, cdecl, source):
         """Similar to a C cast: returns an instance of the named C
         type initialized with the given 'source'.  The source is
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -58,6 +58,77 @@
         assert tb.tb_frame.f_code.co_name == 'cb'
         assert tb.tb_frame.f_locals['n'] == 234
 
+    def test_ffi_new_allocator_2(self):
+        ffi = FFI(backend=self.Backend())
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        def myfree(raw):
+            seen.append(raw)
+        alloc1 = ffi.new_allocator(myalloc, myfree)
+        alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+                                   should_clear_after_alloc=False)
+        p1 = alloc1("int[10]")
+        p2 = alloc2("int[]", 10)
+        assert seen == [40, 40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert ffi.typeof(p2) == ffi.typeof("int[]")
+        assert ffi.sizeof(p2) == 40
+        assert p1[5] == 0
+        assert p2[6] == ord('X') * 0x01010101
+        raw1 = ffi.cast("char *", p1)
+        raw2 = ffi.cast("char *", p2)
+        del p1, p2
+        retries = 0
+        while len(seen) != 4:
+            retries += 1
+            assert retries <= 5
+            import gc; gc.collect()
+        assert seen == [40, 40, raw1, raw2]
+        assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+        assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+    def test_ffi_new_allocator_3(self):
+        ffi = FFI(backend=self.Backend())
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        alloc1 = ffi.new_allocator(myalloc)    # no 'free'
+        p1 = alloc1("int[10]")
+        assert seen == [40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert p1[5] == 0
+
+    def test_ffi_new_allocator_4(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(TypeError, ffi.new_allocator, free=lambda x: None)
+        #
+        def myalloc2(size):
+            raise LookupError
+        alloc2 = ffi.new_allocator(myalloc2)
+        py.test.raises(LookupError, alloc2, "int[5]")
+        #
+        def myalloc3(size):
+            return 42
+        alloc3 = ffi.new_allocator(myalloc3)
+        e = py.test.raises(TypeError, alloc3, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata object (got int)"
+        #
+        def myalloc4(size):
+            return ffi.cast("int", 42)
+        alloc4 = ffi.new_allocator(myalloc4)
+        e = py.test.raises(TypeError, alloc4, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+        #
+        def myalloc5(size):
+            return ffi.NULL
+        alloc5 = ffi.new_allocator(myalloc5)
+        py.test.raises(MemoryError, alloc5, "int[5]")
+
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -225,3 +225,95 @@
     n = (1 << 29) + 42
     code, message = ffi.getwinerror(code=n)
     assert code == n
+
+def test_ffi_new_allocator_1():
+    ffi = _cffi1_backend.FFI()
+    alloc1 = ffi.new_allocator()
+    alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
+    for retry in range(100):
+        p1 = alloc1("int[10]")
+        p2 = alloc2("int[10]")
+        combination = 0
+        for i in range(10):
+            assert p1[i] == 0
+            combination |= p2[i]
+            p1[i] = -42
+            p2[i] = -43
+        if combination != 0:
+            break
+        del p1, p2
+        import gc; gc.collect()
+    else:
+        raise AssertionError("cannot seem to get an int[10] not "
+                             "completely cleared")
+
+def test_ffi_new_allocator_2():
+    ffi = _cffi1_backend.FFI()
+    seen = []
+    def myalloc(size):
+        seen.append(size)
+        return ffi.new("char[]", "X" * size)
+    def myfree(raw):
+        seen.append(raw)
+    alloc1 = ffi.new_allocator(myalloc, myfree)
+    alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+                               should_clear_after_alloc=False)
+    p1 = alloc1("int[10]")
+    p2 = alloc2("int[]", 10)
+    assert seen == [40, 40]
+    assert ffi.typeof(p1) == ffi.typeof("int[10]")
+    assert ffi.sizeof(p1) == 40
+    assert ffi.typeof(p2) == ffi.typeof("int[]")
+    assert ffi.sizeof(p2) == 40
+    assert p1[5] == 0
+    assert p2[6] == ord('X') * 0x01010101
+    raw1 = ffi.cast("char *", p1)
+    raw2 = ffi.cast("char *", p2)
+    del p1, p2
+    retries = 0
+    while len(seen) != 4:
+        retries += 1
+        assert retries <= 5
+        import gc; gc.collect()
+    assert seen == [40, 40, raw1, raw2]
+    assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+    assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+def test_ffi_new_allocator_3():
+    ffi = _cffi1_backend.FFI()
+    seen = []
+    def myalloc(size):
+        seen.append(size)
+        return ffi.new("char[]", "X" * size)
+    alloc1 = ffi.new_allocator(myalloc)    # no 'free'
+    p1 = alloc1("int[10]")
+    assert seen == [40]
+    assert ffi.typeof(p1) == ffi.typeof("int[10]")
+    assert ffi.sizeof(p1) == 40
+    assert p1[5] == 0
+
+def test_ffi_new_allocator_4():
+    ffi = _cffi1_backend.FFI()
+    py.test.raises(TypeError, ffi.new_allocator, free=lambda x: None)
+    #
+    def myalloc2(size):
+        raise LookupError
+    alloc2 = ffi.new_allocator(myalloc2)
+    py.test.raises(LookupError, alloc2, "int[5]")
+    #
+    def myalloc3(size):
+        return 42
+    alloc3 = ffi.new_allocator(myalloc3)
+    e = py.test.raises(TypeError, alloc3, "int[5]")
+    assert str(e.value) == "alloc() must return a cdata object (got int)"
+    #
+    def myalloc4(size):
+        return ffi.cast("int", 42)
+    alloc4 = ffi.new_allocator(myalloc4)
+    e = py.test.raises(TypeError, alloc4, "int[5]")
+    assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+    #
+    def myalloc5(size):
+        return ffi.NULL
+    alloc5 = ffi.new_allocator(myalloc5)
+    py.test.raises(MemoryError, alloc5, "int[5]")

From noreply at buildbot.pypy.org  Mon Jul  6 19:12:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 19:12:40 +0200 (CEST)
Subject: [pypy-commit] cffi default: Explicitly protect against variables
 that end up resolving at address NULL, 
Message-ID: <20150706171240.ECF5A1C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2214:57afc244cbc2
Date: 2015-07-06 19:09 +0200
http://bitbucket.org/cffi/cffi/changeset/57afc244cbc2/

Log:	Explicitly protect against variables that end up resolving at
	address NULL, which can occur in random cases like
	test_macro_var_callback

diff --git a/c/cglob.c b/c/cglob.c
--- a/c/cglob.c
+++ b/c/cglob.c
@@ -4,6 +4,7 @@
 typedef struct {
     PyObject_HEAD
 
+    PyObject         *gs_name;
     CTypeDescrObject *gs_type;
     char             *gs_data;
     gs_fetch_addr_fn  gs_fetch_addr;
@@ -12,6 +13,7 @@
 
 static void glob_support_dealloc(GlobSupportObject *gs)
 {
+    Py_DECREF(gs->gs_name);
     Py_DECREF(gs->gs_type);
     PyObject_Del(gs);
 }
@@ -41,14 +43,16 @@
 
 #define GlobSupport_Check(ob)  (Py_TYPE(ob) == &GlobSupport_Type)
 
-static PyObject *make_global_var(CTypeDescrObject *type, char *addr,
-                                 gs_fetch_addr_fn fetch_addr)
+static PyObject *make_global_var(PyObject *name, CTypeDescrObject *type,
+                                 char *addr, gs_fetch_addr_fn fetch_addr)
 {
     GlobSupportObject *gs = PyObject_New(GlobSupportObject, &GlobSupport_Type);
     if (gs == NULL)
         return NULL;
 
+    Py_INCREF(name);
     Py_INCREF(type);
+    gs->gs_name = name;
     gs->gs_type = type;
     gs->gs_data = addr;
     gs->gs_fetch_addr = fetch_addr;
@@ -68,18 +72,27 @@
         save_errno();
         Py_END_ALLOW_THREADS
     }
+    if (data == NULL) {
+        PyErr_Format(FFIError, "global variable '%s' is at address NULL",
+                     PyText_AS_UTF8(gs->gs_name));
+        return NULL;
+    }
     return data;
 }
 
 static PyObject *read_global_var(GlobSupportObject *gs)
 {
     void *data = fetch_global_var_addr(gs);
+    if (data == NULL)
+        return NULL;
     return convert_to_object(data, gs->gs_type);
 }
 
 static int write_global_var(GlobSupportObject *gs, PyObject *obj)
 {
     void *data = fetch_global_var_addr(gs);
+    if (data == NULL)
+        return -1;
     return convert_from_object(data, gs->gs_type, obj);
 }
 
@@ -91,7 +104,10 @@
         return NULL;
 
     data = fetch_global_var_addr(gs);
-    x = new_simple_cdata(data, (CTypeDescrObject *)ptrtype);
+    if (data != NULL)
+        x = new_simple_cdata(data, (CTypeDescrObject *)ptrtype);
+    else
+        x = NULL;
     Py_DECREF(ptrtype);
     return x;
 }
diff --git a/c/lib_obj.c b/c/lib_obj.c
--- a/c/lib_obj.c
+++ b/c/lib_obj.c
@@ -324,7 +324,7 @@
                 if (address == NULL)
                     return NULL;
             }
-            x = make_global_var(ct, address, NULL);
+            x = make_global_var(name, ct, address, NULL);
         }
         Py_DECREF(ct);
         break;
@@ -335,7 +335,7 @@
                             _CFFI_GETARG(g->type_op));
         if (ct == NULL)
             return NULL;
-        x = make_global_var(ct, NULL, (gs_fetch_addr_fn)g->address);
+        x = make_global_var(name, ct, NULL, (gs_fetch_addr_fn)g->address);
         Py_DECREF(ct);
         break;
 
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1128,3 +1128,22 @@
     assert values[2] == 42
     assert p[-1] == 41
     assert p[+1] == 42
+    #
+    # if get_my_value raises or returns nonsense, the exception is printed
+    # to stderr like with any callback, but then the C expression 'my_value'
+    # expand to '*NULL'.  We assume here that '&my_value' will return NULL
+    # without segfaulting, and check for NULL when accessing the variable.
+    @ffi.callback("int *(*)(void)")
+    def get_my_value():
+        raise LookupError
+    lib.get_my_value = get_my_value
+    py.test.raises(ffi.error, getattr, lib, 'my_value')
+    py.test.raises(ffi.error, setattr, lib, 'my_value', 50)
+    py.test.raises(ffi.error, ffi.addressof, lib, 'my_value')
+    @ffi.callback("int *(*)(void)")
+    def get_my_value():
+        return "hello"
+    lib.get_my_value = get_my_value
+    py.test.raises(ffi.error, getattr, lib, 'my_value')
+    e = py.test.raises(ffi.error, setattr, lib, 'my_value', 50)
+    assert str(e.value) == "global variable 'my_value' is at address NULL"

From noreply at buildbot.pypy.org  Mon Jul  6 19:12:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 19:12:42 +0200 (CEST)
Subject: [pypy-commit] cffi default: Python 3 compatibility
Message-ID: <20150706171242.05F141C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2215:eadeccf56e80
Date: 2015-07-06 19:11 +0200
http://bitbucket.org/cffi/cffi/changeset/eadeccf56e80/

Log:	Python 3 compatibility

diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -62,7 +62,7 @@
         seen = []
         def myalloc(size):
             seen.append(size)
-            return ffi.new("char[]", "X" * size)
+            return ffi.new("char[]", b"X" * size)
         def myfree(raw):
             seen.append(raw)
         alloc1 = ffi.new_allocator(myalloc, myfree)
@@ -94,7 +94,7 @@
         seen = []
         def myalloc(size):
             seen.append(size)
-            return ffi.new("char[]", "X" * size)
+            return ffi.new("char[]", b"X" * size)
         alloc1 = ffi.new_allocator(myalloc)    # no 'free'
         p1 = alloc1("int[10]")
         assert seen == [40]
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -251,7 +251,7 @@
     seen = []
     def myalloc(size):
         seen.append(size)
-        return ffi.new("char[]", "X" * size)
+        return ffi.new("char[]", b"X" * size)
     def myfree(raw):
         seen.append(raw)
     alloc1 = ffi.new_allocator(myalloc, myfree)
@@ -283,7 +283,7 @@
     seen = []
     def myalloc(size):
         seen.append(size)
-        return ffi.new("char[]", "X" * size)
+        return ffi.new("char[]", b"X" * size)
     alloc1 = ffi.new_allocator(myalloc)    # no 'free'
     p1 = alloc1("int[10]")
     assert seen == [40]
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1115,7 +1115,8 @@
     #
     @ffi.callback("int *(*)(void)")
     def get_my_value():
-        return values + it.next()
+        for nextvalue in it:
+            return values + nextvalue
     lib.get_my_value = get_my_value
     #
     values[0] = 41

From noreply at buildbot.pypy.org  Mon Jul  6 19:20:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 19:20:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update to cffi/57afc244cbc2
Message-ID: <20150706172039.3E8C21C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78471:bdc5e9f6119d
Date: 2015-07-06 19:20 +0200
http://bitbucket.org/pypy/pypy/changeset/bdc5e9f6119d/

Log:	Update to cffi/57afc244cbc2

diff --git a/pypy/module/_cffi_backend/cglob.py b/pypy/module/_cffi_backend/cglob.py
--- a/pypy/module/_cffi_backend/cglob.py
+++ b/pypy/module/_cffi_backend/cglob.py
@@ -1,3 +1,4 @@
+from pypy.interpreter.error import oefmt
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.module._cffi_backend.cdataobj import W_CData
@@ -10,26 +11,34 @@
 class W_GlobSupport(W_Root):
     _immutable_fields_ = ['w_ctype', 'ptr', 'fetch_addr']
 
-    def __init__(self, space, w_ctype, ptr=lltype.nullptr(rffi.CCHARP.TO),
+    def __init__(self, space, name, w_ctype, ptr=lltype.nullptr(rffi.CCHARP.TO),
                  fetch_addr=lltype.nullptr(rffi.VOIDP.TO)):
         self.space = space
+        self.name = name
         self.w_ctype = w_ctype
         self.ptr = ptr
         self.fetch_addr = fetch_addr
 
     def fetch_global_var_addr(self):
         if self.ptr:
-            return self.ptr
-        if not we_are_translated():
-            FNPTR = rffi.CCallback([], rffi.VOIDP)
-            fetch_addr = rffi.cast(FNPTR, self.fetch_addr)
-            result = fetch_addr()
+            result = self.ptr
         else:
-            # careful in translated versions: we need to call fetch_addr,
-            # but in a GIL-releasing way.  The easiest is to invoke a
-            # llexternal() helper.
-            result = pypy__cffi_fetch_var(self.fetch_addr)
-        return rffi.cast(rffi.CCHARP, result)
+            if not we_are_translated():
+                FNPTR = rffi.CCallback([], rffi.VOIDP)
+                fetch_addr = rffi.cast(FNPTR, self.fetch_addr)
+                result = fetch_addr()
+            else:
+                # careful in translated versions: we need to call fetch_addr,
+                # but in a GIL-releasing way.  The easiest is to invoke a
+                # llexternal() helper.
+                result = pypy__cffi_fetch_var(self.fetch_addr)
+            result = rffi.cast(rffi.CCHARP, result)
+        if not result:
+            from pypy.module._cffi_backend import ffi_obj
+            ffierror = ffi_obj.get_ffi_error(self.space)
+            raise oefmt(ffierror, "global variable '%s' is at address NULL",
+                        self.name)
+        return result
 
     def read_global_var(self):
         return self.w_ctype.convert_to_object(self.fetch_global_var_addr())
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -115,12 +115,12 @@
                 ptr = rffi.cast(rffi.CCHARP, g.c_address)
                 if not ptr:   # for dlopen() style
                     ptr = self.cdlopen_fetch(attr)
-                w_result = cglob.W_GlobSupport(space, w_ct, ptr=ptr)
+                w_result = cglob.W_GlobSupport(space, attr, w_ct, ptr=ptr)
                 #
             elif op == cffi_opcode.OP_GLOBAL_VAR_F:
                 w_ct = realize_c_type.realize_c_type(
                     self.ffi, self.ctx.c_types, getarg(g.c_type_op))
-                w_result = cglob.W_GlobSupport(space, w_ct,
+                w_result = cglob.W_GlobSupport(space, attr, w_ct,
                                                fetch_addr=g.c_address)
                 #
             elif (op == cffi_opcode.OP_CONSTANT_INT or
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1057,3 +1057,22 @@
         assert values[2] == 42
         assert p[-1] == 41
         assert p[+1] == 42
+        #
+        # if get_my_value raises or returns nonsense, the exception is printed
+        # to stderr like with any callback, but then the C expression 'my_value'
+        # expand to '*NULL'.  We assume here that '&my_value' will return NULL
+        # without segfaulting, and check for NULL when accessing the variable.
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            raise LookupError
+        lib.get_my_value = get_my_value
+        raises(ffi.error, getattr, lib, 'my_value')
+        raises(ffi.error, setattr, lib, 'my_value', 50)
+        raises(ffi.error, ffi.addressof, lib, 'my_value')
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            return "hello"
+        lib.get_my_value = get_my_value
+        raises(ffi.error, getattr, lib, 'my_value')
+        e = raises(ffi.error, setattr, lib, 'my_value', 50)
+        assert str(e.value) == "global variable 'my_value' is at address NULL"

From noreply at buildbot.pypy.org  Mon Jul  6 19:37:41 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 19:37:41 +0200 (CEST)
Subject: [pypy-commit] cffi default: Document ffi.new_allocator()
Message-ID: <20150706173741.C3D501C1047@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2216:0d2cff1af99f
Date: 2015-07-06 19:38 +0200
http://bitbucket.org/cffi/cffi/changeset/0d2cff1af99f/

Log:	Document ffi.new_allocator()

diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -820,6 +820,28 @@
 
 **ffi.RLTD_...**: constants: flags for ``ffi.dlopen()``.
 
+
+.. _`alternative allocators`:
+
+**ffi.new_allocator(alloc=None, free=None, should_clear_after_alloc=True)**:
+returns a new allocator.  An "allocator" is a callable that behaves like
+``ffi.new()`` but uses the provided low-level ``alloc`` and ``free``
+functions.  *New in version 1.2.*
+
+.. "versionadded:: 1.2" --- inlined in the previous paragraph
+
+``alloc()`` is invoked with the size as sole argument.  If it returns
+NULL, a MemoryError is raised.  Later, if ``free`` is not None, it will
+be called with the result of ``alloc()`` as argument.  Both can be either
+Python function or directly C functions.  If only ``free`` is None, then no
+free function is called.  If both ``alloc`` and ``free`` are None, the
+default alloc/free combination is used.  (In other words, the call
+``ffi.new(*args)`` is equivalent to ``ffi.new_allocator()(*args)``.)
+
+If ``should_clear_after_alloc`` is set to False, then the memory
+returned by ``alloc()`` is assumed to be already cleared (or you are
+fine with garbage); otherwise CFFI will clear it.
+
 .. _`Preparing and Distributing modules`: cdef.html#loading-libraries
 
 
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -41,7 +41,7 @@
   only for NULL: if you dereference random or dead pointers you might
   still get segfaults.
 
-* Issue #152: callbacks: added an argument ``ffi.callback(...,
+* Issue #152: callbacks__: added an argument ``ffi.callback(...,
   onerror=...)``.  If the main callback function raises an exception
   and ``onerror`` is provided, then ``onerror(exception, exc_value,
   traceback)`` is called.  This is similar to writing a ``try:
@@ -49,6 +49,12 @@
   signal) an exception can occur at the very start of the callback
   function---before it had time to enter the ``try: except:`` block.
 
+* Issue #115: added ``ffi.new_allocator()``, which officializes
+  support for `alternative allocators`__.
+
+.. __: using.html#callbacks
+.. __: using.html#alternative-allocators
+
 
 1.1.2
 =====

From noreply at buildbot.pypy.org  Mon Jul  6 19:42:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 19:42:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update to cffi/0d2cff1af99f
Message-ID: <20150706174222.C89251C0170@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78472:e522adea1e4f
Date: 2015-07-06 19:42 +0200
http://bitbucket.org/pypy/pypy/changeset/e522adea1e4f/

Log:	Update to cffi/0d2cff1af99f

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -63,7 +63,7 @@
         seen = []
         def myalloc(size):
             seen.append(size)
-            return ffi.new("char[]", "X" * size)
+            return ffi.new("char[]", b"X" * size)
         def myfree(raw):
             seen.append(raw)
         alloc1 = ffi.new_allocator(myalloc, myfree)
@@ -95,7 +95,7 @@
         seen = []
         def myalloc(size):
             seen.append(size)
-            return ffi.new("char[]", "X" * size)
+            return ffi.new("char[]", b"X" * size)
         alloc1 = ffi.new_allocator(myalloc)    # no 'free'
         p1 = alloc1("int[10]")
         assert seen == [40]
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -252,7 +252,7 @@
     seen = []
     def myalloc(size):
         seen.append(size)
-        return ffi.new("char[]", "X" * size)
+        return ffi.new("char[]", b"X" * size)
     def myfree(raw):
         seen.append(raw)
     alloc1 = ffi.new_allocator(myalloc, myfree)
@@ -284,7 +284,7 @@
     seen = []
     def myalloc(size):
         seen.append(size)
-        return ffi.new("char[]", "X" * size)
+        return ffi.new("char[]", b"X" * size)
     alloc1 = ffi.new_allocator(myalloc)    # no 'free'
     p1 = alloc1("int[10]")
     assert seen == [40]
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1116,7 +1116,8 @@
     #
     @ffi.callback("int *(*)(void)")
     def get_my_value():
-        return values + it.next()
+        for nextvalue in it:
+            return values + nextvalue
     lib.get_my_value = get_my_value
     #
     values[0] = 41
@@ -1129,3 +1130,22 @@
     assert values[2] == 42
     assert p[-1] == 41
     assert p[+1] == 42
+    #
+    # if get_my_value raises or returns nonsense, the exception is printed
+    # to stderr like with any callback, but then the C expression 'my_value'
+    # expand to '*NULL'.  We assume here that '&my_value' will return NULL
+    # without segfaulting, and check for NULL when accessing the variable.
+    @ffi.callback("int *(*)(void)")
+    def get_my_value():
+        raise LookupError
+    lib.get_my_value = get_my_value
+    py.test.raises(ffi.error, getattr, lib, 'my_value')
+    py.test.raises(ffi.error, setattr, lib, 'my_value', 50)
+    py.test.raises(ffi.error, ffi.addressof, lib, 'my_value')
+    @ffi.callback("int *(*)(void)")
+    def get_my_value():
+        return "hello"
+    lib.get_my_value = get_my_value
+    py.test.raises(ffi.error, getattr, lib, 'my_value')
+    e = py.test.raises(ffi.error, setattr, lib, 'my_value', 50)
+    assert str(e.value) == "global variable 'my_value' is at address NULL"

From noreply at buildbot.pypy.org  Mon Jul  6 19:46:41 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  6 Jul 2015 19:46:41 +0200 (CEST)
Subject: [pypy-commit] pypy unicode-dtype: close branch before merging
Message-ID: <20150706174641.A00FB1C0170@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-dtype
Changeset: r78473:54e026a9119b
Date: 2015-07-06 18:44 +0100
http://bitbucket.org/pypy/pypy/changeset/54e026a9119b/

Log:	close branch before merging


From noreply at buildbot.pypy.org  Mon Jul  6 19:46:43 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  6 Jul 2015 19:46:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge branch 'unicode-dtype'
Message-ID: <20150706174643.23C681C0170@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78474:d62f1b272ea9
Date: 2015-07-06 18:44 +0100
http://bitbucket.org/pypy/pypy/changeset/d62f1b272ea9/

Log:	merge branch 'unicode-dtype'

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -196,7 +196,12 @@
                     "'%T' object is not iterable", self)
 
     def descr_str(self, space):
-        return space.wrap(self.get_dtype(space).itemtype.str_format(self, add_quotes=False))
+        tp = self.get_dtype(space).itemtype
+        return space.wrap(tp.str_format(self, add_quotes=False))
+
+    def descr_repr(self, space):
+        tp = self.get_dtype(space).itemtype
+        return space.wrap(tp.str_format(self, add_quotes=True))
 
     def descr_format(self, space, w_spec):
         return space.format(self.item(space), w_spec)
@@ -618,16 +623,25 @@
         return W_StringBox(arr, 0, arr.dtype)
 
 class W_UnicodeBox(W_CharacterBox):
+    def __init__(self, value):
+        self._value = value
+
+    def convert_to(self, space, dtype):
+        if dtype.is_unicode():
+            return self
+        elif dtype.is_object():
+            return W_ObjectBox(space.wrap(self._value))
+        else:
+            raise oefmt(space.w_NotImplementedError,
+                        "Conversion from unicode not implemented yet")
+
+    def get_dtype(self, space):
+        from pypy.module.micronumpy.descriptor import new_unicode_dtype
+        return new_unicode_dtype(space, len(self._value))
+
     def descr__new__unicode_box(space, w_subtype, w_arg):
-        raise oefmt(space.w_NotImplementedError, "Unicode is not supported yet")
-        from pypy.module.micronumpy.descriptor import new_unicode_dtype
-        arg = space.unicode_w(space.unicode_from_object(w_arg))
-        # XXX size computations, we need tests anyway
-        arr = VoidBoxStorage(len(arg), new_unicode_dtype(space, len(arg)))
-        # XXX not this way, we need store
-        #for i in range(len(arg)):
-        #    arr.storage[i] = arg[i]
-        return W_UnicodeBox(arr, 0, arr.dtype)
+        value = space.unicode_w(space.unicode_from_object(w_arg))
+        return W_UnicodeBox(value)
 
 class W_ObjectBox(W_GenericBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY.OBJECT)
@@ -649,7 +663,7 @@
     __getitem__ = interp2app(W_GenericBox.descr_getitem),
     __iter__ = interp2app(W_GenericBox.descr_iter),
     __str__ = interp2app(W_GenericBox.descr_str),
-    __repr__ = interp2app(W_GenericBox.descr_str),
+    __repr__ = interp2app(W_GenericBox.descr_repr),
     __format__ = interp2app(W_GenericBox.descr_format),
     __int__ = interp2app(W_GenericBox.descr_int),
     __long__ = interp2app(W_GenericBox.descr_long),
diff --git a/pypy/module/micronumpy/casting.py b/pypy/module/micronumpy/casting.py
--- a/pypy/module/micronumpy/casting.py
+++ b/pypy/module/micronumpy/casting.py
@@ -325,6 +325,8 @@
         return complex_dtype
     elif space.isinstance_w(w_obj, space.w_str):
         return variable_dtype(space, 'S%d' % space.len_w(w_obj))
+    elif space.isinstance_w(w_obj, space.w_unicode):
+        return new_unicode_dtype(space, space.len_w(w_obj))
     return object_dtype
 
 @signature(ann.instance(W_Dtype), ann.instance(W_Dtype), returns=ann.bool())
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1052,20 +1052,6 @@
         assert d.name == "unicode256"
         assert d.num == 19
 
-    def test_string_boxes(self):
-        from numpy import str_
-        assert isinstance(str_(3), str_)
-
-    def test_unicode_boxes(self):
-        from numpy import unicode_
-        import sys
-        if '__pypy__' in sys.builtin_module_names:
-            exc = raises(NotImplementedError, unicode_, 3)
-            assert exc.value.message.find('not supported yet') >= 0
-        else:
-            u = unicode_(3)
-            assert isinstance(u, unicode)
-
     def test_character_dtype(self):
         import numpy as np
         from numpy import array, character
@@ -1133,7 +1119,7 @@
 
     def test_array_from_record(self):
         import numpy as np
-        a = np.array(('???', -999, -12345678.9), 
+        a = np.array(('???', -999, -12345678.9),
                      dtype=[('c', '|S3'), ('a', '<i8'), ('b', '<f8')])
         # Change the order of the keys
         b = np.array(a, dtype=[('a', '<i8'), ('b', '<f8'), ('c', '|S3')])
@@ -1141,7 +1127,7 @@
         assert b.dtype.fields['a'][1] == 0
         assert b['a'] == -999
         a = np.array(('N/A', 1e+20, 1e+20, 999999),
-                     dtype=[('name', '|S4'), ('x', '<f8'), 
+                     dtype=[('name', '|S4'), ('x', '<f8'),
                             ('y', '<f8'), ('block', '<i8', (2, 3))])
         assert (a['block'] == 999999).all()
 
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1,3 +1,4 @@
+# -*- encoding: utf-8 -*-
 import py
 import sys
 
@@ -322,6 +323,14 @@
         assert b.flags['C']
         assert (b == a).all()
 
+    def test_unicode(self):
+        import numpy as np
+        a = np.array([3, u'A&#255;', ''], dtype='U3')
+        assert a.shape == (3,)
+        assert a.dtype == np.dtype('U3')
+        assert a[0] == u'3'
+        assert a[1] == u'A&#255;'
+
     def test_dtype_attribute(self):
         import numpy as np
         a = np.array(40000, dtype='uint16')
@@ -380,6 +389,9 @@
         assert zeros((), dtype='S') == ''
         assert zeros((), dtype='S').shape == ()
         assert zeros((), dtype='S').dtype == '|S1'
+        assert zeros(5, dtype='U')[4] == u''
+        assert zeros(5, dtype='U').shape == (5,)
+        assert zeros(5, dtype='U').dtype == '<U1'
 
     def test_check_shape(self):
         import numpy as np
@@ -2423,6 +2435,12 @@
         a.fill(12)
         assert (a == '1').all()
 
+    def test_unicode_filling(self):
+        import numpy as np
+        a = np.empty((10,10), dtype='U1')
+        a.fill(12)
+        assert (a == u'1').all()
+
     def test_boolean_indexing(self):
         import numpy as np
         a = np.zeros((1, 3))
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -171,4 +171,8 @@
         assert 'a' * 100 in str(a)
         b = a.astype('S')
         assert 'a' * 100 in str(b)
-
+        a = np.array([123], dtype='U')
+        assert a[0] == u'123'
+        b = a.astype('O')
+        assert b[0] == u'123'
+        assert type(b[0]) is unicode
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -1,3 +1,4 @@
+# -*- encoding:utf-8 -*-
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
 class AppTestScalar(BaseNumpyAppTest):
@@ -457,3 +458,25 @@
 
         for t in complex64, complex128:
             _do_test(t, 17j, -17j)
+
+    def test_string_boxes(self):
+        from numpy import str_
+        assert isinstance(str_(3), str_)
+        assert str_(3) == '3'
+        assert str(str_(3)) == '3'
+        assert repr(str_(3)) == "'3'"
+
+    def test_unicode_boxes(self):
+        from numpy import unicode_
+        u = unicode_(3)
+        assert isinstance(u, unicode)
+        assert u == u'3'
+
+    def test_unicode_repr(self):
+        from numpy import unicode_
+        u = unicode_(3)
+        assert str(u) == '3'
+        assert repr(u) == "u'3'"
+        u = unicode_(u'A&#255;')
+        # raises(UnicodeEncodeError, "str(u)")  # XXX
+        assert repr(u) == repr(u'A&#255;')
diff --git a/pypy/module/micronumpy/test/test_selection.py b/pypy/module/micronumpy/test/test_selection.py
--- a/pypy/module/micronumpy/test/test_selection.py
+++ b/pypy/module/micronumpy/test/test_selection.py
@@ -210,22 +210,28 @@
             assert (c == a).all(), msg
 
     def test_sort_unicode(self):
+        import sys
         from numpy import array
         # test unicode sorts.
         s = 'aaaaaaaa'
-        try:
-            a = array([s + chr(i) for i in range(101)], dtype=unicode)
-            b = a[::-1].copy()
-        except:
-            skip('unicode type not supported yet')
-        for kind in ['q', 'm', 'h'] :
+        a = array([s + chr(i) for i in range(101)], dtype=unicode)
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h']:
             msg = "unicode sort, kind=%s" % kind
-            c = a.copy();
-            c.sort(kind=kind)
-            assert (c == a).all(), msg
-            c = b.copy();
-            c.sort(kind=kind)
-            assert (c == a).all(), msg
+            c = a.copy()
+            if '__pypy__' in sys.builtin_module_names:
+                exc = raises(NotImplementedError, "c.sort(kind=kind)")
+                assert 'non-numeric types' in exc.value.message
+            else:
+                c.sort(kind=kind)
+                assert (c == a).all(), msg
+            c = b.copy()
+            if '__pypy__' in sys.builtin_module_names:
+                exc = raises(NotImplementedError, "c.sort(kind=kind)")
+                assert 'non-numeric types' in exc.value.message
+            else:
+                c.sort(kind=kind)
+                assert (c == a).all(), msg
 
     def test_sort_objects(self):
         # test object array sorts.
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -11,7 +11,7 @@
     most_neg_value_of, LONG_BIT
 from rpython.rlib.rawstorage import (alloc_raw_storage,
     raw_storage_getitem_unaligned, raw_storage_setitem_unaligned)
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rstruct.ieee import (float_pack, float_unpack, unpack_float,
                                        pack_float80, unpack_float80)
 from rpython.rlib.rstruct.nativefmttable import native_is_bigendian
@@ -50,6 +50,7 @@
             pass
         return _raw_storage_getitem_unaligned(T, storage, offset)
 '''
+
 def simple_unary_op(func):
     specialize.argtype(1)(func)
     @functools.wraps(func)
@@ -2177,7 +2178,7 @@
             self._store(storage, i, offset, box, width)
 
 class UnicodeType(FlexibleType):
-    T = lltype.Char
+    T = lltype.UniChar
     num = NPY.UNICODE
     kind = NPY.UNICODELTR
     char = NPY.UNICODELTR
@@ -2189,58 +2190,121 @@
     def coerce(self, space, dtype, w_item):
         if isinstance(w_item, boxes.W_UnicodeBox):
             return w_item
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "coerce (probably from set_item) not implemented for unicode type"))
+        value = space.unicode_w(space.unicode_from_object(w_item))
+        return boxes.W_UnicodeBox(value)
 
     def store(self, arr, i, offset, box, native):
         assert isinstance(box, boxes.W_UnicodeBox)
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        value = box._value
+        with arr as storage:
+            self._store(storage, i, offset, box, arr.dtype.elsize)
+
+    @jit.unroll_safe
+    def _store(self, storage, i, offset, box, width):
+        size = min(width // 4, len(box._value))
+        for k in range(size):
+            index = i + offset + 4*k
+            data = rffi.cast(Int32.T, ord(box._value[k]))
+            raw_storage_setitem_unaligned(storage, index, data)
+        for k in range(size, width // 4):
+            index = i + offset + 4*k
+            data = rffi.cast(Int32.T, 0)
+            raw_storage_setitem_unaligned(storage, index, data)
 
     def read(self, arr, i, offset, dtype):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        if dtype is None:
+            dtype = arr.dtype
+        size = dtype.elsize // 4
+        builder = UnicodeBuilder(size)
+        with arr as storage:
+            for k in range(size):
+                index = i + offset + 4*k
+                codepoint = raw_storage_getitem_unaligned(
+                    Int32.T, arr.storage, index)
+                char = unichr(codepoint)
+                if char == u'\0':
+                    break
+                builder.append(char)
+        return boxes.W_UnicodeBox(builder.build())
 
     def str_format(self, item, add_quotes=True):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(item, boxes.W_UnicodeBox)
+        if add_quotes:
+            w_unicode = self.to_builtin_type(self.space, item)
+            return self.space.str_w(self.space.repr(w_unicode))
+        else:
+            # Same as W_UnicodeBox.descr_repr() but without quotes and prefix
+            from rpython.rlib.runicode import unicode_encode_unicode_escape
+            return unicode_encode_unicode_escape(item._value,
+                                                 len(item._value), 'strict')
 
     def to_builtin_type(self, space, box):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(box, boxes.W_UnicodeBox)
+        return space.wrap(box._value)
 
     def eq(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value == v2._value
 
     def ne(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value != v2._value
 
     def lt(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value < v2._value
 
     def le(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value <= v2._value
 
     def gt(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value > v2._value
 
     def ge(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        return v1._value >= v2._value
 
     def logical_and(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        if bool(v1) and bool(v2):
+            return Bool._True
+        return Bool._False
 
     def logical_or(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        if bool(v1) or bool(v2):
+            return Bool._True
+        return Bool._False
 
     def logical_not(self, v):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
-
-    @str_binary_op
+        assert isinstance(v, boxes.W_UnicodeBox)
+        return not bool(v)
+
     def logical_xor(self, v1, v2):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v1, boxes.W_UnicodeBox)
+        assert isinstance(v2, boxes.W_UnicodeBox)
+        a = bool(v1)
+        b = bool(v2)
+        return (not b and a) or (not a and b)
 
     def bool(self, v):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(v, boxes.W_UnicodeBox)
+        return bool(v._value)
 
     def fill(self, storage, width, native, box, start, stop, offset, gcstruct):
-        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+        assert isinstance(box, boxes.W_UnicodeBox)
+        for i in xrange(start, stop, width):
+            self._store(storage, i, offset, box, width)
 
 
 class VoidType(FlexibleType):

From noreply at buildbot.pypy.org  Mon Jul  6 19:46:44 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  6 Jul 2015 19:46:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <20150706174644.534181C0170@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78475:a72e61e7457b
Date: 2015-07-06 18:46 +0100
http://bitbucket.org/pypy/pypy/changeset/a72e61e7457b/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -33,3 +33,7 @@
 
 .. branch: cffi-callback-onerror
 .. branch: cffi-new-allocator
+
+.. branch: unicode-dtype
+
+Partial implementation of unicode dtype and unicode scalars.

From noreply at buildbot.pypy.org  Mon Jul  6 21:33:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  6 Jul 2015 21:33:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Extra operation we get now (unlikely to
	be noticeable)
Message-ID: <20150706193347.33F2C1C0207@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78476:5394cfa4447e
Date: 2015-07-06 21:34 +0200
http://bitbucket.org/pypy/pypy/changeset/5394cfa4447e/

Log:	Extra operation we get now (unlikely to be noticeable)

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -422,6 +422,7 @@
         guard_no_exception(descr=...)
         i112 = int_signext(i160, 2)
         setfield_gc(p167, ConstPtr(ptr85), descr=<FieldP pypy.module._cffi_backend.cdataobj.W_CData.inst_ctype .+>)
+        setfield_gc(p167, -1, descr=<FieldS pypy.module._cffi_backend.cdataobj.W_CDataNewOwning.inst_length .+>)
         i114 = int_ne(i160, i112)
         guard_false(i114, descr=...)
         --TICK--

From noreply at buildbot.pypy.org  Tue Jul  7 09:36:32 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 09:36:32 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start passing first
	optimizeopt tests
Message-ID: <20150707073632.BD4801C101F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78477:e72ba7da5f04
Date: 2015-07-06 16:52 +0200
http://bitbucket.org/pypy/pypy/changeset/e72ba7da5f04/

Log:	start passing first optimizeopt tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -55,7 +55,8 @@
             # the short preamble doesn't have fail descrs, they are patched in when it is used
             expected_short = self.parse(expected_short, want_fail_descr=False)
 
-        preamble = self.unroll_and_optimize(loop, call_pure_results)
+        info = self.unroll_and_optimize(loop, call_pure_results)
+        preamble = info.preamble
 
         #
         print
@@ -70,7 +71,7 @@
         print
         if expected_short:
             print "Short Preamble:"
-            short = loop.operations[0].getdescr().short_preamble
+            short = info.short_preamble
             print '\n'.join([str(o) for o in short])
             print
 
@@ -193,7 +194,7 @@
         p3 = cast_int_to_ptr(i2)
         #jump(i2) <- think about it
         """
-        self.optimize_loop(ops, expected, expected_short=short)
+        self.optimize_loop(ops, expected) #, expected_short=short)
 
     def test_reverse_of_cast_2(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -358,6 +358,11 @@
         except AttributeError:
             return ""
 
+class Info(object):
+    def __init__(self, preamble, short_preamble=None):
+        self.preamble = preamble
+        self.short_preamble = short_preamble
+
 class Storage(compile.ResumeGuardDescr):
     "for tests."
     def __init__(self, metainterp_sd=None, original_greenkey=None):
@@ -445,7 +450,7 @@
         preamble.inputargs = start_label.getarglist()
         preamble.operations = [start_label] + preamble_ops
         loop.operations = [end_label] + ops
-        return preamble
+        return Info(preamble)
 
     def foo(self):
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -15,8 +15,9 @@
 
 
 class PreambleOp(AbstractResOp):
-    def __init__(self, op, info):
+    def __init__(self, op, preamble_op, info):
         self.op = op
+        self.preamble_op = preamble_op
         self.info = info
 
     def getarg(self, i):
@@ -29,7 +30,7 @@
 class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
         op = preamble_op.op
-        self.optunroll.short.append(op)
+        self.optunroll.short.append(preamble_op.preamble_op)
         if preamble_op.info:
             preamble_op.info.make_guards(op, self.optunroll.short)
         return op
@@ -45,6 +46,7 @@
     def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
         self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
                                              optimizations)
+        self.short = []
         self.optimizer.optunroll = self
 
     def get_virtual_state(self, args):
@@ -219,8 +221,17 @@
         return ExportedState([], [])
 
     def import_state(self, targetop, exported_state):
+        # the mapping between input args (from old label) and what we need
+        # to actually emit
         for source, target in exported_state.inputarg_mapping:
             xxx
+        # import the optimizer state, starting from boxes that can be produced
+        # by short preamble
+        for op, preamble_op in exported_state.short_boxes.items():
+            if preamble_op.is_always_pure():
+                self.pure(op.getopnum(), PreambleOp(op, preamble_op, None))
+            else:
+                yyy
         return
         self.inputargs = targetop.getarglist()
         target_token = targetop.getdescr()

From noreply at buildbot.pypy.org  Tue Jul  7 09:36:34 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 09:36:34 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix fix fix
Message-ID: <20150707073634.0264E1C1048@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78478:7b6353983251
Date: 2015-07-06 17:04 +0200
http://bitbucket.org/pypy/pypy/changeset/7b6353983251/

Log:	fix fix fix

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -285,14 +285,6 @@
         else:
             zzz
 
-    def setinfo_from_preamble(self, op, old_info):
-        if isinstance(old_info, info.PtrInfo):
-            if op.is_constant():
-                return # nothing we can learn
-            known_class = old_info.get_known_class(self.cpu)
-            if known_class:
-                self.make_constant_class(op, known_class, False)
-
     def get_box_replacement(self, op):
         from rpython.jit.metainterp.optimizeopt.unroll import PreambleOp
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -81,7 +81,6 @@
         if expected_preamble:
             self.assert_equal(preamble, convert_old_style_to_targets(expected_preamble, jump=False),
                               text_right='expected preamble')
-            assert preamble.operations[-1].getdescr() == loop.operations[0].getdescr()
         if expected_short:
             short_preamble = TreeLoop('short preamble')
             assert short[0].getopnum() == rop.LABEL
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -10,6 +10,7 @@
 from rpython.jit.metainterp.compile import LoopCompileData
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN
+from rpython.jit.codewriter import heaptracker
 
 class FakeOptimizer(object):
     optearlyforce = None
@@ -71,3 +72,16 @@
         assert vs.state[0].level == LEVEL_UNKNOWN
         op = preamble.operations[0]
         assert es.short_boxes == {op: op}
+
+    def test_guard_class(self):
+        loop = """
+        [p0]
+        guard_class(p0, ConstClass(node_vtable)) []
+        jump(p0)
+        """
+        es, loop, preamble = self.optimize(loop)
+        p0 = loop.inputargs[0]
+        assert (heaptracker.adr2int(self.node_vtable_adr) ==
+                es.exported_infos[p0]._known_class.getint())
+
+
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -513,7 +513,7 @@
                       loop.operations
     if not jump:
         assert newloop.operations[-1].getopnum() == rop.JUMP
-        newloop.operations[-1] = ResOperation(rop.LABEL, newloop.operations[-1].getarglist(), descr=FakeDescr())
+        newloop.operations = newloop.operations[:-1]
     return newloop
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -3,6 +3,7 @@
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
 from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes
 from rpython.jit.metainterp.optimize import InvalidLoop
+from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
      Optimization
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
@@ -35,6 +36,15 @@
             preamble_op.info.make_guards(op, self.optunroll.short)
         return op
 
+    def setinfo_from_preamble(self, op, old_info):
+        op = self.get_box_replacement(op)
+        if isinstance(old_info, info.PtrInfo):
+            if op.is_constant():
+                return # nothing we can learn
+            known_class = old_info.get_known_class(self.cpu)
+            if known_class:
+                self.make_constant_class(op, known_class, False)
+
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -173,7 +183,11 @@
         inparg_mapping = [(start_label.getarg(i), end_args[i])
                           for i in range(len(end_args)) if
                           start_label.getarg(i) is not end_args[i]]
-        return ExportedState(inparg_mapping, virtual_state, [], sb.short_boxes)
+        infos = {}
+        for arg in end_args:
+            infos[arg] = self.optimizer.getinfo(arg)
+        return ExportedState(inparg_mapping, virtual_state, infos,
+                             sb.short_boxes)
 
 
         inputargs = virtual_state.make_inputargs(jump_args, self.optimizer)
@@ -224,7 +238,8 @@
         # the mapping between input args (from old label) and what we need
         # to actually emit
         for source, target in exported_state.inputarg_mapping:
-            xxx
+            if source is not target:
+                source.set_forwarded(target)
         # import the optimizer state, starting from boxes that can be produced
         # by short preamble
         for op, preamble_op in exported_state.short_boxes.items():
@@ -232,6 +247,9 @@
                 self.pure(op.getopnum(), PreambleOp(op, preamble_op, None))
             else:
                 yyy
+
+        for op, info in exported_state.exported_infos.iteritems():
+            self.optimizer.setinfo_from_preamble(op, info)
         return
         self.inputargs = targetop.getarglist()
         target_token = targetop.getdescr()

From noreply at buildbot.pypy.org  Tue Jul  7 09:36:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 09:36:38 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: invent new names for inputargs
 too, we need to rethink the check_lazy_fail_args at some point
Message-ID: <20150707073638.AAB0A1C1339@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78482:b69837aebecf
Date: 2015-07-07 09:36 +0200
http://bitbucket.org/pypy/pypy/changeset/b69837aebecf/

Log:	invent new names for inputargs too, we need to rethink the
	check_lazy_fail_args at some point

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -5,7 +5,8 @@
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
      ConstIntBound, MININT, MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, AbstractResOp, GuardResOp
+from rpython.jit.metainterp.resoperation import rop, AbstractResOp, GuardResOp,\
+     OpHelpers
 from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.rlib.objectmodel import specialize, we_are_translated
@@ -19,6 +20,13 @@
 llhelper.CONST_NULLREF = llhelper.CONST_NULL
 REMOVED = AbstractResOp()
 
+class LoopInfo(object):
+    pass
+
+class BasicLoopInfo(LoopInfo):
+    def __init__(self, inputargs):
+        self.inputargs = inputargs
+
 
 class Optimization(object):
     next_optimization = None
@@ -286,15 +294,9 @@
             zzz
 
     def get_box_replacement(self, op):
-        from rpython.jit.metainterp.optimizeopt.unroll import PreambleOp
-
-        orig_op = op
         if op is None:
             return op
-        res = op.get_box_replacement()
-        if isinstance(res, PreambleOp):
-            xxx
-        return res
+        return op.get_box_replacement()
 
     def force_box(self, op):
         op = self.get_box_replacement(op)
@@ -432,7 +434,12 @@
             return CONST_0
 
     def propagate_all_forward(self, inputargs, ops, call_pure_results=None):
-        self.init_inparg_dict_from(inputargs)
+        newargs = []
+        for inparg in inputargs:
+            new_arg = OpHelpers.inputarg_from_tp(inparg.type)
+            inparg.set_forwarded(new_arg)
+            newargs.append(new_arg)
+        self.init_inparg_dict_from(newargs)
         self.call_pure_results = call_pure_results
         for op in ops:
             self._really_emitted_operation = None
@@ -441,7 +448,7 @@
         #self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
-        return None, self._newoperations
+        return BasicLoopInfo(newargs), self._newoperations
 
     def send_extra_operation(self, op):
         self.first_optimization.propagate_forward(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -84,7 +84,9 @@
         call_pure_results = self._convert_call_pure_results(call_pure_results)
         compile_data = compile.SimpleCompileData(label_op, loop.operations,
                                                  call_pure_results)
-        _, ops = self._do_optimize_loop(compile_data)
+        info, ops = self._do_optimize_loop(compile_data)
+        label_op = ResOperation(rop.LABEL, info.inputargs)
+        loop.inputargs = info.inputargs
         loop.operations = [label_op] + ops
         #print '\n'.join([str(o) for o in loop.operations])
         self.loop = loop
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -3332,7 +3332,6 @@
         self.optimize_loop(ops, expected)
 
     def test_int_and_or_with_zero(self):
-        xxx
         ops = """
         [i0, i1]
         i2 = int_and(i0, 0)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -420,7 +420,6 @@
         #
         compile_data.enable_opts = self.enable_opts
         state = optimize_trace(metainterp_sd, None, compile_data)
-        compile_data.forget_optimization_info()
         return state
 
     def _convert_call_pure_results(self, d):
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -5,7 +5,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
-     Optimization
+     Optimization, LoopInfo
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
         BadVirtualState, VirtualStatesCantMatch)
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
@@ -257,7 +257,8 @@
         # by short preamble
         for op, preamble_op in exported_state.short_boxes.items():
             if preamble_op.is_always_pure():
-                self.pure(op.getopnum(), PreambleOp(op, preamble_op, None))
+                self.pure(op.getopnum(), PreambleOp(op, preamble_op,
+                                                self.optimizer.getinfo(op)))
             else:
                 yyy
 
@@ -694,8 +695,6 @@
                                       'it has at the start of the target loop')
             i += 1
 
-class LoopInfo(object):
-    pass
 
 class UnrollInfo(LoopInfo):
     """ A state after optimizing the peeled loop, contains the following:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -34,10 +34,12 @@
 
     def get_box_replacement(op):
         orig_op = op
+        c = 0
         while (op.get_forwarded() is not None and
                not op.get_forwarded().is_info_class):
+            c += 1
             op = op.get_forwarded()
-        if op is not orig_op:
+        if op is not orig_op and c > 1:
             orig_op.set_forwarded(op)
         return op
 
@@ -1136,3 +1138,13 @@
                 opnum == rop.CALL_RELEASE_GIL_R or
                 opnum == rop.CALL_RELEASE_GIL_F or
                 opnum == rop.CALL_RELEASE_GIL_N)
+
+    @staticmethod
+    def inputarg_from_tp(tp):
+        if tp == 'i':
+            return InputArgInt()
+        elif tp == 'r':
+            return InputArgRef()
+        else:
+            assert tp == 'f'
+            return InputArgFloat()

From noreply at buildbot.pypy.org  Tue Jul  7 09:36:35 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 09:36:35 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: hopefully a better fix
Message-ID: <20150707073635.28EA21C1206@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78479:ab1fed37b3d6
Date: 2015-07-06 18:35 +0200
http://bitbucket.org/pypy/pypy/changeset/ab1fed37b3d6/

Log:	hopefully a better fix

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -466,11 +466,8 @@
         assert not op.is_call_pure()
         orig_op = op
         op = self.replace_op_with(op, op.getopnum())
-        # XXX look in C and maybe specialize on number of args
         for i in range(op.numargs()):
             arg = self.force_box(op.getarg(i))
-            #self.ensure_imported(value)
-            #    newbox = value.force_box(self)
             op.setarg(i, arg)
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -177,6 +177,9 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    #def test_cast_1(self):
+    #    xxx
+
     def test_reverse_of_cast_1(self):
         ops = """
         [i0]
@@ -3332,6 +3335,7 @@
         self.optimize_loop(ops, expected)
 
     def test_int_and_or_with_zero(self):
+        xxx
         ops = """
         [i0, i1]
         i2 = int_and(i0, 0)
@@ -5341,6 +5345,7 @@
         self.optimize_loop(ops, expected)
 
     def test_lshift_rshift(self):
+        xxx
         ops = """
         [i1, i2, i2b, i1b]
         i3 = int_lshift(i1, i2)
@@ -6214,6 +6219,7 @@
         self.optimize_strunicode_loop(ops, expected, expected)
 
     def test_newstr_2(self):
+        xxx
         ops = """
         [i0, i1]
         p1 = newstr(2)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -443,6 +443,7 @@
         start_state, preamble_ops = self._do_optimize_loop(preamble_data,
                                                   call_pure_results)
         preamble_data.forget_optimization_info()
+        end_label = ResOperation(rop.LABEL, start_state.end_args)
         loop_data = compile.UnrolledLoopData(end_label, jump_op,
                                              ops + [jump_op], start_state)
         _, ops = self._do_optimize_loop(loop_data, call_pure_results)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -71,14 +71,14 @@
     
     def optimize_preamble(self, start_label, end_label, ops):
         self._check_no_forwarding([[start_label, end_label], ops])
-        self.optimizer.propagate_all_forward(start_label.getarglist(), ops)
+        self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops)
         exported_state = self.export_state(start_label, end_label)
         return exported_state, self.optimizer._newoperations
 
     def optimize_peeled_loop(self, start_label, end_jump, ops, state):
         self._check_no_forwarding([[start_label, end_jump], ops])
         self.import_state(start_label, state)
-        self.optimizer.propagate_all_forward(start_label.getarglist(), ops)
+        self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops)
         return None, self.optimizer._newoperations
 
     def random_garbage(self):
@@ -186,7 +186,7 @@
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
-        return ExportedState(inparg_mapping, virtual_state, infos,
+        return ExportedState(end_args, inparg_mapping, virtual_state, infos,
                              sb.short_boxes)
 
 
@@ -696,8 +696,9 @@
     * short boxes - a mapping op -> preamble_op
     """
     
-    def __init__(self, inputarg_mapping, virtual_state, exported_infos,
-                 short_boxes):
+    def __init__(self, end_args, inputarg_mapping, virtual_state,
+                 exported_infos, short_boxes):
+        self.end_args = end_args
         self.inputarg_mapping = inputarg_mapping
         self.virtual_state = virtual_state
         self.exported_infos = exported_infos
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -92,6 +92,9 @@
     def getarglist(self):
         raise NotImplementedError
 
+    def getarglist_copy(self):
+        return self.getarglist()
+
     def getarg(self, i):
         raise NotImplementedError
 
@@ -131,7 +134,7 @@
         from rpython.jit.metainterp.history import DONT_CHANGE
         
         if args is None:
-            args = self.getarglist()
+            args = self.getarglist_copy()
         if descr is None:
             descr = self.getdescr()
         if descr is DONT_CHANGE:
@@ -591,6 +594,9 @@
     def getarglist(self):
         return self._args
 
+    def getarglist_copy(self):
+        return self._args[:]
+
     def numargs(self):
         return len(self._args)
 

From noreply at buildbot.pypy.org  Tue Jul  7 09:36:36 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 09:36:36 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix for guard_value
Message-ID: <20150707073636.5A5CC1C120E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78480:127dc1dae3be
Date: 2015-07-06 18:49 +0200
http://bitbucket.org/pypy/pypy/changeset/127dc1dae3be/

Log:	fix for guard_value

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -75,7 +75,7 @@
 
 class UnrolledLoopData(CompileData):
     """ This represents label() ops jump with extra info that's from the
-    run of LoopCompileData
+    run of LoopCompileData. Jump goes to the same label
     """
     def __init__(self, start_label, end_jump, operations, state,
                  call_pure_results=None):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -445,12 +445,14 @@
         preamble_data.forget_optimization_info()
         end_label = ResOperation(rop.LABEL, start_state.end_args)
         loop_data = compile.UnrolledLoopData(end_label, jump_op,
-                                             ops + [jump_op], start_state)
+                                             ops, start_state)
         _, ops = self._do_optimize_loop(loop_data, call_pure_results)
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_label.getarglist()
         preamble.operations = [start_label] + preamble_ops
-        loop.operations = [end_label] + ops
+        emit_end_label = ResOperation(rop.LABEL, start_state.end_args)
+        loop.inputargs = start_state.end_args
+        loop.operations = [emit_end_label] + ops
         return Info(preamble)
 
     def foo(self):
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -79,6 +79,12 @@
         self._check_no_forwarding([[start_label, end_jump], ops])
         self.import_state(start_label, state)
         self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops)
+        jump_args = [self.get_box_replacement(op)
+                     for op in end_jump.getarglist()]
+        jump_args = state.virtual_state.make_inputargs(jump_args,
+                                                       self.optimizer)
+        jump_op = ResOperation(rop.JUMP, jump_args)
+        self.optimizer._newoperations.append(jump_op)
         return None, self.optimizer._newoperations
 
     def random_garbage(self):
@@ -186,7 +192,8 @@
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
-        return ExportedState(end_args, inparg_mapping, virtual_state, infos,
+        label_args = virtual_state.make_inputargs(end_args, self.optimizer)
+        return ExportedState(label_args, inparg_mapping, virtual_state, infos,
                              sb.short_boxes)
 
 

From noreply at buildbot.pypy.org  Tue Jul  7 09:36:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 09:36:37 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: improve checks for short
	preamble
Message-ID: <20150707073637.7B3AF1C1328@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78481:59e4de30e491
Date: 2015-07-06 19:00 +0200
http://bitbucket.org/pypy/pypy/changeset/59e4de30e491/

Log:	improve checks for short preamble

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -177,9 +177,6 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
-    #def test_cast_1(self):
-    #    xxx
-
     def test_reverse_of_cast_1(self):
         ops = """
         [i0]
@@ -196,7 +193,7 @@
         p3 = cast_int_to_ptr(i2)
         #jump(i2) <- think about it
         """
-        self.optimize_loop(ops, expected) #, expected_short=short)
+        self.optimize_loop(ops, expected, expected_short=short)
 
     def test_reverse_of_cast_2(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -446,14 +446,14 @@
         end_label = ResOperation(rop.LABEL, start_state.end_args)
         loop_data = compile.UnrolledLoopData(end_label, jump_op,
                                              ops, start_state)
-        _, ops = self._do_optimize_loop(loop_data, call_pure_results)
+        loop_info, ops = self._do_optimize_loop(loop_data, call_pure_results)
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_label.getarglist()
         preamble.operations = [start_label] + preamble_ops
         emit_end_label = ResOperation(rop.LABEL, start_state.end_args)
         loop.inputargs = start_state.end_args
         loop.operations = [emit_end_label] + ops
-        return Info(preamble)
+        return Info(preamble, loop_info.short_preamble)
 
     def foo(self):
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -56,7 +56,6 @@
     def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
         self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
                                              optimizations)
-        self.short = []
         self.optimizer.optunroll = self
 
     def get_virtual_state(self, args):
@@ -76,6 +75,7 @@
         return exported_state, self.optimizer._newoperations
 
     def optimize_peeled_loop(self, start_label, end_jump, ops, state):
+        self.short = []
         self._check_no_forwarding([[start_label, end_jump], ops])
         self.import_state(start_label, state)
         self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops)
@@ -85,7 +85,13 @@
                                                        self.optimizer)
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer._newoperations.append(jump_op)
-        return None, self.optimizer._newoperations
+        return (UnrollInfo(self.make_short_preamble(start_label.getarglist())),
+                self.optimizer._newoperations)
+
+    def make_short_preamble(self, args):
+        label = ResOperation(rop.LABEL, args)
+        short = [label] + self.short
+        return short
 
     def random_garbage(self):
         # WTF is the rest of this function
@@ -688,18 +694,28 @@
                                       'it has at the start of the target loop')
             i += 1
 
-class ShortPreambleBuilder(object):
-    """ A place that builds short preamble and the necessary
-    arguments for the label and the jump
+class LoopInfo(object):
+    pass
+
+class UnrollInfo(LoopInfo):
+    """ A state after optimizing the peeled loop, contains the following:
+
+    * short_preamble - list of operations that go into short preamble
     """
-
-class ExportedState(object):
+    def __init__(self, short_preamble):
+        self.short_preamble = short_preamble
+            
+class ExportedState(LoopInfo):
     """ Exported state consists of a few pieces of information:
 
     * inputarg_mapping - a list of tuples with original inputarg box
                          as the first element and the second element being
                          what it maps to (potentially const)
     * exported_infos - a mapping from ops to infos, including inputargs
+    * end_args - arguments that end up in the label leading to the next
+                 iteration
+    * virtual_state - instance of VirtualState representing current state
+                      of virtuals at this label
     * short boxes - a mapping op -> preamble_op
     """
     

From noreply at buildbot.pypy.org  Tue Jul  7 11:44:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 11:44:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: add an option to rename
	inputargs
Message-ID: <20150707094439.0A6201C1048@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78483:6080a2b3e0bf
Date: 2015-07-07 11:37 +0200
http://bitbucket.org/pypy/pypy/changeset/6080a2b3e0bf/

Log:	add an option to rename inputargs

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -433,12 +433,16 @@
         else:
             return CONST_0
 
-    def propagate_all_forward(self, inputargs, ops, call_pure_results=None):
-        newargs = []
-        for inparg in inputargs:
-            new_arg = OpHelpers.inputarg_from_tp(inparg.type)
-            inparg.set_forwarded(new_arg)
-            newargs.append(new_arg)
+    def propagate_all_forward(self, inputargs, ops, call_pure_results=None,
+                              rename_inputargs=True):
+        if rename_inputargs:
+            newargs = []
+            for inparg in inputargs:
+                new_arg = OpHelpers.inputarg_from_tp(inparg.type)
+                inparg.set_forwarded(new_arg)
+                newargs.append(new_arg)
+        else:
+            newargs = inputargs
         self.init_inparg_dict_from(newargs)
         self.call_pure_results = call_pure_results
         for op in ops:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -30,7 +30,6 @@
         operations = operations[:-1]
 
         preamble = TreeLoop('preamble')
-        preamble.inputargs = inputargs
 
         token = JitCellToken()
         start_label = ResOperation(rop.LABEL, inputargs, descr=TargetToken(token))
@@ -38,6 +37,7 @@
         compile_data = LoopCompileData(start_label, stop_label, operations)
         start_state, newops = self._do_optimize_loop(compile_data)
         preamble.operations = newops
+        preamble.inputargs = start_state.renamed_inputargs
         return start_state, loop, preamble
 
 class TestUnroll(BaseTestUnroll):
@@ -80,7 +80,7 @@
         jump(p0)
         """
         es, loop, preamble = self.optimize(loop)
-        p0 = loop.inputargs[0]
+        p0 = preamble.inputargs[0]
         assert (heaptracker.adr2int(self.node_vtable_adr) ==
                 es.exported_infos[p0]._known_class.getint())
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -447,7 +447,8 @@
                                              ops, start_state)
         loop_info, ops = self._do_optimize_loop(loop_data, call_pure_results)
         preamble = TreeLoop('preamble')
-        preamble.inputargs = start_label.getarglist()
+        preamble.inputargs = start_state.renamed_inputargs
+        start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs)
         preamble.operations = [start_label] + preamble_ops
         emit_end_label = ResOperation(rop.LABEL, start_state.end_args)
         loop.inputargs = start_state.end_args
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -70,15 +70,18 @@
     
     def optimize_preamble(self, start_label, end_label, ops):
         self._check_no_forwarding([[start_label, end_label], ops])
-        self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops)
-        exported_state = self.export_state(start_label, end_label)
+        info, newops = self.optimizer.propagate_all_forward(
+            start_label.getarglist()[:], ops)
+        exported_state = self.export_state(start_label, end_label,
+                                           info.inputargs)
         return exported_state, self.optimizer._newoperations
 
     def optimize_peeled_loop(self, start_label, end_jump, ops, state):
         self.short = []
         self._check_no_forwarding([[start_label, end_jump], ops])
         self.import_state(start_label, state)
-        self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops)
+        self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops,
+                                             rename_inputargs=False)
         jump_args = [self.get_box_replacement(op)
                      for op in end_jump.getarglist()]
         jump_args = state.virtual_state.make_inputargs(jump_args,
@@ -186,7 +189,7 @@
         return stop_target.targeting_jitcell_token is start_target.targeting_jitcell_token
 
 
-    def export_state(self, start_label, end_label):
+    def export_state(self, start_label, end_label, renamed_inputargs):
         original_label_args = end_label.getarglist()
         end_args = [self.get_box_replacement(a) for a in original_label_args]
         virtual_state = self.get_virtual_state(end_args)
@@ -200,7 +203,7 @@
             infos[arg] = self.optimizer.getinfo(arg)
         label_args = virtual_state.make_inputargs(end_args, self.optimizer)
         return ExportedState(label_args, inparg_mapping, virtual_state, infos,
-                             sb.short_boxes)
+                             sb.short_boxes, renamed_inputargs)
 
 
         inputargs = virtual_state.make_inputargs(jump_args, self.optimizer)
@@ -716,12 +719,14 @@
     * virtual_state - instance of VirtualState representing current state
                       of virtuals at this label
     * short boxes - a mapping op -> preamble_op
+    * renamed_inputargs - the start label arguments in optimized version
     """
     
     def __init__(self, end_args, inputarg_mapping, virtual_state,
-                 exported_infos, short_boxes):
+                 exported_infos, short_boxes, renamed_inputargs):
         self.end_args = end_args
         self.inputarg_mapping = inputarg_mapping
         self.virtual_state = virtual_state
         self.exported_infos = exported_infos
         self.short_boxes = short_boxes
+        self.renamed_inputargs = renamed_inputargs

From noreply at buildbot.pypy.org  Tue Jul  7 11:44:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 11:44:40 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix a bunch of tests
Message-ID: <20150707094440.3812A1C1048@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78484:37dac6d48e99
Date: 2015-07-07 11:44 +0200
http://bitbucket.org/pypy/pypy/changeset/37dac6d48e99/

Log:	fix a bunch of tests

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -203,6 +203,7 @@
             self.next_optimization.propagate_forward(postponed_op)
 
     def produce_potential_short_preamble_ops(self, sb):
+        return
         descrkeys = self.cached_fields.keys()
         if not we_are_translated():
             # XXX Pure operation of boxes that are cached in several places will
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -325,6 +325,8 @@
         self._newoperations = []
 
     def make_equal_to(self, op, newop):
+        if op is newop:
+            return
         opinfo = op.get_forwarded()
         if opinfo is not None:
             assert isinstance(opinfo, info.AbstractInfo)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -530,7 +530,7 @@
     def test_ooisnull_oononnull_via_virtual(self):
         ops = """
         [p0]
-        pv = new_with_vtable(ConstClass(node_vtable))
+        pv = new_with_vtable(descr=nodesize)
         setfield_gc(pv, p0, descr=valuedescr)
         guard_nonnull(p0) []
         p1 = getfield_gc_r(pv, descr=valuedescr)
@@ -775,7 +775,7 @@
         [i1, p2, p3]
         i3 = getfield_gc_i(p3, descr=valuedescr)
         escape_n(i3)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i1, descr=valuedescr)
         jump(i1, p1, p2)
         """
@@ -789,7 +789,7 @@
         [i1, p2]
         i3 = getfield_gc_i(p2, descr=valuedescr)
         escape_n(i3)
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         setfield_gc(p3, i1, descr=valuedescr)
         jump(i1, p3)
         """
@@ -801,7 +801,7 @@
         [i1, p2, p3]
         i3 = getfield_gc_i(p3, descr=valuedescr)
         escape_n(i3)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i1, descr=valuedescr)
         p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
@@ -820,7 +820,7 @@
         [i1, p2]
         i3 = getfield_gc_i(p2, descr=valuedescr)
         escape_n(i3)
-        p4 = new_with_vtable(ConstClass(node_vtable))
+        p4 = new_with_vtable(descr=nodesize)
         p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, i1, descr=valuedescr)
@@ -835,7 +835,7 @@
         p3sub = getfield_gc_r(p3, descr=nextdescr)
         i3 = getfield_gc_i(p3sub, descr=valuedescr)
         escape_n(i3)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
@@ -857,7 +857,7 @@
         [i1, p2, p2sub]
         i3 = getfield_gc_i(p2sub, descr=valuedescr)
         escape_n(i3)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
@@ -873,7 +873,7 @@
         i2b = int_is_true(i2)
         guard_true(i2b) []
         setfield_gc(p2, i2, descr=nextdescr)
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         jump(p2, p3)
         """
         preamble = """
@@ -890,7 +890,7 @@
         i2 = int_sub(i1, 1)
         i2b = int_is_true(i2)
         guard_true(i2b) []
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         setfield_gc(p3, i2, descr=nextdescr)
         jump(p3, i2)
         """
@@ -981,7 +981,7 @@
         [i, p0]
         i0 = getfield_gc_i(p0, descr=valuedescr)
         i1 = int_add(i0, i)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i1, descr=valuedescr)
         jump(i, p1)
         """
@@ -1003,7 +1003,7 @@
         [f, p0]
         f0 = getfield_gc_f(p0, descr=floatdescr)
         f1 = float_add(f0, f)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, f1, descr=floatdescr)
         jump(f, p1)
         """
@@ -1073,7 +1073,7 @@
         [p0]
         i0 = getfield_gc_i(p0, descr=valuedescr)
         guard_value(i0, 0) []
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         # the field 'value' has its default value of 0
         jump(p1)
         """
@@ -1092,7 +1092,7 @@
     def test_virtual_3(self):
         ops = """
         [i]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i, descr=valuedescr)
         i0 = getfield_gc_i(p1, descr=valuedescr)
         i1 = int_add(i0, 1)
@@ -1112,7 +1112,7 @@
         i1 = getfield_gc_i(p0, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i3 = int_add(i0, i1)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i2, descr=valuedescr)
         jump(i3, p1)
         """
@@ -1141,7 +1141,7 @@
         i3 = int_add(i0, i1)
         p2 = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2, i1, descr=valuedescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i2, descr=valuedescr)
         setfield_gc(p1, p2, descr=nextdescr)
         jump(i3, p1)
@@ -1235,7 +1235,7 @@
     def test_virtual_constant_isnull(self):
         ops = """
         [i0]
-        p0 = new_with_vtable(ConstClass(node_vtable))
+        p0 = new_with_vtable(descr=nodesize)
         setfield_gc(p0, NULL, descr=nextdescr)
         p2 = getfield_gc_r(p0, descr=nextdescr)
         i1 = ptr_eq(p2, NULL)
@@ -1254,7 +1254,7 @@
     def test_virtual_constant_isnonnull(self):
         ops = """
         [i0]
-        p0 = new_with_vtable(ConstClass(node_vtable))
+        p0 = new_with_vtable(descr=nodesize)
         setfield_gc(p0, ConstPtr(myptr), descr=nextdescr)
         p2 = getfield_gc_r(p0, descr=nextdescr)
         i1 = ptr_eq(p2, NULL)
@@ -1270,7 +1270,7 @@
         ops = """
         [i0, p1, p3]
         i28 = int_add(i0, 1)
-        p30 = new_with_vtable(ConstClass(node_vtable))
+        p30 = new_with_vtable(descr=nodesize)
         setfield_gc(p30, i28, descr=nextdescr)
         setfield_gc(p3, p30, descr=valuedescr)
         p45 = getfield_gc_r(p3, descr=valuedescr)
@@ -1281,7 +1281,7 @@
         [i0, p1, p3]
         i28 = int_add(i0, 1)
         i29 = int_add(i0, 2)
-        p30 = new_with_vtable(ConstClass(node_vtable))
+        p30 = new_with_vtable(descr=nodesize)
         setfield_gc(p30, i28, descr=nextdescr)
         setfield_gc(p3, p30, descr=valuedescr)
         p46 = same_as(p30) # This same_as should be killed by backend
@@ -1291,7 +1291,7 @@
         [i0, p1, p3]
         i28 = int_add(i0, 1)
         i29 = int_add(i0, 2)
-        p30 = new_with_vtable(ConstClass(node_vtable))
+        p30 = new_with_vtable(descr=nodesize)
         setfield_gc(p30, i28, descr=nextdescr)
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
@@ -1301,7 +1301,7 @@
     def test_nonvirtual_1(self):
         ops = """
         [i]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i, descr=valuedescr)
         i0 = getfield_gc_i(p1, descr=valuedescr)
         i1 = int_add(i0, 1)
@@ -1312,7 +1312,7 @@
         expected = """
         [i]
         i1 = int_add(i, 1)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i, descr=valuedescr)
         escape_n(p1)
         escape_n(p1)
@@ -1326,7 +1326,7 @@
         i0 = getfield_gc_i(p0, descr=valuedescr)
         escape_n(p0)
         i1 = int_add(i0, i)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i1, descr=valuedescr)
         jump(i, p1)
         """
@@ -1339,7 +1339,7 @@
         """
         expected = """
         [i, i1]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i1, descr=valuedescr)
         escape_n(p1)
         i2 = int_add(i1, i)
@@ -1350,7 +1350,7 @@
     def test_nonvirtual_later(self):
         ops = """
         [i]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i, descr=valuedescr)
         i1 = getfield_gc_i(p1, descr=valuedescr)
         escape_n(p1)
@@ -1360,7 +1360,7 @@
         """
         expected = """
         [i]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i, descr=valuedescr)
         escape_n(p1)
         i2 = getfield_gc_i(p1, descr=valuedescr)
@@ -1372,7 +1372,7 @@
     def test_nonvirtual_write_null_fields_on_force(self):
         ops = """
         [i]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i, descr=valuedescr)
         i1 = getfield_gc_i(p1, descr=valuedescr)
         setfield_gc(p1, 0, descr=valuedescr)
@@ -1382,7 +1382,7 @@
         """
         expected = """
         [i]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, 0, descr=valuedescr)
         escape_n(p1)
         i2 = getfield_gc_i(p1, descr=valuedescr)
@@ -1393,7 +1393,7 @@
     def test_getfield_gc_pure_1(self):
         ops = """
         [i]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i, descr=valuedescr)
         i1 = getfield_gc_pure_i(p1, descr=valuedescr)
         jump(i1)
@@ -1471,7 +1471,7 @@
         setfield_gc(p31, 0, descr=adescr)
         p33 = new_array(0, descr=arraydescr)
         setfield_gc(p31, p33, descr=bdescr)
-        p35 = new_with_vtable(ConstClass(node_vtable))
+        p35 = new_with_vtable(descr=nodesize)
         setfield_gc(p35, p31, descr=valuedescr)
         jump(p0, p35)
         """
@@ -1487,7 +1487,7 @@
         p18 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
-        p35 = new_with_vtable(ConstClass(node_vtable))
+        p35 = new_with_vtable(descr=nodesize)
         setfield_gc(p35, p31, descr=valuedescr)
         jump(p0, p35)
         """
@@ -1502,7 +1502,7 @@
         [p0, p8, p18, p19]
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
-        p35 = new_with_vtable(ConstClass(node_vtable))
+        p35 = new_with_vtable(descr=nodesize)
         setfield_gc(p35, p31, descr=valuedescr)
         jump(p0, p35, p19, p18)
         """
@@ -1677,7 +1677,7 @@
     def test_varray_forced_1(self):
         ops = """
         []
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         setfield_gc(p2, 3, descr=valuedescr)
         i1 = getfield_gc_i(p2, descr=valuedescr)    # i1 = const 3
         p1 = new_array(i1, descr=arraydescr)
@@ -2336,7 +2336,7 @@
     def test_duplicate_setfield_5(self):
         ops = """
         [p0, i1]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p0, p1, descr=nextdescr)
         setfield_raw(i1, i1, descr=valuedescr)    # random op with side-effects
@@ -2398,7 +2398,7 @@
         # a virtual, which we try hard to keep virtual
         ops = """
         [p1, i2, i3]
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, p2, descr=nextdescr)
         guard_true(i3) []
         i4 = int_neg(i2)
@@ -2428,7 +2428,7 @@
     def test_duplicate_setfield_residual_guard_3(self):
         ops = """
         [p1, i2, i3]
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p1, p2, descr=nextdescr)
         guard_true(i3) []
@@ -2688,7 +2688,7 @@
     def test_duplicate_setfield_virtual(self):
         ops = """
         [p1, i2, i3, p4]
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
         guard_true(i3) []
@@ -2699,7 +2699,7 @@
         [p1, i2, i3, p4]
         guard_true(i3) [p1, p4]
         i4 = int_neg(i2)
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
         i101 = same_as(i4)
@@ -2708,7 +2708,7 @@
         expected = """
         [p1, i2, i4, p4, i5]
         guard_true(i4) [p1, p4]
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
         jump(p1, i2, i5, p4, i5)
@@ -2722,7 +2722,7 @@
         guard_nonnull(p4) []
         escape_n(p4)
         #
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         p3 = escape_r()
         setfield_gc(p2, p3, descr=nextdescr)
         jump(i0, p2)
@@ -2772,8 +2772,8 @@
         guard_class(p3, ConstClass(node_vtable)) []
         setfield_gc(p3, p2, descr=otherdescr)
         p1a = new_with_vtable(ConstClass(node_vtable2))
-        p2a = new_with_vtable(ConstClass(node_vtable))
-        p3a = new_with_vtable(ConstClass(node_vtable))
+        p2a = new_with_vtable(descr=nodesize)
+        p3a = new_with_vtable(descr=nodesize)
         escape_n(p3a)
         setfield_gc(p1a, p2a, descr=nextdescr)
         setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2786,7 +2786,7 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc_r(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
-        p3a = new_with_vtable(ConstClass(node_vtable))
+        p3a = new_with_vtable(descr=nodesize)
         setfield_gc(p3, p2, descr=otherdescr)
         escape_n(p3a)
         jump(p3a)
@@ -2798,9 +2798,9 @@
         # p3 = getfield_gc(p1, descr=otherdescr)# p3a
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
-        # p2a = new_with_vtable(ConstClass(node_vtable))
-        p3anew = new_with_vtable(ConstClass(node_vtable))
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        # p2a = new_with_vtable(descr=nodesize)
+        p3anew = new_with_vtable(descr=nodesize)
+        p2 = new_with_vtable(descr=nodesize)
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
         escape_n(p3anew)
         jump(p3anew)
@@ -2820,9 +2820,9 @@
         guard_nonnull(12) []
         guard_class(p3, ConstClass(node_vtable)) []
         p1a = new_with_vtable(ConstClass(node_vtable2))
-        p2a = new_with_vtable(ConstClass(node_vtable))
+        p2a = new_with_vtable(descr=nodesize)
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
+        p3a = new_with_vtable(descr=nodesize)
         escape_n(p3a)
         setfield_gc(p1a, p2a, descr=nextdescr)
         setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2836,8 +2836,8 @@
         p3 = getfield_gc_r(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
-        p3a = new_with_vtable(ConstClass(node_vtable))
-        p2a = new_with_vtable(ConstClass(node_vtable))
+        p3a = new_with_vtable(descr=nodesize)
+        p2a = new_with_vtable(descr=nodesize)
         setfield_gc(p3, p2a, descr=otherdescr)
         escape_n(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
@@ -2846,8 +2846,8 @@
         """
         expected = """
         [p2, p3]
-        p3a = new_with_vtable(ConstClass(node_vtable))
-        p2a = new_with_vtable(ConstClass(node_vtable))
+        p3a = new_with_vtable(descr=nodesize)
+        p2a = new_with_vtable(descr=nodesize)
         setfield_gc(p3, p2a, descr=otherdescr)
         escape_n(p3a)
         jump(p2a, p3a)
@@ -2857,7 +2857,7 @@
     def test_bug_4(self):
         ops = """
         [p9]
-        p30 = new_with_vtable(ConstClass(node_vtable))
+        p30 = new_with_vtable(descr=nodesize)
         setfield_gc(ConstPtr(myptr), p9, descr=nextdescr)
         jump(p30)
         """
@@ -2868,7 +2868,7 @@
         """
         expected = """
         []
-        p30 = new_with_vtable(ConstClass(node_vtable))
+        p30 = new_with_vtable(descr=nodesize)
         setfield_gc(ConstPtr(myptr), p30, descr=nextdescr)
         jump()
         """
@@ -2906,7 +2906,7 @@
         [p1]
         guard_isnull(p1) []
         #
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         jump(p2)
         """
         self.raises(InvalidLoop, self.optimize_loop, ops, "crash!")
@@ -2916,7 +2916,7 @@
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
         #
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         escape_n(p2)      # prevent it from staying Virtual
         jump(p2)
         """
@@ -2928,8 +2928,8 @@
         p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_isnull(p2) []
         #
-        p3 = new_with_vtable(ConstClass(node_vtable))
-        p4 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
+        p4 = new_with_vtable(descr=nodesize)
         setfield_gc(p3, p4, descr=nextdescr)
         jump(p3)
         """
@@ -2938,7 +2938,7 @@
     def test_invalid_loop_guard_value_of_virtual(self):
         ops = """
         [p1]
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         guard_value(p2, ConstPtr(myptr)) []
         jump(p2)
         """
@@ -3277,7 +3277,7 @@
     def test_ovf_guard_in_short_preamble1(self):
         ops = """
         [p8, p11, i24]
-        p26 = new_with_vtable(ConstClass(node_vtable))
+        p26 = new_with_vtable(descr=nodesize)
         setfield_gc(p26, i24, descr=adescr)
         i34 = getfield_gc_pure_i(p11, descr=valuedescr)
         i35 = getfield_gc_pure_i(p26, descr=adescr)
@@ -3303,7 +3303,7 @@
         i22 = getfield_gc_i(p16, descr=nextdescr)
         i23 = int_mul(i17, i22)
         i24 = int_add(i21, i23)
-        p26 = new_with_vtable(ConstClass(node_vtable))
+        p26 = new_with_vtable(descr=nodesize)
         setfield_gc(p26, i24, descr=adescr)
         i28 = int_add(i17, 1)
         setfield_gc(p8, i28, descr=nextdescr)
@@ -3312,7 +3312,7 @@
         guard_nonnull(p12) []
         i36 = int_add_ovf(i34, i35)
         guard_no_overflow() []
-        p38 = new_with_vtable(ConstClass(node_vtable))
+        p38 = new_with_vtable(descr=nodesize)
         setfield_gc(p38, i36, descr=adescr)
         jump(p8, p11, p26)
         """
@@ -3937,8 +3937,8 @@
         ops = """
         [p0, i1]
         #
-        p1 = new_with_vtable(ConstClass(node_vtable))
-        p1b = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
+        p1b = new_with_vtable(descr=nodesize)
         setfield_gc(p1b, 252, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
         #
@@ -3963,8 +3963,8 @@
         guard_not_forced() [i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
-        p1b = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
+        p1b = new_with_vtable(descr=nodesize)
         setfield_gc(p1b, 252, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
         setfield_gc(p2, p1, descr=virtualforceddescr)
@@ -3977,8 +3977,8 @@
         ops = """
         [p0, i1]
         #
-        p1 = new_with_vtable(ConstClass(node_vtable))
-        p1b = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
+        p1b = new_with_vtable(descr=nodesize)
         setfield_gc(p1b, i1, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
         #
@@ -4003,8 +4003,8 @@
         guard_not_forced() [p2, i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
-        p1b = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
+        p1b = new_with_vtable(descr=nodesize)
         setfield_gc(p1b, i1, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
         setfield_gc(p2, p1, descr=virtualforceddescr)
@@ -4023,8 +4023,8 @@
         ops = """
         [p0, i1]
         #
-        p1 = new_with_vtable(ConstClass(node_vtable))
-        p1b = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
+        p1b = new_with_vtable(descr=nodesize)
         setfield_gc(p1b, i1, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
         #
@@ -4071,7 +4071,7 @@
     def test_vref_virtual_after_finish(self):
         ops = """
         [i1]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         p2 = virtual_ref(p1, 7)
         escape_n(p2)
         virtual_ref_finish(p2, p1)
@@ -4086,7 +4086,7 @@
         setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p2, p3, descr=virtualtokendescr)
         escape_n(p2)
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         call_may_force_n(i1, descr=mayforcevirtdescr)
@@ -4820,7 +4820,7 @@
         guard_no_overflow() []
         i4360p = int_sub_ovf(i4362, 1)
         guard_no_overflow() []
-        p4364 = new_with_vtable(ConstClass(node_vtable))
+        p4364 = new_with_vtable(descr=nodesize)
         setfield_gc(p4364, i4362, descr=valuedescr)
         jump(p4364)
         """
@@ -5341,7 +5341,6 @@
         self.optimize_loop(ops, expected)
 
     def test_lshift_rshift(self):
-        xxx
         ops = """
         [i1, i2, i2b, i1b]
         i3 = int_lshift(i1, i2)
@@ -5945,7 +5944,7 @@
         guard_class(p14, 17273920) []
         guard_class(p14, 17273920) []
 
-        p75 = new_with_vtable(ConstClass(node_vtable))
+        p75 = new_with_vtable(descr=nodesize)
         setfield_gc(p75, p14, descr=inst_w_seq)
         setfield_gc(p75, 0, descr=inst_index)
         guard_class(p75, ConstClass(node_vtable)) []
@@ -6026,7 +6025,7 @@
         ops = """
         [p0, p1, pinv]
         i1 = getfield_gc_i(pinv, descr=valuedescr)
-        p2 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
         setfield_gc(p2, i1, descr=nextdescr)
         """
         py.test.skip("no test here")
@@ -6215,7 +6214,6 @@
         self.optimize_strunicode_loop(ops, expected, expected)
 
     def test_newstr_2(self):
-        xxx
         ops = """
         [i0, i1]
         p1 = newstr(2)
@@ -7314,7 +7312,7 @@
         [p19, p20, p21]
         p1 = getfield_gc_r(p20, descr=valuedescr)
         p2 = getfield_gc_r(p1, descr=otherdescr)
-        pv = new_with_vtable(ConstClass(node_vtable))
+        pv = new_with_vtable(descr=nodesize)
         setfield_gc(pv, p19, descr=valuedescr)
         p22 = getfield_gc_r(p19, descr=otherdescr)
         guard_value(p19, ConstPtr(myptr)) []
@@ -7435,7 +7433,7 @@
 
         ops = """
         [p0]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, p0, descr=valuedescr)
         escape_n(p1)
         p2 = getfield_gc_pure_r(p1, descr=valuedescr)
@@ -7444,7 +7442,7 @@
         """
         expected = """
         [p0]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, p0, descr=valuedescr)
         escape_n(p1)
         escape_n(p0)
@@ -7457,7 +7455,7 @@
         [i0, i1]
         p0 = escape_r()
         i2 = escape_i()
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         setarrayitem_gc(p0, 2, p1, descr=arraydescr)
         guard_true(i2) []
         setarrayitem_gc(p0, 2, p0, descr=arraydescr)
@@ -7476,7 +7474,7 @@
     def test_force_virtualizable_virtual(self):
         ops = """
         [i0]
-        p1 = new_with_vtable(ConstClass(node_vtable))
+        p1 = new_with_vtable(descr=nodesize)
         cond_call(1, 123, p1, descr=clear_vable)
         jump(i0)
         """
@@ -7630,7 +7628,7 @@
     def test_duplicated_virtual(self):
         ops = """
         [p1, p2]
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         jump(p3, p3)
         """
         expected = """
@@ -7642,7 +7640,7 @@
     def test_duplicated_aliased_virtual(self):
         ops = """
         [p1, p2]
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         setfield_gc(p3, p3, descr=nextdescr)
         p4 = getfield_gc_r(p3, descr=nextdescr)
         jump(p3, p4)
@@ -7658,7 +7656,7 @@
         [p1, p2, i0]
         i2 = int_lt(i0, 10)
         guard_true(i2) [p1, p2]
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         setfield_gc(p3, p3, descr=nextdescr)
         p4 = getfield_gc_r(p3, descr=nextdescr)
         i1 = int_add(i0, 1)
@@ -7676,8 +7674,8 @@
     def test_chained_virtuals(self):
         ops = """
         [p0, p1]
-        p2 = new_with_vtable(ConstClass(node_vtable))
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p2 = new_with_vtable(descr=nodesize)
+        p3 = new_with_vtable(descr=nodesize)
         setfield_gc(p2, p3, descr=nextdescr)
         jump(p2, p3)
         """
@@ -8000,7 +7998,7 @@
         i11 = getfield_gc_pure_i(p8, descr=valuedescr)
         i13 = int_add_ovf(i11, 1)
         guard_no_overflow() []
-        p22 = new_with_vtable(ConstClass(node_vtable))
+        p22 = new_with_vtable(descr=nodesize)
         setfield_gc(p22, i13, descr=valuedescr)
         setfield_gc(ConstPtr(myptr), p22, descr=adescr)
         jump(p22, p22)
@@ -8010,7 +8008,7 @@
         call_n(i9, descr=nonwritedescr)
         i13 = int_add_ovf(i9, 1)
         guard_no_overflow() []
-        p22 = new_with_vtable(ConstClass(node_vtable))
+        p22 = new_with_vtable(descr=nodesize)
         setfield_gc(p22, i13, descr=valuedescr)
         setfield_gc(ConstPtr(myptr), p22, descr=adescr)
         jump(p22, i13)
@@ -8293,14 +8291,14 @@
         p2 = getfield_gc_r(p1, descr=valuedescr)
         guard_nonnull_class(p2, ConstClass(node_vtable)) []
         call_n(p2, descr=nonwritedescr)
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, p3, descr=valuedescr)
         jump(p1)
         """
         expected = """
         [p1, p2]
         call_n(p2, descr=nonwritedescr)
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, p3, descr=valuedescr)
         jump(p1, p3)
         """
@@ -8312,14 +8310,14 @@
         p2 = getarrayitem_gc_r(p1, 3, descr=arraydescr)
         guard_nonnull_class(p2, ConstClass(node_vtable)) []
         call_n(p2, descr=nonwritedescr)
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         setarrayitem_gc(p1, 3, p3, descr=arraydescr)
         jump(p1)
         """
         expected = """
         [p1, p2]
         call_n(p2, descr=nonwritedescr)
-        p3 = new_with_vtable(ConstClass(node_vtable))
+        p3 = new_with_vtable(descr=nodesize)
         setarrayitem_gc(p1, 3, p3, descr=arraydescr)
         jump(p1, p3)
         """
@@ -8455,9 +8453,9 @@
         p71 = getfield_gc_pure_r(p69, descr=quasifielddescr) # inst_code
         guard_value(p71, -4247) []
 
-        p106 = new_with_vtable(ConstClass(node_vtable))
+        p106 = new_with_vtable(descr=nodesize)
         p108 = new_array(3, descr=arraydescr)
-        p110 = new_with_vtable(ConstClass(node_vtable))
+        p110 = new_with_vtable(descr=nodesize)
         setfield_gc(p110, ConstPtr(myptr2), descr=otherdescr) # inst_w_function
         setarrayitem_gc(p108, 0, p110, descr=arraydescr)
         setfield_gc(p106, p108, descr=nextdescr) # inst_storage
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -82,6 +82,7 @@
         return self._forwarded
 
     def set_forwarded(self, forwarded_to):
+        assert forwarded_to is not self
         self._forwarded = forwarded_to
 
     # methods implemented by the arity mixins

From noreply at buildbot.pypy.org  Tue Jul  7 12:09:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 12:09:04 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix for guard_value on floats
Message-ID: <20150707100904.8CBF91C101F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78485:087a1c3233fb
Date: 2015-07-07 12:09 +0200
http://bitbucket.org/pypy/pypy/changeset/087a1c3233fb/

Log:	fix for guard_value on floats

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -313,6 +313,10 @@
                 if old_guard_op is not None:
                     op = self.replace_guard_class_with_guard_value(op, info,
                                                               old_guard_op)
+        elif arg0.type == 'f':
+            arg0 = self.get_box_replacement(arg0)
+            if arg0.is_constant():
+                return
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
         self.optimize_guard(op, constbox)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5722,5 +5722,19 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_float_guard_value(self):
+        ops = """
+        [f0]
+        guard_value(f0, 3.5) []
+        guard_value(f0, 3.5) []
+        finish(f0)
+        """
+        expected = """
+        [f0]
+        guard_value(f0, 3.5) []
+        finish(3.5)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Tue Jul  7 14:03:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 14:03:57 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: kill this atrocious function
Message-ID: <20150707120357.6EFAC1C13F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78486:58c572a71daa
Date: 2015-07-07 14:03 +0200
http://bitbucket.org/pypy/pypy/changeset/58c572a71daa/

Log:	kill this atrocious function

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -455,53 +455,6 @@
         loop.operations = [emit_end_label] + ops
         return Info(preamble, loop_info.short_preamble)
 
-    def foo(self):
-        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        self.add_guard_future_condition(loop)
-        operations =  loop.operations
-        jumpop = operations[-1]
-        assert jumpop.getopnum() == rop.JUMP
-        inputargs = loop.inputargs
-
-        jump_args = jumpop.getarglist()[:]
-        operations = operations[:-1]
-
-        preamble = TreeLoop('preamble')
-        preamble.inputargs = inputargs
-
-        token = JitCellToken()
-        preamble.operations = [ResOperation(rop.LABEL, inputargs, descr=TargetToken(token))] + \
-                              operations +  \
-                              [ResOperation(rop.LABEL, jump_args, descr=token)]
-        start_state = self._do_optimize_loop(preamble, call_pure_results,
-                                             export_state=True)
-
-        assert preamble.operations[-1].getopnum() == rop.LABEL
-
-        loop.operations = [preamble.operations[-1]] + \
-                          operations + \
-                          [ResOperation(rop.JUMP, jump_args[:],
-                                        descr=token)]
-        
-        assert loop.operations[-1].getopnum() == rop.JUMP
-        assert loop.operations[0].getopnum() == rop.LABEL
-        loop.inputargs = loop.operations[0].getarglist()
-
-        self._do_optimize_loop(loop, call_pure_results, start_state,
-                               export_state=False)
-        extra_same_as = []
-        while loop.operations[0].getopnum() != rop.LABEL:
-            extra_same_as.append(loop.operations[0])
-            del loop.operations[0]
-
-        # Hack to prevent random order of same_as ops
-        extra_same_as.sort(key=lambda op: str(preamble.operations).find(str(op.getarg(0))))
-
-        for op in extra_same_as:
-            preamble.operations.insert(-1, op)
-
-        return preamble
-
 
 class FakeDescr(compile.ResumeGuardDescr):
     def clone_if_mutable(self):

From noreply at buildbot.pypy.org  Tue Jul  7 14:03:58 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  7 Jul 2015 14:03:58 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start working on virtualstate
Message-ID: <20150707120358.8E68C1C13F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78487:cb124aaa4888
Date: 2015-07-07 14:03 +0200
http://bitbucket.org/pypy/pypy/changeset/cb124aaa4888/

Log:	start working on virtualstate

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -290,8 +290,9 @@
             return self.getptrinfo(op)
         elif op.type == 'i':
             return self.getintbound(op)
-        else:
-            zzz
+        elif op.type == 'f':
+            if self.get_box_replacement(op).is_constant():
+                return info.FloatConstInfo(self.get_box_replacement(op))
 
     def get_box_replacement(self, op):
         if op is None:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -2,7 +2,7 @@
 import py
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
      VStructStateInfo, LEVEL_CONSTANT,\
-     VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes,\
+     VArrayStateInfo, NotVirtualStateInfo, VirtualState,\
      GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
 from rpython.jit.metainterp.history import ConstInt, ConstPtr
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
@@ -10,12 +10,13 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \
                                                            equaloplists
-from rpython.jit.metainterp.optimizeopt.intutils import IntBound
+from rpython.jit.metainterp.optimizeopt.intutils import IntBound, ConstIntBound
 from rpython.jit.metainterp.history import TreeLoop, JitCellToken
 from rpython.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeMetaInterpStaticData
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.jit.metainterp import resume
+from rpython.jit.metainterp.optimizeopt import info
 
 class FakeOptimizer(Optimizer):
     def __init__(self):
@@ -81,7 +82,7 @@
     def test_make_inputargs(self):
         optimizer = FakeOptimizer()
         args = [InputArgInt()]
-        info0 = NotVirtualStateInfo(optimizer, args[0])
+        info0 = NotVirtualStateInfo(args[0], None)
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == args
         info0.level = LEVEL_CONSTANT
@@ -107,9 +108,9 @@
             self.check_invalid(info1, info2, state=state)
             assert info1 in state.bad and info2 in state.bad
 
-        for BoxType in (InputArgInt, InputArgFloat, InputArgPtr):
-            info1 = NotVirtualStateInfo(OptValue(BoxType()))
-            info2 = NotVirtualStateInfo(OptValue(BoxType()))
+        for BoxType in (InputArgInt, InputArgFloat, InputArgRef):
+            info1 = NotVirtualStateInfo(BoxType(), None)
+            info2 = NotVirtualStateInfo(BoxType(), None)
             postest(info1, info2)
 
         info1, info2 = VArrayStateInfo(42), VArrayStateInfo(42)
@@ -125,31 +126,27 @@
         postest(info1, info2)
 
     def test_NotVirtualStateInfo_generalization(self):
-        def isgeneral(value1, value2):
-            info1 = NotVirtualStateInfo(value1)
+        def isgeneral(tp1, info1, tp2, info2):
+            info1 = NotVirtualStateInfo(tp1, info1)
             info1.position = 0
-            info2 = NotVirtualStateInfo(value2)
+            info2 = NotVirtualStateInfo(tp2, info2)
             info2.position = 0
             return VirtualState([info1]).generalization_of(VirtualState([info2]), cpu=self.cpu)
 
-        assert isgeneral(OptValue(BoxInt()), OptValue(ConstInt(7)))
-        assert not isgeneral(OptValue(ConstInt(7)), OptValue(BoxInt()))
+        assert isgeneral('i', None, 'i', ConstIntBound(7))
+        assert not isgeneral('i', ConstIntBound(7), 'i', None)
 
-        ptr = PtrOptValue(BoxPtr())
-        nonnull = PtrOptValue(BoxPtr())
-        nonnull.make_nonnull(None)
-        knownclass = PtrOptValue(BoxPtr())
-        clsbox = self.cpu.ts.cls_of_box(BoxPtr(self.myptr))
-        knownclass.make_constant_class(None, clsbox)
-        const = PtrOptValue(BoxPtr)
-        const.make_constant_class(None, clsbox)
-        const.make_constant(ConstPtr(self.myptr))
+        ptr = info.PtrInfo()
+        nonnull = info.NonNullPtrInfo()
+        clsbox = self.cpu.ts.cls_of_box(InputArgRef(self.myptr))
+        knownclass = info.InstancePtrInfo(known_class=clsbox)
+        const = info.ConstPtrInfo(ConstPtr(self.myptr))
         inorder = [ptr, nonnull, knownclass, const]
         for i in range(len(inorder)):
             for j in range(i, len(inorder)):
-                assert isgeneral(inorder[i], inorder[j])
+                assert isgeneral('r', inorder[i], 'r', inorder[j])
                 if i != j:
-                    assert not isgeneral(inorder[j], inorder[i])
+                    assert not isgeneral('r', inorder[j], 'r', inorder[i])
 
         value1 = IntOptValue(BoxInt())
         value2 = IntOptValue(BoxInt())
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -10,7 +10,8 @@
 
 LEVEL_UNKNOWN = '\x00'
 LEVEL_NONNULL = '\x01'
-LEVEL_CONSTANT = '\x02'
+LEVEL_KNOWNCLASS = '\x02'
+LEVEL_CONSTANT = '\x03'
 
 class BadVirtualState(Exception):
     pass
@@ -282,11 +283,10 @@
     lenbound = None
     intbound = None
     
-    def __init__(self, optimizer, box):
-        info = optimizer.getinfo(box)
+    def __init__(self, type, info):
         if info and info.is_constant():
             self.level = LEVEL_CONSTANT
-        elif box.type == 'r' and info and info.is_nonnull():
+        elif type == 'r' and info and info.is_nonnull():
             self.level = LEVEL_NONNULL
         else:
             self.level = LEVEL_UNKNOWN
@@ -602,7 +602,7 @@
 
     def visit_not_virtual(self, box):
         is_opaque = box in self.optimizer.opaque_pointers
-        return NotVirtualStateInfo(self.optimizer, box)
+        return NotVirtualStateInfo(box, self.optimizer.getinfo(box))
 
     def visit_virtual(self, known_class, fielddescrs):
         return VirtualStateInfo(known_class, fielddescrs)

From noreply at buildbot.pypy.org  Tue Jul  7 14:45:00 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  7 Jul 2015 14:45:00 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: import stmgc/f29c44d46d58
Message-ID: <20150707124500.04C071C1048@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gcc
Changeset: r78488:7a66ba31f3ee
Date: 2015-07-07 13:46 +0100
http://bitbucket.org/pypy/pypy/changeset/7a66ba31f3ee/

Log:	import stmgc/f29c44d46d58

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-8b922e88252b
+f29c44d46d58
diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c
--- a/rpython/translator/stm/src_stm/stm/forksupport.c
+++ b/rpython/translator/stm/src_stm/stm/forksupport.c
@@ -20,7 +20,7 @@
     s_mutex_lock();
 
     dprintf(("forksupport_prepare\n"));
-    fprintf(stderr, "[forking: for now, this operation can take some time]\n");
+    //fprintf(stderr, "[forking: for now, this operation can take some time]\n");
 
     stm_thread_local_t *this_tl = NULL;
     stm_thread_local_t *tl = stm_all_thread_locals;

From noreply at buildbot.pypy.org  Tue Jul  7 19:17:58 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  7 Jul 2015 19:17:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: Begin implementing axes parameter in
	np.transpose()
Message-ID: <20150707171758.7A3B41C1048@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78489:7fb7e3c2065c
Date: 2015-07-07 17:45 +0100
http://bitbucket.org/pypy/pypy/changeset/7fb7e3c2065c/

Log:	Begin implementing axes parameter in np.transpose()

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -268,13 +268,15 @@
             view = chunks.apply(space, orig_arr)
             view.implementation.setslice(space, w_value)
 
-    def transpose(self, orig_array):
+    def transpose(self, orig_array, axes=None):
         if len(self.get_shape()) < 2:
             return self
         strides = []
         backstrides = []
         shape = []
-        for i in range(len(self.get_shape()) - 1, -1, -1):
+        if axes is None:
+            axes = range(len(self.get_shape()) - 1, -1, -1)
+        for i in axes:
             strides.append(self.get_strides()[i])
             backstrides.append(self.get_backstrides()[i])
             shape.append(self.get_shape()[i])
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -393,15 +393,24 @@
             w_shape = space.newtuple(args_w)
         return self.reshape(space, w_shape)
 
-    def descr_get_transpose(self, space):
-        return W_NDimArray(self.implementation.transpose(self))
+    def descr_get_transpose(self, space, axes=None):
+        return W_NDimArray(self.implementation.transpose(self, axes))
 
     def descr_transpose(self, space, args_w):
-        if not (len(args_w) == 0 or
+        if len(args_w) == 1 and space.isinstance_w(args_w[0], space.w_tuple):
+            args_w = space.fixedview(args_w[0])
+        if (len(args_w) == 0 or
                 len(args_w) == 1 and space.is_none(args_w[0])):
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                "axes unsupported for transpose"))
-        return self.descr_get_transpose(space)
+            return self.descr_get_transpose(space)
+        else:
+            axes = []
+            for w_arg in args_w:
+                try:
+                    axes.append(support.index_w(space, w_arg))
+                except OperationError:
+                    raise oefmt(space.w_TypeError, "an integer is required")
+            return self.descr_get_transpose(space, axes)
+
 
     @unwrap_spec(axis1=int, axis2=int)
     def descr_swapaxes(self, space, axis1, axis2):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2765,6 +2765,11 @@
         b = a.T
         assert b.shape == (3, 2, 4)
         assert(b[0, :, 0] == [0, 3]).all()
+
+        c = a.transpose((1, 0, 2))
+        assert c.shape == (2, 4, 3)
+        assert (c.transpose(1, 0, 2) == a).all()
+
         b[:, 0, 0] = 1000
         assert(a[0, 0, :] == [1000, 1000, 1000]).all()
         a = array(range(5))
@@ -2775,9 +2780,6 @@
         assert(b[:, 0] == a[0, :]).all()
         assert (a.transpose() == b).all()
         assert (a.transpose(None) == b).all()
-        import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, a.transpose, (1, 0, 2))
 
     def test_flatiter(self):
         from numpy import array, flatiter, arange, zeros

From noreply at buildbot.pypy.org  Tue Jul  7 19:17:59 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  7 Jul 2015 19:17:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Handle errors in np.transpose()
Message-ID: <20150707171759.B2D621C1206@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78490:0e674dc322e0
Date: 2015-07-07 18:10 +0100
http://bitbucket.org/pypy/pypy/changeset/0e674dc322e0/

Log:	Handle errors in np.transpose()

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -403,12 +403,21 @@
                 len(args_w) == 1 and space.is_none(args_w[0])):
             return self.descr_get_transpose(space)
         else:
+            if len(args_w) != self.ndims():
+                raise oefmt(space.w_ValueError, "axes don't match array")
             axes = []
+            axes_seen = [False] * self.ndims()
             for w_arg in args_w:
                 try:
-                    axes.append(support.index_w(space, w_arg))
+                    axis = support.index_w(space, w_arg)
                 except OperationError:
                     raise oefmt(space.w_TypeError, "an integer is required")
+                if axis < 0 or axis >= self.ndims():
+                    raise oefmt(space.w_ValueError, "invalid axis for this array")
+                if axes_seen[axis] is True:
+                    raise oefmt(space.w_ValueError, "repeated axis in transpose")
+                axes.append(axis)
+                axes_seen[axis] = True
             return self.descr_get_transpose(space, axes)
 
 
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2781,6 +2781,14 @@
         assert (a.transpose() == b).all()
         assert (a.transpose(None) == b).all()
 
+    def test_transpose_error(self):
+        import numpy as np
+        a = np.arange(24).reshape(2, 3, 4)
+        raises(ValueError, a.transpose, 2, 1)
+        raises(ValueError, a.transpose, 1, 0, 3)
+        raises(ValueError, a.transpose, 1, 0, 1)
+        raises(TypeError, a.transpose, 1, 0, '2')
+
     def test_flatiter(self):
         from numpy import array, flatiter, arange, zeros
         a = array([[10, 30], [40, 60]])

From noreply at buildbot.pypy.org  Wed Jul  8 09:34:01 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed,  8 Jul 2015 09:34:01 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: unroll does not ignore guard early exit
 anymore (generates wrong code)
Message-ID: <20150708073401.763561C1305@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78491:ca4edaf2af71
Date: 2015-07-06 15:32 +0200
http://bitbucket.org/pypy/pypy/changeset/ca4edaf2af71/

Log:	unroll does not ignore guard early exit anymore (generates wrong
	code)

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1507,7 +1507,6 @@
     # ________________________________________
 
     def not_implemented_op(self, op):
-        import pdb; pdb.set_trace()
         not_implemented("not implemented operation: %s" % op.getopname())
 
     def not_implemented_op_with_guard(self, op, guard_op):
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -10,6 +10,8 @@
     xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
     X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, AddressLoc)
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem.lloperation import llop
 
 # duplicated for easy migration, def in assembler.py as well
 # DUP START
@@ -18,6 +20,12 @@
 
 def heap(addr):
     return AddressLoc(ImmedLoc(addr), imm0, 0, 0)
+
+def not_implemented(msg):
+    msg = '[x86/vector_ext] %s\n' % msg
+    if we_are_translated():
+        llop.debug_print(lltype.Void, msg)
+    raise NotImplementedError(msg)
 # DUP END
 
 class VectorAssemblerMixin(object):
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -735,6 +735,7 @@
             arg = self.parent_trace_label_args[i]
             if isinstance(arg, BoxVectorAccum):
                 self.bridge_label_args[i] = arg
+                label.setarg(i, arg)
         self.inputargs = self.bridge_label_args
 
         return token
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -263,6 +263,7 @@
         self.output_type = pack.output_type
         #
         self.check_if_pack_supported(pack)
+
         #
         if self.must_be_full_but_is_not(pack):
             for op in pack.operations:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1359,20 +1359,20 @@
 
 
     def test_abc(self):
-        py.test.skip()
         trace="""
-        # int32 sum
-        label(p0, p19, i18, i24, i14, i8, i25, descr=TargetToken(140320937897104))
-        guard_early_exit(descr=<Guard0x7f9f03ab6310>) [p0, p19, i18, i14, i24]
-        i27 = raw_load(i8, i24, descr=<ArrayS 4>)
-        guard_not_invalidated(descr=<Guard0x7f9f03af5050>) [p0, i27, p19, i18, i14, i24]
-        i28 = int_add(i14, i27)
-        i29 = int_signext(i28, 4)
-        i30 = int_add(i18, 1)
-        i31 = int_add(i24, 4)
-        i32 = int_ge(i30, i25)
-        guard_false(i32, descr=<Guard0x7f9f03ab17d0>) [p0, i29, i30, i31, p19, None, None, None]
-        jump(p0, p19, i30, i31, i29, i8, i25, descr=TargetToken(140320937897104))
+        [i16, i17, i18, i5, p6, p7, f19, p9, p10, p11, p12, p13, p14, p15, i20, i21]
+        guard_early_exit() []
+        f22 = raw_load(i20, i18, descr=<ArrayF 8>)
+        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7fc428762410>) [i5, i18, i17, i16, p15, p14, p13, p12, p11, p10, p9, p7, p6, f22, f19]
+        f23 = raw_load(i21, i17, descr=<ArrayF 8>)
+        f24 = float_mul(f22, f23)
+        f25 = float_add(f19, f24)
+        i27 = int_add(i18, 8)
+        i29 = int_add(i17, 8)
+        i30 = int_lt(i16, i5)
+        guard_true(i30, descr=<rpython.jit.metainterp.compile.ResumeGuardTrueDescr object at 0x7fc4287689d0>) [i5, i27, i29, i16, p15, p14, p13, p12, p11, p10, p9, p7, p6, f25, None]
+        i33 = int_add(i16, 1)
+        jump(i33, i29, i27, i5, p6, p7, f25, p9, p10, p11, p12, p13, p14, p15, i20, i21)
         """
         # schedule 885 -> ptype is non for raw_load?
         opt = self.vectorize(self.parse_loop(trace))
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -36,8 +36,6 @@
             imp.import_value(value)
 
     def emit_operation(self, op):
-        if op.getopnum() == rop.GUARD_EARLY_EXIT:
-            return
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
         if self.emitting_dissabled:
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -598,14 +598,19 @@
         assert isinstance(tgt_op, GuardResOp)
         assert isinstance(op, GuardResOp)
         olddescr = op.getdescr()
-        descr = CompileLoopVersionDescr()
+        descr = None
+        guard_true_false = tgt_op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE)
+        if guard_true_false:
+            descr = CompileLoopVersionDescr()
+        else:
+            descr = ResumeAtLoopHeaderDescr()
         if olddescr:
             descr.copy_all_attributes_from(olddescr)
         #
         tgt_op.setdescr(descr)
         tgt_op.rd_snapshot = op.rd_snapshot
         tgt_op.setfailargs(op.getfailargs())
-        if tgt_op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
+        if guard_true_false:
             self.orig_loop_version.adddescr(tgt_op, descr)
             tgt_op.setfailargs(label_node.getoperation().getarglist()[:])
             tgt_op.rd_snapshot = None
@@ -853,7 +858,6 @@
         for pack in self.packs:
             if not pack.is_accumulating():
                 continue
-            import pdb; pdb.set_trace()
             accum = pack.accum
             # create a new vector box for the parameters
             box = pack.input_type.new_vector_box()
@@ -878,5 +882,6 @@
                               [box, accum.var, ConstInt(0), ConstInt(1)], result)
             sched_data.invariant_oplist.append(op)
             # rename the variable with the box
+            sched_data.setvector_of_box(accum.var, 0, result) # prevent it from expansion
             renamer.start_renaming(accum.var, result)
 

From noreply at buildbot.pypy.org  Wed Jul  8 09:34:02 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed,  8 Jul 2015 09:34:02 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: ironed out the problems with the bridge
	creation
Message-ID: <20150708073402.D7E751C1311@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78492:a026d96015e4
Date: 2015-07-08 09:34 +0200
http://bitbucket.org/pypy/pypy/changeset/a026d96015e4/

Log:	ironed out the problems with the bridge creation the fail arguments
	now save the regloc of the scalar variable, the actual position is
	saved on the descriptor and reconstructed later

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -528,14 +528,12 @@
     def test_prod(self):
         result = self.run("prod")
         assert int(result) == 576
-        self.check_trace_count(1)
-        self.check_vectorized(2, 1)
+        self.check_vectorized(1, 1)
 
     def test_prod_zero(self):
         result = self.run("prod_zero")
         assert int(result) == 0
-        self.check_trace_count(1)
-        self.check_vectorized(2, 1)
+        self.check_vectorized(1, 1)
 
 
     def define_max():
@@ -767,8 +765,7 @@
     def test_setslice(self):
         result = self.run("setslice")
         assert result == 5.5
-        self.check_trace_count(1)
-        self.check_vectorized(2, 1)
+        self.check_vectorized(1, 1)
 
     def define_virtual_slice():
         return """
@@ -806,7 +803,6 @@
     def test_flat_getitem(self):
         result = self.run("flat_getitem")
         assert result == 10.0
-        self.check_trace_count(1)
         self.check_vectorized(0,0)
 
     def define_flat_setitem():
@@ -820,7 +816,6 @@
     def test_flat_setitem(self):
         result = self.run("flat_setitem")
         assert result == 1.0
-        self.check_trace_count(1)
         self.check_vectorized(1,0) # TODO this can be improved
 
     def define_dot():
@@ -847,8 +842,7 @@
     def test_argsort(self):
         result = self.run("argsort")
         assert result == 6
-        self.check_trace_count(1)
-        self.check_vectorized(2,1) # vec. setslice
+        self.check_vectorized(1,1) # vec. setslice
 
     def define_where():
         return """
@@ -862,7 +856,6 @@
     def test_where(self):
         result = self.run("where")
         assert result == -40
-        self.check_trace_count(1)
         self.check_vectorized(1, 0) # TODO might be possible to vectorize
 
     def define_searchsorted():
@@ -877,7 +870,6 @@
         result = self.run("searchsorted")
         assert result == 0
         self.check_trace_count(6)
-        # TODO?
 
     def define_int_mul_array():
         return """
@@ -908,8 +900,7 @@
     def test_slice(self):
         result = self.run("slice")
         assert result == 18
-        self.check_trace_count(1)
-        self.check_vectorized(2,1)
+        self.check_vectorized(1,1)
 
     def define_multidim_slice():
         return """
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -8,7 +8,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.metainterp.history import (Const, Box, VOID,
-    BoxVector, ConstInt, BoxVectorAccum)
+    BoxVector, ConstInt)
 from rpython.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT
 from rpython.jit.metainterp.compile import CompileLoopVersionDescr
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -26,6 +26,7 @@
     ConstFloat, BoxInt, BoxFloat, BoxVector, BoxVectorAccum, INT, REF,
     FLOAT, VECTOR, TargetToken)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.rlib import rgc
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint
@@ -304,7 +305,32 @@
         self.assembler.regalloc_perform_math(op, arglocs, result_loc)
 
     def locs_for_fail(self, guard_op):
-        return [self.loc(v) for v in guard_op.getfailargs()]
+        faillocs = []
+        descr = guard_op.getdescr()
+        for v in guard_op.getfailargs():
+            if v is not None and isinstance(v, BoxVectorAccum):
+                loc = self.loc(v.scalar_var)
+                self.update_accumulation_loc(v, descr)
+                faillocs.append(loc)
+            else:
+                faillocs.append(self.loc(v))
+
+        return faillocs
+
+    def update_accumulation_loc(self, accumbox, descr):
+        """ Saves the location to the AccumInfo object.
+        Necessary to reconstruct the values at a guard exit.
+        """
+        box = accumbox.scalar_var
+        assert isinstance(descr, ResumeGuardDescr)
+        accum_info = descr.rd_accum_list
+        while accum_info:
+            if accum_info.box is box:
+                accum_info.loc = self.loc(accumbox)
+                break
+            accum_info = accum_info.prev
+        else:
+            raise AssertionError("accum box has no accum_info entry")
 
     def perform_with_guard(self, op, guard_op, arglocs, result_loc):
         faillocs = self.locs_for_fail(guard_op)
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -12,6 +12,7 @@
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem import lltype
 
 # duplicated for easy migration, def in assembler.py as well
 # DUP START
@@ -65,23 +66,21 @@
         accum_info = faildescr.rd_accum_list
         while accum_info:
             pos = accum_info.position
-            loc = fail_locs[pos]
+            loc = accum_info.loc
+            tgtloc = fail_locs[pos]
+            # the upper elements will be lost if saved to the stack!
             assert isinstance(loc, RegLoc)
-            arg = fail_args[pos]
-            if isinstance(arg, BoxVectorAccum):
-                arg = arg.scalar_var
+            if not isinstance(tgtloc, RegLoc):
+                tgtloc = regalloc.force_allocate_reg(accum_info.box)
+            arg = accum_info.box
             assert arg is not None
-            tgtloc = regalloc.force_allocate_reg(arg, fail_args)
             if accum_info.operation == '+':
-                # reduction using plus
                 self._accum_reduce_sum(arg, loc, tgtloc)
             elif accum_info.operation == '*':
                 self._accum_reduce_mul(arg, loc, tgtloc)
             else:
                 not_implemented("accum operator %s not implemented" %
                                             (accum_info.operation)) 
-            fail_locs[pos] = tgtloc
-            regalloc.possibly_free_var(arg)
             accum_info = accum_info.prev
 
     def _accum_reduce_mul(self, arg, accumloc, targetloc):
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -711,9 +711,9 @@
             i += 1
         assert label.getopnum() == rop.LABEL
         self.label_pos = i
-        self.parent_trace_label_args = None
-        self.bridge_label_args = label.getarglist()
-        self.inputargs = None
+        #self.parent_trace_label_args = None
+        #self.bridge_label_args = label.getarglist()
+        self.inputargs = label.getarglist()
 
     def adddescr(self, op, descr):
         self.faildescrs.append((op, descr))
@@ -730,14 +730,6 @@
         label.setdescr(token)
         jump.setdescr(token)
 
-        assert len(self.bridge_label_args) <= len(self.parent_trace_label_args)
-        for i in range(len(self.bridge_label_args)):
-            arg = self.parent_trace_label_args[i]
-            if isinstance(arg, BoxVectorAccum):
-                self.bridge_label_args[i] = arg
-                label.setarg(i, arg)
-        self.inputargs = self.bridge_label_args
-
         return token
 
 class TreeLoop(object):
@@ -809,30 +801,38 @@
     def seen_args(inputargs):
         seen = {}
         for arg in inputargs:
+            if arg is None:
+                continue
             if isinstance(arg, BoxVectorAccum):
                 seen[arg.scalar_var] = None
-                seen[arg] = None
             else:
                 seen[arg] = None
         return seen
 
     @staticmethod
+    def check_if_box_was_seen(box, seen):
+        if box is not None:
+            assert isinstance(box, Box)
+            if isinstance(box, BoxVectorAccum):
+                assert box in seen or box.scalar_var in seen
+            else:
+                assert box in seen
+
+    @staticmethod
     def check_consistency_of_branch(operations, seen):
         "NOT_RPYTHON"
         for op in operations:
             for i in range(op.numargs()):
                 box = op.getarg(i)
                 if isinstance(box, Box):
-                    assert box in seen
+                    TreeLoop.check_if_box_was_seen(box, seen)
             if op.is_guard():
                 assert op.getdescr() is not None
                 if hasattr(op.getdescr(), '_debug_suboperations'):
                     ops = op.getdescr()._debug_suboperations
                     TreeLoop.check_consistency_of_branch(ops, seen.copy())
                 for box in op.getfailargs() or []:
-                    if box is not None:
-                        assert isinstance(box, Box)
-                        assert box in seen
+                    TreeLoop.check_if_box_was_seen(box, seen)
             else:
                 assert op.getfailargs() is None
             box = op.result
@@ -844,8 +844,12 @@
                 inputargs = op.getarglist()
                 for box in inputargs:
                     assert isinstance(box, Box), "LABEL contains %r" % (box,)
-                seen = dict.fromkeys(inputargs)
-                assert len(seen) == len(inputargs), (
+                seen = TreeLoop.seen_args(inputargs)
+                seen_count = len(seen)
+                for arg in seen:
+                    if isinstance(arg, BoxVectorAccum):
+                        seen_count -= 1
+                assert seen_count == len(inputargs), (
                     "duplicate Box in the LABEL arguments")
 
         assert operations[-1].is_final()
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -861,10 +861,12 @@
     PLUS = '+'
     MULTIPLY = '*'
 
-    def __init__(self, var, pos, operator):
+    def __init__(self, opnum, var, pos):
         self.var = var
         self.pos = pos
-        self.operator = operator
+        self.operator = Accum.PLUS
+        if opnum == rop.FLOAT_MUL:
+            self.operator = Accum.MULTIPLY
 
 class Pack(object):
     """ A pack is a set of n statements that are:
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -222,11 +222,6 @@
                 args = guard.getfailargs()
             for i,arg in enumerate(args):
                 value = self.rename_map.get(arg,arg)
-                if value is not arg and isinstance(value, BoxVectorAccum):
-                    descr = guard.getdescr()
-                    assert isinstance(descr,ResumeGuardDescr)
-                    ai = AccumInfo(descr.rd_accum_list, i, value.operator)
-                    descr.rd_accum_list = ai
                 args[i] = value
             return args
         return None
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -8,11 +8,11 @@
 import py
 import time
 
-from rpython.jit.metainterp.resume import Snapshot
+from rpython.jit.metainterp.resume import Snapshot, AccumInfo
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
 from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
 from rpython.jit.metainterp.compile import (ResumeAtLoopHeaderDescr,
-        CompileLoopVersionDescr, invent_fail_descr_for_op)
+        CompileLoopVersionDescr, invent_fail_descr_for_op, ResumeGuardDescr)
 from rpython.jit.metainterp.history import (ConstInt, VECTOR, FLOAT, INT,
         BoxVector, BoxFloat, BoxInt, ConstFloat, TargetToken, JitCellToken, Box,
         BoxVectorAccum, LoopVersion)
@@ -31,21 +31,6 @@
 from rpython.rlib.jit import Counters
 from rpython.rtyper.lltypesystem import lltype, rffi
 
-def debug_print_operations(loop):
-    """ NOT_RPYTHON """
-    if not we_are_translated():
-        print('--- loop instr numbered ---')
-        def ps(snap):
-            if snap.prev is None:
-                return []
-            return ps(snap.prev) + snap.boxes[:]
-        for i,op in enumerate(loop.operations):
-            print "[",str(i).center(2," "),"]",op,
-            if op.is_guard():
-                print op.getfailargs()
-            else:
-                print ""
-
 def optimize_vector(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, cost_threshold):
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
@@ -72,7 +57,7 @@
 
         aligned_vector_version = LoopVersion(loop, aligned=True)
 
-        loop.versions = [orig_version] #, aligned_vector_version]
+        loop.versions = [orig_version]
 
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "post vectorize")
@@ -198,8 +183,6 @@
 
         self.emit_unrolled_operation(label_op)
 
-        self.orig_loop_version.parent_trace_label_args = label_op.getarglist()[:]
-
         renamer = Renamer()
         oi = 0
         pure = True
@@ -495,6 +478,17 @@
                 assert node.emitted
         if vector and not self.costmodel.profitable():
             return
+        if vector:
+            # add accumulation info to the descriptor
+            for guard_node in self.dependency_graph.guards:
+                op = guard_node.getoperation()
+                failargs = op.getfailargs()
+                for i,arg in enumerate(failargs):
+                    if isinstance(arg, BoxVectorAccum):
+                        descr = op.getdescr()
+                        assert isinstance(descr,ResumeGuardDescr)
+                        ai = AccumInfo(descr.rd_accum_list, i, arg.operator, arg.scalar_var)
+                        descr.rd_accum_list = ai
         self.loop.operations = \
             sched_data.prepend_invariant_operations(self._newoperations)
         self.clear_newoperations()
@@ -837,10 +831,7 @@
                 # of leading/preceding signext/floatcast instructions needs to be
                 # considered. => tree pattern matching problem.
                 return None
-            operator = Accum.PLUS
-            if opnum == rop.FLOAT_MUL:
-                operator = Accum.MULTIPLY
-            accum = Accum(accum_var, accum_pos, operator)
+            accum = Accum(opnum, accum_var, accum_pos)
             return AccumPair(lnode, rnode, ptype, ptype, accum)
 
         return None
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -35,11 +35,13 @@
         self.pc = pc
 
 class AccumInfo(object):
-    __slots__ = ('prev', 'position', 'operation')
-    def __init__(self, prev, position, operation):
+    __slots__ = ('prev', 'position', 'operation', 'box', 'loc')
+    def __init__(self, prev, position, operation, box):
         self.prev = prev
         self.operation = operation
         self.position = position
+        self.box = box
+        self.loc = None
 
 def _ensure_parent_resumedata(framestack, n):
     target = framestack[n]

From noreply at buildbot.pypy.org  Wed Jul  8 12:33:16 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  8 Jul 2015 12:33:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix a few freebsd issues
Message-ID: <20150708103316.91FAA1C139F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78493:f4196519dd86
Date: 2015-07-08 12:33 +0200
http://bitbucket.org/pypy/pypy/changeset/f4196519dd86/

Log:	fix a few freebsd issues

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -23,11 +23,16 @@
 # running make inside the src dir
 DYNAMIC_VMPROF = False
 
+if sys.platform.startswith('linux'):
+    libs = ['dl']
+else:
+    libs = []
+
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['vmprof.h', 'trampoline.h'],
     separate_module_files = [SRC.join('trampoline.vmprof.s')],
-    libraries = ['dl'],
+    libraries = libs,
     
     post_include_bits=["""
         int pypy_vmprof_init(void);
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
--- a/pypy/module/_vmprof/src/config.h
+++ b/pypy/module/_vmprof/src/config.h
@@ -1,2 +1,6 @@
 #define HAVE_SYS_UCONTEXT_H
+#if defined(__FreeBSD__) || defined(__APPLE__)
+#define PC_FROM_UCONTEXT uc_mcontext.mc_rip
+#else
 #define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
+#endif
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -33,6 +33,9 @@
 //#include <libunwind.h>
 
 #include "vmprof.h"
+#if defined(__FreeBSD__) || defined(__APPLE__)
+#define sighandler_t sig_t
+#endif
 
 #define _unused(x) ((void)x)
 

From noreply at buildbot.pypy.org  Wed Jul  8 12:42:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  8 Jul 2015 12:42:05 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: Mention -fno-ivopts
Message-ID: <20150708104205.B7EDD1C026F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1893:a2ba6745aadd
Date: 2015-07-08 12:42 +0200
http://bitbucket.org/pypy/stmgc/changeset/a2ba6745aadd/

Log:	Mention -fno-ivopts

diff --git a/gcc-seg-gs/README.txt b/gcc-seg-gs/README.txt
--- a/gcc-seg-gs/README.txt
+++ b/gcc-seg-gs/README.txt
@@ -32,3 +32,6 @@
     #!/bin/bash
     BUILD=/..../build      # <- insert full path
     exec $BUILD/gcc/xgcc -B $BUILD/gcc "$@"
+
+GCC has a bug, likely in the "ivopts" optimization, that can be worked
+around by specifying "-fno-ivopts".

From noreply at buildbot.pypy.org  Wed Jul  8 12:44:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  8 Jul 2015 12:44:58 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: Expand the explanation and roll back
	69f1abc8e3fe.
Message-ID: <20150708104458.54F781C08F3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1894:088f807586c2
Date: 2015-07-08 12:45 +0200
http://bitbucket.org/pypy/stmgc/changeset/088f807586c2/

Log:	Expand the explanation and roll back 69f1abc8e3fe.

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -151,10 +151,8 @@
 
     /* The sync queue used to synchronize newly allocated objs to
        other segments */
-    stm_char *_sq_fragments[SYNC_QUEUE_SIZE];
-    int _sq_fragsizes[SYNC_QUEUE_SIZE];
-    stm_char **sq_fragments;   /* indirection to work around a gcc issue */
-    int *sq_fragsizes;
+    stm_char *sq_fragments[SYNC_QUEUE_SIZE];
+    int sq_fragsizes[SYNC_QUEUE_SIZE];
     int sq_len;
 
     /* For nursery_mark */
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -116,9 +116,6 @@
         pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
         highest_overflow_number = pr->overflow_number;
         pr->pub.transaction_read_version = 0xff;
-
-        pr->sq_fragments = pr->_sq_fragments;
-        pr->sq_fragsizes = pr->_sq_fragsizes;
     }
 
     /* The pages are shared lazily, as remap_file_pages() takes a relatively
diff --git a/gcc-seg-gs/README.txt b/gcc-seg-gs/README.txt
--- a/gcc-seg-gs/README.txt
+++ b/gcc-seg-gs/README.txt
@@ -31,7 +31,8 @@
 
     #!/bin/bash
     BUILD=/..../build      # <- insert full path
-    exec $BUILD/gcc/xgcc -B $BUILD/gcc "$@"
+    exec $BUILD/gcc/xgcc -B $BUILD/gcc -fno-ivopts "$@"
 
-GCC has a bug, likely in the "ivopts" optimization, that can be worked
-around by specifying "-fno-ivopts".
+So far, GCC has a bug in the presence of multiple address spaces, likely
+in the "ivopts" optimization.  It can be worked around by specifying
+"-fno-ivopts" like above.  https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66768

From noreply at buildbot.pypy.org  Wed Jul  8 12:46:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  8 Jul 2015 12:46:34 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: import stmgc/088f807586c2
Message-ID: <20150708104634.61A1E1C08F3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c8-gcc
Changeset: r78494:3a5bd2b925aa
Date: 2015-07-08 11:48 +0100
http://bitbucket.org/pypy/pypy/changeset/3a5bd2b925aa/

Log:	import stmgc/088f807586c2

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-f29c44d46d58
+088f807586c2
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -151,10 +151,8 @@
 
     /* The sync queue used to synchronize newly allocated objs to
        other segments */
-    stm_char *_sq_fragments[SYNC_QUEUE_SIZE];
-    int _sq_fragsizes[SYNC_QUEUE_SIZE];
-    stm_char **sq_fragments;   /* indirection to work around a gcc issue */
-    int *sq_fragsizes;
+    stm_char *sq_fragments[SYNC_QUEUE_SIZE];
+    int sq_fragsizes[SYNC_QUEUE_SIZE];
     int sq_len;
 
     /* For nursery_mark */
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -116,9 +116,6 @@
         pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
         highest_overflow_number = pr->overflow_number;
         pr->pub.transaction_read_version = 0xff;
-
-        pr->sq_fragments = pr->_sq_fragments;
-        pr->sq_fragsizes = pr->_sq_fragsizes;
     }
 
     /* The pages are shared lazily, as remap_file_pages() takes a relatively

From noreply at buildbot.pypy.org  Wed Jul  8 13:29:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  8 Jul 2015 13:29:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Elidable-ize the convertion from ascii
	string to unicode
Message-ID: <20150708112943.E8A6F1C026F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78495:8dd96fd0cca6
Date: 2015-07-08 13:29 +0200
http://bitbucket.org/pypy/pypy/changeset/8dd96fd0cca6/

Log:	Elidable-ize the convertion from ascii string to unicode

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -6,7 +6,7 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
-    unicode_encode_ascii, unicode_encode_utf_8)
+    unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
 
 from pypy.interpreter import unicodehelper
 from pypy.interpreter.baseobjspace import W_Root
@@ -481,9 +481,13 @@
         if encoding == 'ascii':
             # XXX error handling
             s = space.charbuf_w(w_obj)
-            eh = unicodehelper.decode_error_handler(space)
-            return space.wrap(str_decode_ascii(
-                    s, len(s), None, final=True, errorhandler=eh)[0])
+            try:
+                u = fast_str_decode_ascii(s)
+            except ValueError:
+                eh = unicodehelper.decode_error_handler(space)
+                u = str_decode_ascii(     # try again, to get the error right
+                    s, len(s), None, final=True, errorhandler=eh)[0]
+            return space.wrap(u)
         if encoding == 'utf-8':
             s = space.charbuf_w(w_obj)
             eh = unicodehelper.decode_error_handler(space)
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1009,6 +1009,16 @@
             result.append(r)
     return result.build(), pos
 
+# An elidable version, for a subset of the cases
+ at jit.elidable
+def fast_str_decode_ascii(s):
+    result = UnicodeBuilder(len(s))
+    for c in s:
+        if ord(c) >= 128:
+            raise ValueError
+        result.append(unichr(ord(c)))
+    return result.build()
+
 
 # Specialize on the errorhandler when it's a constant
 @specialize.arg_or_var(3)
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -139,6 +139,12 @@
             for encoding in "utf-8 latin-1 ascii".split():
                 self.checkdecode(chr(i), encoding)
 
+    def test_fast_str_decode_ascii(self):
+        u = runicode.fast_str_decode_ascii("abc\x00\x7F")
+        assert type(u) is unicode
+        assert u == u"abc\x00\x7F"
+        py.test.raises(ValueError, runicode.fast_str_decode_ascii, "ab\x80")
+
     def test_all_first_256(self):
         for i in range(256):
             for encoding in ("utf-7 utf-8 latin-1 utf-16 utf-16-be utf-16-le "

From noreply at buildbot.pypy.org  Wed Jul  8 15:03:16 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed,  8 Jul 2015 15:03:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix np.mod to behave like Python '%',
	not like math.fmod
Message-ID: <20150708130316.CBFD81C08F3@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78496:df9cec2f0353
Date: 2015-07-08 14:03 +0100
http://bitbucket.org/pypy/pypy/changeset/df9cec2f0353/

Log:	Fix np.mod to behave like Python '%', not like math.fmod

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -540,6 +540,15 @@
         for v in [float('inf'), float('-inf'), float('nan'), float('-nan')]:
             assert math.isnan(fmod(v, 2))
 
+    def test_mod(self):
+        from numpy import mod
+        assert mod(5, 3) == 2
+        assert mod(5, -3) == -1
+        assert mod(-5, 3) == 1
+        assert mod(-5, -3) == -2
+        assert mod(2.5, 1) == 0.5
+        assert mod(-1.5, 2) == 0.5
+
     def test_minimum(self):
         from numpy import array, minimum, nan, isnan
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -759,7 +759,21 @@
 
     @simple_binary_op
     def mod(self, v1, v2):
-        return math.fmod(v1, v2)
+        # partial copy of pypy.objspace.std.floatobject.W_FloatObject.descr_mod
+        if v2 == 0.0:
+            return rfloat.NAN
+        mod = math.fmod(v1, v2)
+        if mod:
+            # ensure the remainder has the same sign as the denominator
+            if (v2 < 0.0) != (mod < 0.0):
+                mod += v2
+        else:
+            # the remainder is zero, and in the presence of signed zeroes
+            # fmod returns different results across platforms; ensure
+            # it has the same sign as the denominator; we'd like to do
+            # "mod = v2 * 0.0", but that may get optimized away
+            mod = rfloat.copysign(0.0, v2)
+        return mod
 
     @simple_binary_op
     def pow(self, v1, v2):

From noreply at buildbot.pypy.org  Wed Jul  8 15:09:06 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  8 Jul 2015 15:09:06 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: Count the extra segment prefix in the
 estimation of instruction sizes
Message-ID: <20150708130906.575491C08F3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1895:a89f21f5670b
Date: 2015-07-08 15:09 +0200
http://bitbucket.org/pypy/stmgc/changeset/a89f21f5670b/

Log:	Count the extra segment prefix in the estimation of instruction
	sizes

diff --git a/gcc-seg-gs/gcc-5.1.0-patch.diff b/gcc-seg-gs/gcc-5.1.0-patch.diff
--- a/gcc-seg-gs/gcc-5.1.0-patch.diff
+++ b/gcc-seg-gs/gcc-5.1.0-patch.diff
@@ -120,14 +120,37 @@
        x = XEXP (x, 0);
        /* Avoid (%rip) for call operands.  */
        if (CONSTANT_ADDRESS_P (x) && code == 'P'
-@@ -51816,6 +51830,130 @@
+@@ -26015,6 +26029,7 @@
+   for (i = recog_data.n_operands - 1; i >= 0; --i)
+     if (MEM_P (recog_data.operand[i]))
+       {
++	int addr_space;
+         constrain_operands_cached (insn, reload_completed);
+         if (which_alternative != -1)
+ 	  {
+@@ -26030,7 +26045,13 @@
+ 	    if (*constraints == 'X')
+ 	      continue;
+ 	  }
+-	return memory_address_length (XEXP (recog_data.operand[i], 0), false);
++
++	/**** <AR> ****/
++	addr_space = MEM_ADDR_SPACE(recog_data.operand[i]);
++	/* account for one byte segment prefix for SEG_FS/SEG_GS addr spaces */
++	return (addr_space == ADDR_SPACE_GENERIC ? 0 : 1)
++	/**** </AR> ****/
++	  + memory_address_length (XEXP (recog_data.operand[i], 0), false);
+       }
+   return 0;
+ }
+@@ -51816,6 +51837,130 @@
  }
  #endif
  
 +
 +/***** <AR> *****/
 +
-+/*** GS segment register addressing mode ***/
++/*** FS/GS segment register addressing mode ***/
 +
 +static machine_mode
 +ix86_addr_space_pointer_mode (addr_space_t as)
@@ -182,7 +205,7 @@
 +
 +/* The default, SEG_FS and SEG_GS address spaces are all "subsets" of
 +   each other. */
-+bool static 
++bool static
 +ix86_addr_space_subset_p (addr_space_t subset, addr_space_t superset)
 +{
 +  gcc_assert (subset == ADDR_SPACE_GENERIC ||

From noreply at buildbot.pypy.org  Wed Jul  8 16:51:10 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  8 Jul 2015 16:51:10 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Try __bytes__ before __index__ when
 converting an object to bytes.
Message-ID: <20150708145110.E21571C026F@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78497:db201c02e7fc
Date: 2015-07-05 08:40 +0200
http://bitbucket.org/pypy/pypy/changeset/db201c02e7fc/

Log:	Try __bytes__ before __index__ when converting an object to bytes.

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -683,6 +683,14 @@
             raise OperationError(space.w_TypeError, space.wrap(
                     "encoding or errors without string argument"))
         return []
+    # Some object with __bytes__ special method
+    w_bytes_method = space.lookup(w_source, "__bytes__")
+    if w_bytes_method is not None:
+        w_bytes = space.get_and_call_function(w_bytes_method, w_source)
+        if not space.isinstance_w(w_bytes, space.w_bytes):
+            raise oefmt(space.w_TypeError,
+                        "__bytes__ returned non-bytes (type '%T')", w_bytes)
+        return [c for c in space.bytes_w(w_bytes)]
     # Is it an integer?
     # Note that we're calling space.getindex_w() instead of space.int_w().
     try:
@@ -707,7 +715,7 @@
         w_source = encode_object(space, w_source, encoding, errors)
         # and continue with the encoded string
 
-    return makebytesdata_w(space, w_source)
+    return _convert_from_buffer_or_iterable(space, w_source)
 
 def makebytesdata_w(space, w_source):
     w_bytes_method = space.lookup(w_source, "__bytes__")
@@ -717,7 +725,9 @@
             raise oefmt(space.w_TypeError,
                         "__bytes__ returned non-bytes (type '%T')", w_bytes)
         return [c for c in space.bytes_w(w_bytes)]
+    return _convert_from_buffer_or_iterable(space, w_source)
 
+def _convert_from_buffer_or_iterable(space, w_source):
     # String-like argument
     try:
         buf = space.buffer_w(w_source, space.BUF_FULL_RO)
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -757,6 +757,14 @@
                 return 3
         raises(TypeError, bytes, WithInt())
 
+    def test_fromobject___bytes__(self):
+        class WithIndex:
+            def __bytes__(self):
+                return b'a'
+            def __index__(self):
+                return 3
+        assert bytes(WithIndex()) == b'a'
+
     def test_getnewargs(self):
         assert  b"foo".__getnewargs__() == (b"foo",)
 

From noreply at buildbot.pypy.org  Wed Jul  8 16:51:12 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed,  8 Jul 2015 16:51:12 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Fix __qualname__ of built-in class
	methods.
Message-ID: <20150708145112.440311C026F@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78498:1c0541a5be95
Date: 2015-07-08 16:50 +0200
http://bitbucket.org/pypy/pypy/changeset/1c0541a5be95/

Log:	Fix __qualname__ of built-in class methods.

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import W_Root, SpaceCache
 from pypy.interpreter.error import oefmt, OperationError
 from pypy.interpreter.function import (
-    Function, StaticMethod, FunctionWithFixedCode)
+    Function, StaticMethod, ClassMethod, FunctionWithFixedCode)
 from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\
      descr_get_dict, dict_descr, Member, TypeDef
 from pypy.interpreter.astcompiler.misc import mangle
@@ -1313,10 +1313,12 @@
             # Set the __qualname__ of member functions
             for name in rawdict:
                 w_obj = dict_w[name]
+                if isinstance(w_obj, ClassMethod):
+                    w_obj = w_obj.w_function
                 if isinstance(w_obj, FunctionWithFixedCode):
                     qualname = w_type.getqualname(space) + '.' + name
                     w_obj.fset_func_qualname(space, space.wrap(qualname))
-                
+
         if hasattr(typedef, 'flag_sequence_bug_compat'):
             w_type.flag_sequence_bug_compat = typedef.flag_sequence_bug_compat
         w_type.lazyloaders = lazyloaders

From noreply at buildbot.pypy.org  Wed Jul  8 22:36:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  8 Jul 2015 22:36:43 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20150708203643.DDB171C101F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r623:ff2ef583cac4
Date: 2015-07-08 22:37 +0200
http://bitbucket.org/pypy/pypy.org/changeset/ff2ef583cac4/

Log:	update the values

diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting NumPy in PyPy.
   Current status:
-    we have $16091 left
+    we have $11891 left
   in the account.</br> <a href="numpydonate.html">Read proposal</a></span>
   </li>
   <li>

From noreply at buildbot.pypy.org  Thu Jul  9 09:27:57 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  9 Jul 2015 09:27:57 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: resolved typo in pypy_c test
Message-ID: <20150709072757.122881C06B9@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78499:e6096e85113e
Date: 2015-07-08 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/e6096e85113e/

Log:	resolved typo in pypy_c test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -7,7 +7,7 @@
 class TestMicroNumPy(BaseTestPyPyC):
 
     arith_comb = [('+','float','float', 4*3427,   3427, 1.0,3.0),
-             ('+','float','int',   9*7844,   7843, 4.0,5.0),
+             ('+','float','int',   9*7843,   7843, 4.0,5.0),
              ('+','int','float',   8*2571,   2571, 9.0,-1.0),
              ('+','float','int',   -18*2653,   2653, 4.0,-22.0),
              ('+','int','int',     -1*1499,   1499, 24.0,-25.0),
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -258,7 +258,7 @@
         self.sched_data = sched_data
         self.preamble_ops = oplist
         self.costmodel = sched_data.costmodel
-        #self.update_input_output(pack)
+        #
         self.input_type = pack.input_type
         self.output_type = pack.output_type
         #

From noreply at buildbot.pypy.org  Thu Jul  9 09:27:58 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  9 Jul 2015 09:27:58 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: commented the debugging timing calls
Message-ID: <20150709072758.43A701C06B9@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78500:d70afeafa621
Date: 2015-07-09 09:27 +0200
http://bitbucket.org/pypy/pypy/changeset/d70afeafa621/

Log:	commented the debugging timing calls

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -15,7 +15,7 @@
 
 call2_driver = jit.JitDriver(
     name='numpy_call2',
-    greens=['shapelen', 'func', 'calc_dtype', 'res_dtype'],
+    greens=['shapelen', 'func', 'calc_dtype', 'res_dtype', 'left', 'right'],
     reds='auto', vectorize=True)
 
 def call2(space, shape, func, calc_dtype, w_lhs, w_rhs, out):
@@ -40,7 +40,9 @@
     res_dtype = out.get_dtype()
     while not out_iter.done(out_state):
         call2_driver.jit_merge_point(shapelen=shapelen, func=func,
-                                     calc_dtype=calc_dtype, res_dtype=res_dtype)
+                                     calc_dtype=calc_dtype, res_dtype=res_dtype,
+                                     left=left_iter is None,
+                                     right=right_iter is None)
         if left_iter:
             w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
             left_state = left_iter.next(left_state)
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -208,6 +208,7 @@
                                        version.operations, jitcell_token)
                 versioned_loop.original_jitcell_token = jitcell_token
                 record_loop_or_bridge(metainterp_sd, versioned_loop)
+    loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -6,7 +6,7 @@
 """
 
 import py
-import time
+#XXXimport time
 
 from rpython.jit.metainterp.resume import Snapshot, AccumInfo
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
@@ -42,21 +42,18 @@
         # it won't be possible to vectorize. There are too many
         # guards that prevent parallel execution of instructions
         return
-    start = -1
-    end = -1
     try:
         debug_start("vec-opt-loop")
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "pre vectorize")
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZE_TRY)
-        start = time.clock()
+        #
+        #XXXstart = time.clock()
         opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, cost_threshold, orig_version)
         opt.propagate_all_forward()
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
         gso.propagate_all_forward(opt.loop)
-        end = time.clock()
-
-        aligned_vector_version = LoopVersion(loop, aligned=True)
-
+        #XXXend = time.clock()
+        #
         loop.versions = [orig_version]
 
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
@@ -64,15 +61,15 @@
         debug_stop("vec-opt-loop")
         #
         # XXX
-        ns = int((end-start)*10.0**9)
-        debug_start("xxx-clock")
-        debug_print("vecopt unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \
-                      (opt.unroll_count+1,
-                       gso.strength_reduced,
-                       len(orig_ops),
-                       len(loop.operations),
-                       ns))
-        debug_stop("xxx-clock")
+        #XXXns = int((end-start)*10.0**9)
+        #XXXdebug_start("xxx-clock")
+        #XXXdebug_print("vecopt unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \
+        #XXX              (opt.unroll_count+1,
+        #XXX               gso.strength_reduced,
+        #XXX               len(orig_ops),
+        #XXX               len(loop.operations),
+        #XXX               ns))
+        #XXXdebug_stop("xxx-clock")
     except NotAVectorizeableLoop:
         debug_stop("vec-opt-loop")
         # vectorization is not possible
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -32,35 +32,35 @@
 import time
 
 # XXX XXX XXX
-class XXXBench(object):
-    def __init__(self, name, uid, vec):
-        self.t = []
-        if name is None:
-            name = "<unkown>"
-        if uid is None:
-            uid = 0
-        self.name = str(name)
-        self.unique_id = hex(uid)
-        self.vec = vec
-
-    def xxx_clock_start(self):
-        now = time.clock()
-        self.t.append(now)
-        debug_start("xxx-clock")
-        debug_print("start name: %s id: %s clock: %f" % \
-                (self.name, self.unique_id, now) )
-        debug_stop("xxx-clock")
-
-    def xxx_clock_stop(self, fail=False):
-        end = time.clock()
-        assert len(self.t) > 0
-        start = self.t[-1]
-        del self.t[-1]
-        ns = (end - start) * 10**9
-        debug_start("xxx-clock")
-        debug_print("stop name: %s id: %s clock: %f exe time: %dns fail? %d vec? %d" % \
-                    (self.name, self.unique_id, end, int(ns), int(fail), int(self.vec)))
-        debug_stop("xxx-clock")
+#class XXXBench(object):
+#    def __init__(self, name, uid, vec):
+#        self.t = []
+#        if name is None:
+#            name = "<unkown>"
+#        if uid is None:
+#            uid = 0
+#        self.name = str(name)
+#        self.unique_id = hex(uid)
+#        self.vec = vec
+#
+#    def xxx_clock_start(self):
+#        now = time.clock()
+#        self.t.append(now)
+#        debug_start("xxx-clock")
+#        debug_print("start name: %s id: %s clock: %f" % \
+#                (self.name, self.unique_id, now) )
+#        debug_stop("xxx-clock")
+#
+#    def xxx_clock_stop(self, fail=False):
+#        end = time.clock()
+#        assert len(self.t) > 0
+#        start = self.t[-1]
+#        del self.t[-1]
+#        ns = (end - start) * 10**9
+#        debug_start("xxx-clock")
+#        debug_print("stop name: %s id: %s clock: %f exe time: %dns fail? %d vec? %d" % \
+#                    (self.name, self.unique_id, end, int(ns), int(fail), int(self.vec)))
+#        debug_stop("xxx-clock")
 
 
 
@@ -439,7 +439,7 @@
         jd.result_type = history.getkind(jd.portal_graph.getreturnvar()
                                          .concretetype)[0]
         # XXX XXX XXX
-        jd.xxxbench = XXXBench(jd.jitdriver.name, id(jd), jd.vectorize)
+        #jd.xxxbench = XXXBench(jd.jitdriver.name, id(jd), jd.vectorize)
         # XXX XXX XXX
         self.jitdrivers_sd.append(jd)
 
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -370,7 +370,7 @@
                 vinfo.clear_vable_token(virtualizable)
 
             # XXX debug purpose only
-            jitdriver_sd.xxxbench.xxx_clock_start()
+            #jitdriver_sd.xxxbench.xxx_clock_start()
             # XXX debug purpose only end
 
             deadframe = func_execute_token(loop_token, *args)
@@ -379,7 +379,7 @@
             # so that it will keep it alive for a longer time
             warmrunnerdesc.memory_manager.keep_loop_alive(loop_token)
             # XXX debug purpose only
-            jitdriver_sd.xxxbench.xxx_clock_stop(fail=True)
+            #jitdriver_sd.xxxbench.xxx_clock_stop(fail=True)
             # XXX debug purpose only end
             #
             # Handle the failure

From noreply at buildbot.pypy.org  Thu Jul  9 10:16:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 10:16:36 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #213: in case we'd give the error
 message "initializer for ctype
Message-ID: <20150709081636.C95F51C06B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2217:2001880ed1c7
Date: 2015-07-09 10:17 +0200
http://bitbucket.org/cffi/cffi/changeset/2001880ed1c7/

Log:	Issue #213: in case we'd give the error message "initializer for
	ctype 'A' must be a pointer to same type, not cdata 'B'", but with
	A=B, then give instead a different error message to try to clear up
	the confusion

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1060,12 +1060,25 @@
 static int _convert_error(PyObject *init, const char *ct_name,
                           const char *expected)
 {
-    if (CData_Check(init))
-        PyErr_Format(PyExc_TypeError,
-                     "initializer for ctype '%s' must be a %s, "
-                     "not cdata '%s'",
-                     ct_name, expected,
-                     ((CDataObject *)init)->c_type->ct_name);
+    if (CData_Check(init)) {
+        const char *ct_name_2 = ((CDataObject *)init)->c_type->ct_name;
+        if (strcmp(ct_name, ct_name_2) != 0)
+            PyErr_Format(PyExc_TypeError,
+                         "initializer for ctype '%s' must be a %s, "
+                         "not cdata '%s'",
+                         ct_name, expected, ct_name_2);
+        else {
+            /* in case we'd give the error message "initializer for
+               ctype 'A' must be a pointer to same type, not cdata
+               'B'", but with A=B, then give instead a different error
+               message to try to clear up the confusion */
+            PyErr_Format(PyExc_TypeError,
+                         "initializer for ctype '%s' appears indeed to be '%s',"
+                         " but the types are different (check that you are not"
+                         " e.g. mixing up different ffi instances)",
+                         ct_name, ct_name_2);
+        }
+    }
     else
         PyErr_Format(PyExc_TypeError,
                      "initializer for ctype '%s' must be a %s, "
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3413,6 +3413,29 @@
     py.test.raises(RuntimeError, "p[42]")
     py.test.raises(RuntimeError, "p[42] = -1")
 
+def test_mixup():
+    BStruct1 = new_struct_type("foo")
+    BStruct2 = new_struct_type("foo")   # <= same name as BStruct1
+    BStruct3 = new_struct_type("bar")
+    BStruct1Ptr = new_pointer_type(BStruct1)
+    BStruct2Ptr = new_pointer_type(BStruct2)
+    BStruct3Ptr = new_pointer_type(BStruct3)
+    BStruct1PtrPtr = new_pointer_type(BStruct1Ptr)
+    BStruct2PtrPtr = new_pointer_type(BStruct2Ptr)
+    BStruct3PtrPtr = new_pointer_type(BStruct3Ptr)
+    pp1 = newp(BStruct1PtrPtr)
+    pp2 = newp(BStruct2PtrPtr)
+    pp3 = newp(BStruct3PtrPtr)
+    pp1[0] = pp1[0]
+    e = py.test.raises(TypeError, "pp3[0] = pp1[0]")
+    assert str(e.value).startswith("initializer for ctype 'bar *' must be a ")
+    assert str(e.value).endswith(", not cdata 'foo *'")
+    e = py.test.raises(TypeError, "pp2[0] = pp1[0]")
+    assert str(e.value) == ("initializer for ctype 'foo *' appears indeed to "
+                            "be 'foo *', but the types are different (check "
+                            "that you are not e.g. mixing up different ffi "
+                            "instances)")
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "1.2.0"

From noreply at buildbot.pypy.org  Thu Jul  9 10:21:50 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  9 Jul 2015 10:21:50 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: reorder jit driver params
Message-ID: <20150709082150.BB4FD1C06B9@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78501:f845bf5b6e2f
Date: 2015-07-09 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/f845bf5b6e2f/

Log:	reorder jit driver params

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -15,7 +15,7 @@
 
 call2_driver = jit.JitDriver(
     name='numpy_call2',
-    greens=['shapelen', 'func', 'calc_dtype', 'res_dtype', 'left', 'right'],
+    greens=['shapelen', 'func', 'left', 'right', 'calc_dtype', 'res_dtype'],
     reds='auto', vectorize=True)
 
 def call2(space, shape, func, calc_dtype, w_lhs, w_rhs, out):
@@ -40,9 +40,9 @@
     res_dtype = out.get_dtype()
     while not out_iter.done(out_state):
         call2_driver.jit_merge_point(shapelen=shapelen, func=func,
-                                     calc_dtype=calc_dtype, res_dtype=res_dtype,
                                      left=left_iter is None,
-                                     right=right_iter is None)
+                                     right=right_iter is None,
+                                     calc_dtype=calc_dtype, res_dtype=res_dtype)
         if left_iter:
             w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
             left_state = left_iter.next(left_state)

From noreply at buildbot.pypy.org  Thu Jul  9 10:41:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 10:41:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: update to cffi/2001880ed1c7
Message-ID: <20150709084107.9C0811C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78502:177cb8be0682
Date: 2015-07-09 10:40 +0200
http://bitbucket.org/pypy/pypy/changeset/177cb8be0682/

Log:	update to cffi/2001880ed1c7

diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -90,6 +90,16 @@
     def _convert_error(self, expected, w_got):
         space = self.space
         if isinstance(w_got, cdataobj.W_CData):
+            if self.name == w_got.ctype.name:
+                # in case we'd give the error message "initializer for
+                # ctype 'A' must be a pointer to same type, not cdata
+                # 'B'", but with A=B, then give instead a different error
+                # message to try to clear up the confusion
+                return oefmt(space.w_TypeError,
+                             "initializer for ctype '%s' appears indeed to "
+                             "be '%s', but the types are different (check "
+                             "that you are not e.g. mixing up different ffi "
+                             "instances)", self.name, w_got.ctype.name)
             return oefmt(space.w_TypeError,
                          "initializer for ctype '%s' must be a %s, not cdata "
                          "'%s'", self.name, expected, w_got.ctype.name)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3402,6 +3402,29 @@
     py.test.raises(RuntimeError, "p[42]")
     py.test.raises(RuntimeError, "p[42] = -1")
 
+def test_mixup():
+    BStruct1 = new_struct_type("foo")
+    BStruct2 = new_struct_type("foo")   # <= same name as BStruct1
+    BStruct3 = new_struct_type("bar")
+    BStruct1Ptr = new_pointer_type(BStruct1)
+    BStruct2Ptr = new_pointer_type(BStruct2)
+    BStruct3Ptr = new_pointer_type(BStruct3)
+    BStruct1PtrPtr = new_pointer_type(BStruct1Ptr)
+    BStruct2PtrPtr = new_pointer_type(BStruct2Ptr)
+    BStruct3PtrPtr = new_pointer_type(BStruct3Ptr)
+    pp1 = newp(BStruct1PtrPtr)
+    pp2 = newp(BStruct2PtrPtr)
+    pp3 = newp(BStruct3PtrPtr)
+    pp1[0] = pp1[0]
+    e = py.test.raises(TypeError, "pp3[0] = pp1[0]")
+    assert str(e.value).startswith("initializer for ctype 'bar *' must be a ")
+    assert str(e.value).endswith(", not cdata 'foo *'")
+    e = py.test.raises(TypeError, "pp2[0] = pp1[0]")
+    assert str(e.value) == ("initializer for ctype 'foo *' appears indeed to "
+                            "be 'foo *', but the types are different (check "
+                            "that you are not e.g. mixing up different ffi "
+                            "instances)")
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "1.2.0"

From noreply at buildbot.pypy.org  Thu Jul  9 10:41:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 10:41:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: Tweak the example to show a
 non-constant-folded unicode() call
Message-ID: <20150709084108.C9FC51C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78503:eaf153763a84
Date: 2015-07-09 10:40 +0200
http://bitbucket.org/pypy/pypy/changeset/eaf153763a84/

Log:	Tweak the example to show a non-constant-folded unicode() call

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -238,7 +238,7 @@
         log = self.run("""
         def main(n):
             for i in xrange(n):
-                unicode('abc')
+                unicode(str(i))
             return i
         """, [1000])
         loop, = log.loops_by_filename(self.filepath)
@@ -248,10 +248,10 @@
         i50 = int_add(i47, 1)
         setfield_gc(p15, i50, descr=<FieldS pypy.module.__builtin__.functional.W_XRangeIterator.inst_current 8>)
         guard_not_invalidated(descr=...)
-        p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=<Callr . rii EF=5>)
+        p80 = call(ConstClass(ll_str__IntegerR_SignedConst_Signed), i47, descr=<Callr . i EF=3>)
         guard_no_exception(descr=...)
-        p53 = getfield_gc_pure(p52, descr=<FieldP tuple2.item0 .>)
-        guard_nonnull(p53, descr=...)
+        p53 = call(ConstClass(fast_str_decode_ascii), p80, descr=<Callr . r EF=4>)
+        guard_no_exception(descr=...)
         --TICK--
         jump(..., descr=...)
         """)

From noreply at buildbot.pypy.org  Thu Jul  9 10:41:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 10:41:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150709084110.249CD1C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78504:8310cf810f40
Date: 2015-07-09 10:41 +0200
http://bitbucket.org/pypy/pypy/changeset/8310cf810f40/

Log:	merge heads

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -540,6 +540,15 @@
         for v in [float('inf'), float('-inf'), float('nan'), float('-nan')]:
             assert math.isnan(fmod(v, 2))
 
+    def test_mod(self):
+        from numpy import mod
+        assert mod(5, 3) == 2
+        assert mod(5, -3) == -1
+        assert mod(-5, 3) == 1
+        assert mod(-5, -3) == -2
+        assert mod(2.5, 1) == 0.5
+        assert mod(-1.5, 2) == 0.5
+
     def test_minimum(self):
         from numpy import array, minimum, nan, isnan
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -759,7 +759,21 @@
 
     @simple_binary_op
     def mod(self, v1, v2):
-        return math.fmod(v1, v2)
+        # partial copy of pypy.objspace.std.floatobject.W_FloatObject.descr_mod
+        if v2 == 0.0:
+            return rfloat.NAN
+        mod = math.fmod(v1, v2)
+        if mod:
+            # ensure the remainder has the same sign as the denominator
+            if (v2 < 0.0) != (mod < 0.0):
+                mod += v2
+        else:
+            # the remainder is zero, and in the presence of signed zeroes
+            # fmod returns different results across platforms; ensure
+            # it has the same sign as the denominator; we'd like to do
+            # "mod = v2 * 0.0", but that may get optimized away
+            mod = rfloat.copysign(0.0, v2)
+        return mod
 
     @simple_binary_op
     def pow(self, v1, v2):

From noreply at buildbot.pypy.org  Thu Jul  9 13:10:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 13:10:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move "time" one level down the
 hierarchy of modules: nowadays we can't
Message-ID: <20150709111047.CDA131C06B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78505:9242aeaed1e3
Date: 2015-07-09 13:10 +0200
http://bitbucket.org/pypy/pypy/changeset/9242aeaed1e3/

Log:	Move "time" one level down the hierarchy of modules: nowadays we
	can't really do anything without having it translated

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -23,14 +23,14 @@
 default_modules.update([
     "_codecs", "gc", "_weakref", "marshal", "errno", "imp", "math", "cmath",
     "_sre", "_pickle_support", "operator", "parser", "symbol", "token", "_ast",
-    "_io", "_random", "__pypy__", "_testing"
+    "_io", "_random", "__pypy__", "_testing", "time"
 ])
 
 
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",

From noreply at buildbot.pypy.org  Thu Jul  9 13:33:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 13:33:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Only do the final "Create cffi bindings
 for modules" step if the
Message-ID: <20150709113345.0F4451C06B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78506:c8bcecbc4a8e
Date: 2015-07-09 12:35 +0100
http://bitbucket.org/pypy/pypy/changeset/c8bcecbc4a8e/

Log:	Only do the final "Create cffi bindings for modules" step if the
	_cffi_backend module is enabled. It is not, for example, with
	--sandbox translations, which hang when used in this way.

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -297,7 +297,12 @@
         options = make_dict(config)
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
+        if config.objspace.usemodules._cffi_backend:
+            self.hack_for_cffi_modules(driver)
 
+        return self.get_entry_point(config)
+    
+    def hack_for_cffi_modules(self, driver):
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_c to build_cffi_imports
         # this should probably get cleaned up and merged with driver.create_exe
@@ -336,8 +341,6 @@
         driver.default_goal = 'build_cffi_imports'
         # HACKHACKHACK end
 
-        return self.get_entry_point(config)
-
     def jitpolicy(self, driver):
         from pypy.module.pypyjit.policy import PyPyJitPolicy
         from pypy.module.pypyjit.hooks import pypy_hooks

From noreply at buildbot.pypy.org  Thu Jul  9 16:43:43 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  9 Jul 2015 16:43:43 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: reviewed unrolling, and guard relaxation
Message-ID: <20150709144343.06EC11C048F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78507:bc0821d1703e
Date: 2015-07-09 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/bc0821d1703e/

Log:	reviewed unrolling, and guard relaxation

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -103,7 +103,6 @@
         self.packset = None
         self.unroll_count = 0
         self.smallest_type_bytes = 0
-        self.early_exit_idx = -1
         self.sched_data = None
         self.cpu = metainterp_sd.cpu
         self.costmodel = X86_CostModel(cost_threshold, self.cpu.vector_register_size)
@@ -166,8 +165,8 @@
         label_op = loop.operations[0].clone()
         assert label_op.getopnum() == rop.LABEL
         jump_op = loop.operations[op_count-1]
+        assert jump_op.getopnum() in (rop.LABEL, rop.JUMP)
         # use the target token of the label
-        assert jump_op.getopnum() in (rop.LABEL, rop.JUMP)
         target_token = label_op.getdescr()
         if not we_are_translated():
             target_token.assumed_classes = {}
@@ -181,17 +180,14 @@
         self.emit_unrolled_operation(label_op)
 
         renamer = Renamer()
-        oi = 0
         pure = True
         operations = []
         ee_pos = -1
-        ee_guard = None
         for i in range(1,op_count-1):
             op = loop.operations[i].clone()
             opnum = op.getopnum()
             if opnum == rop.GUARD_EARLY_EXIT:
                 ee_pos = i
-                ee_guard = op
 
             if op.is_guard():
                 assert isinstance(op, GuardResOp)
@@ -208,7 +204,7 @@
         orig_jump_args = jump_op.getarglist()[:]
         # it is assumed that #label_args == #jump_args
         label_arg_count = len(orig_jump_args)
-        for i in range(0, unroll_count):
+        for u in range(unroll_count):
             # fill the map with the renaming boxes. keys are boxes from the label
             for i in range(label_arg_count):
                 la = label_op.getarg(i)
@@ -217,7 +213,7 @@
                 if la != ja:
                     renamer.start_renaming(la, ja)
             #
-            for oi, op in enumerate(operations):
+            for i, op in enumerate(operations):
                 if op.getopnum() in prohibit_opnums:
                     continue # do not unroll this operation twice
                 copied_op = op.clone()
@@ -229,33 +225,30 @@
                     copied_op.result = new_assigned_box
                 #
                 args = copied_op.getarglist()
-                for i, arg in enumerate(args):
+                for a, arg in enumerate(args):
                     value = renamer.rename_box(arg)
-                    copied_op.setarg(i, value)
+                    copied_op.setarg(a, value)
                 # not only the arguments, but also the fail args need
                 # to be adjusted. rd_snapshot stores the live variables
                 # that are needed to resume.
                 if copied_op.is_guard():
                     assert isinstance(copied_op, GuardResOp)
                     target_guard = copied_op
-                    # do not overwrite resume at loop header
-                    if target_guard.getdescr().guard_opnum != rop.GUARD_EARLY_EXIT:
+                    descr = target_guard.getdescr()
+                    exits_early = descr.guard_opnum == rop.GUARD_EARLY_EXIT
+                    # early exits already have the right failargs set
+                    if not exits_early:
                         descr = invent_fail_descr_for_op(copied_op.getopnum(), self)
                         olddescr = copied_op.getdescr()
                         if olddescr:
                             descr.copy_all_attributes_from(olddescr)
                         copied_op.setdescr(descr)
-
-                    if oi < ee_pos:
-                        # do not clone the arguments, it is already an early exit
-                        pass
-                    else:
+                        # copy failargs/snapshot
                         copied_op.rd_snapshot = \
                           renamer.rename_rd_snapshot(copied_op.rd_snapshot,
                                                      clone=True)
                         renamed_failargs = \
-                            renamer.rename_failargs(copied_op,
-                                                    clone=True)
+                            renamer.rename_failargs(copied_op, clone=True)
                         copied_op.setfailargs(renamed_failargs)
                 #
                 self.emit_unrolled_operation(copied_op)
@@ -267,14 +260,12 @@
         for i, arg in enumerate(args):
             value = renamer.rename_box(arg)
             jump_op.setarg(i, value)
-
+        #
         self.emit_unrolled_operation(jump_op)
 
     def linear_find_smallest_type(self, loop):
         # O(#operations)
         for i,op in enumerate(loop.operations):
-            if op.getopnum() == rop.GUARD_EARLY_EXIT:
-                self.early_exit_idx = i
             if op.is_raw_array_access():
                 descr = op.getdescr()
                 if not descr.is_array_of_pointers():
@@ -525,11 +516,13 @@
         return arg
 
     def analyse_index_calculations(self):
-        if len(self.loop.operations) <= 1 or self.early_exit_idx == -1:
+        ee_pos = 1
+        ops = self.loop.operations
+        if len(ops) <= 2 or ops[ee_pos].getopnum() != rop.GUARD_EARLY_EXIT:
             return
         self.dependency_graph = graph = DependencyGraph(self.loop)
         label_node = graph.getnode(0)
-        ee_guard_node = graph.getnode(self.early_exit_idx)
+        ee_guard_node = graph.getnode(ee_pos)
         guards = graph.guards
         for guard_node in guards:
             if guard_node is ee_guard_node:

From noreply at buildbot.pypy.org  Thu Jul  9 16:43:44 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  9 Jul 2015 16:43:44 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: continued refactoring of accumulation,
 removed extra accum box, but made vector box more generic
Message-ID: <20150709144344.481081C048F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78508:7b660544d874
Date: 2015-07-09 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/7b660544d874/

Log:	continued refactoring of accumulation, removed extra accum box, but
	made vector box more generic moved some methods down the hierarchy
	to prevent assert isinstance(...)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -3,7 +3,7 @@
 from rpython.jit.backend.llgraph import support
 from rpython.jit.backend.llsupport import symbolic
 from rpython.jit.metainterp.history import AbstractDescr
-from rpython.jit.metainterp.history import Const, getkind, BoxVectorAccum
+from rpython.jit.metainterp.history import Const, getkind
 from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID, VECTOR
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.optimizeopt import intbounds
@@ -31,11 +31,7 @@
             try:
                 newbox = _cache[box]
             except KeyError:
-                if isinstance(box, BoxVectorAccum):
-                    newbox = _cache[box] = \
-                        box.__class__(box, box.scalar_var, box.operator)
-                else:
-                    newbox = _cache[box] = box.__class__()
+                newbox = _cache[box] = box.__class__()
             return newbox
         #
         self.inputargs = map(mapping, inputargs)
@@ -877,10 +873,10 @@
                 value = self.env[box]
             else:
                 value = None
-            if isinstance(box, BoxVectorAccum):
-                if box.operator == '+':
+            if box.getaccum():
+                if box.getaccum().operator == '+':
                     value = sum(value)
-                elif box.operator == '*':
+                elif box.getaccum().operator == '*':
                     def prod(acc, x): return acc * x
                     value = reduce(prod, value, 1)
                 else:
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -23,7 +23,7 @@
 from rpython.jit.codewriter import longlong
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
-    ConstFloat, BoxInt, BoxFloat, BoxVector, BoxVectorAccum, INT, REF,
+    ConstFloat, BoxInt, BoxFloat, BoxVector, INT, REF,
     FLOAT, VECTOR, TargetToken)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.metainterp.compile import ResumeGuardDescr
@@ -308,8 +308,11 @@
         faillocs = []
         descr = guard_op.getdescr()
         for v in guard_op.getfailargs():
-            if v is not None and isinstance(v, BoxVectorAccum):
-                loc = self.loc(v.scalar_var)
+            if v is None:
+                continue
+            accum = v.getaccum()
+            if accum:
+                loc = self.loc(accum.getvar())
                 self.update_accumulation_loc(v, descr)
                 faillocs.append(loc)
             else:
@@ -317,7 +320,7 @@
 
         return faillocs
 
-    def update_accumulation_loc(self, accumbox, descr):
+    def update_accumulation_loc(self, box, accum, descr):
         """ Saves the location to the AccumInfo object.
         Necessary to reconstruct the values at a guard exit.
         """
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -1,6 +1,6 @@
 import py
 from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
-    ConstFloat, BoxInt, BoxFloat, BoxVector, BoxVectorAccum, INT, REF,
+    ConstFloat, BoxInt, BoxFloat, BoxVector, INT, REF,
     FLOAT, VECTOR, TargetToken)
 from rpython.jit.backend.llsupport.descr import (ArrayDescr, CallDescr,
     unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr)
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -198,16 +198,16 @@
     if loop.versions is not None:
         token = jitcell_token
         for version in loop.versions:
-            versioned_loop = create_empty_loop(metainterp)
-            versioned_loop.inputargs = version.inputargs
-            versioned_loop.operations = version.operations
-            versioned_loop.original_jitcell_token = jitcell_token
-            for _, faildescr  in version.faildescrs:
+            for faildescr  in version.faildescrs:
+                vl = create_empty_loop(metainterp)
+                vl.inputargs = version.inputargs
+                vl.operations = version.operations
+                vl.original_jitcell_token = jitcell_token
                 send_bridge_to_backend(jitdriver_sd, metainterp_sd,
                                        faildescr, version.inputargs,
                                        version.operations, jitcell_token)
-                versioned_loop.original_jitcell_token = jitcell_token
-                record_loop_or_bridge(metainterp_sd, versioned_loop)
+                vl.original_jitcell_token = jitcell_token
+                record_loop_or_bridge(metainterp_sd, vl)
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
@@ -514,7 +514,8 @@
 
 class ResumeGuardDescr(ResumeDescr):
     _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
-               'rd_frame_info_list', 'rd_pendingfields', 'status')
+               'rd_frame_info_list', 'rd_pendingfields', 'rd_accum_list',
+               'status')
     
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
@@ -715,6 +716,9 @@
 class ResumeAtLoopHeaderDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_EARLY_EXIT
 
+    def exits_early(self):
+        return True
+
 class CompileLoopVersionDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_EARLY_EXIT
 
@@ -725,6 +729,12 @@
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         assert 0, "this guard must never fail"
 
+    def exits_early(self):
+        return True
+
+    def loop_version(self):
+        return True
+
 class AllVirtuals:
     llopaque = True
     cache = None
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -131,6 +131,8 @@
         # only structured containers can compare their shape (vector box)
         return True
 
+    def getaccum(self):
+        return None
 
 class AbstractDescr(AbstractValue):
     __slots__ = ()
@@ -163,6 +165,15 @@
     def compile_and_attach(self, metainterp, new_loop):
         raise NotImplementedError
 
+    def exits_early(self):
+        # is this guard either a guard_early_exit resop,
+        # or it has been moved before an guard_early_exit
+        return False
+
+    def loop_version(self):
+        # compile a loop version out of this guard?
+        return False
+
 class BasicFinalDescr(AbstractFailDescr):
     final_descr = True
 
@@ -519,24 +530,53 @@
 
 # ____________________________________________________________
 
+class Accum(object):
+    PLUS = '+'
+    MULTIPLY = '*'
+
+    def __init__(self, opnum, var, pos):
+        self.var = var
+        self.pos = pos
+        self.operator = Accum.PLUS
+        if opnum == rop.FLOAT_MUL:
+            self.operator = Accum.MULTIPLY
+
+    def getoriginalbox(self):
+        return self.var
+
+    def getop(self):
+        return self.operator
+
+    def accumulates_value(self):
+        return True
+
+    def save_to_descr(self, descr, position):
+        assert isinstance(descr,ResumeGuardDescr)
+        ai = AccumInfo(descr.rd_accum_list, position, self.operator, self.var)
+        descr.rd_accum_list = ai
+
 class BoxVector(Box):
     type = VECTOR
-    _attrs_ = ('item_type','item_count','item_size','item_signed')
+    _attrs_ = ('item_type','item_count','item_size','item_signed','accum')
     _extended_display = False
 
-    def __init__(self, item_type=FLOAT, item_count=2, item_size=8, item_signed=False):
+    def __init__(self, item_type=FLOAT, item_count=2, item_size=8, item_signed=False, accum=None):
         assert item_type in (FLOAT, INT)
         self.item_type = item_type
         self.item_count = item_count
         self.item_size = item_size
         self.item_signed = item_signed
+        self.accum = None
 
     def gettype(self):
         return self.item_type
+
     def getsize(self):
         return self.item_size
+
     def getsigned(self):
         return self.item_signed
+
     def getcount(self):
         return self.item_count
 
@@ -576,11 +616,8 @@
             return False
         return True
 
-class BoxVectorAccum(BoxVector):
-    def __init__(self, box, var, operator):
-        BoxVector.__init__(self, box.item_type, box.item_count, box.item_size, box.item_signed)
-        self.scalar_var = var
-        self.operator = operator
+    def getaccum(self):
+        return self.accum
 
 # ____________________________________________________________
 
@@ -697,8 +734,8 @@
 
 class LoopVersion(object):
 
-    def __init__(self, loop, aligned=False):
-        self.operations = loop.operations
+    def __init__(self, operations, opt_ops, aligned=False):
+        self.operations = operations
         self.aligned = aligned
         self.faildescrs = []
         #
@@ -711,9 +748,18 @@
             i += 1
         assert label.getopnum() == rop.LABEL
         self.label_pos = i
-        #self.parent_trace_label_args = None
-        #self.bridge_label_args = label.getarglist()
         self.inputargs = label.getarglist()
+        for op in opt_ops:
+            if op.is_guard():
+                descr = op.getdescr()
+                if descr.loop_version():
+                    # currently there is only ONE versioning,
+                    # that is the original loop after unrolling.
+                    # if there are more possibilites, let the descr
+                    # know which loop version he preferes
+                    self.faildescrs.append(descr)
+                    op.setfailargs(self.inputargs)
+                    op.rd_snapshot = None
 
     def adddescr(self, op, descr):
         self.faildescrs.append((op, descr))
@@ -772,6 +818,13 @@
     def get_operations(self):
         return self.operations
 
+    def find_first_index(self, opnum):
+        """ return the first operation having the same opnum or -1 """
+        for i,op in enumerate(self.operations):
+            if op.getopnum() == opnum:
+                return i
+        return -1
+
     def get_display_text(self):    # for graphpage.py
         return self.name + '\n' + repr(self.inputargs)
 
@@ -803,20 +856,14 @@
         for arg in inputargs:
             if arg is None:
                 continue
-            if isinstance(arg, BoxVectorAccum):
-                seen[arg.scalar_var] = None
-            else:
-                seen[arg] = None
+            seen[arg] = None
         return seen
 
     @staticmethod
     def check_if_box_was_seen(box, seen):
         if box is not None:
             assert isinstance(box, Box)
-            if isinstance(box, BoxVectorAccum):
-                assert box in seen or box.scalar_var in seen
-            else:
-                assert box in seen
+            assert box in seen
 
     @staticmethod
     def check_consistency_of_branch(operations, seen):
@@ -846,9 +893,6 @@
                     assert isinstance(box, Box), "LABEL contains %r" % (box,)
                 seen = TreeLoop.seen_args(inputargs)
                 seen_count = len(seen)
-                for arg in seen:
-                    if isinstance(arg, BoxVectorAccum):
-                        seen_count -= 1
                 assert seen_count == len(inputargs), (
                     "duplicate Box in the LABEL arguments")
 
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -857,17 +857,6 @@
 
         return oplist
 
-class Accum(object):
-    PLUS = '+'
-    MULTIPLY = '*'
-
-    def __init__(self, opnum, var, pos):
-        self.var = var
-        self.pos = pos
-        self.operator = Accum.PLUS
-        if opnum == rop.FLOAT_MUL:
-            self.operator = Accum.MULTIPLY
-
 class Pack(object):
     """ A pack is a set of n statements that are:
         * isomorphic
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -213,7 +213,6 @@
         return True
 
     def rename_failargs(self, guard, clone=False):
-        from rpython.jit.metainterp.history import BoxVectorAccum
         from rpython.jit.metainterp.compile import ResumeGuardDescr
         if guard.getfailargs() is not None:
             if clone:
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -6,7 +6,7 @@
 """
 
 import py
-#XXXimport time
+import time
 
 from rpython.jit.metainterp.resume import Snapshot, AccumInfo
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
@@ -15,13 +15,13 @@
         CompileLoopVersionDescr, invent_fail_descr_for_op, ResumeGuardDescr)
 from rpython.jit.metainterp.history import (ConstInt, VECTOR, FLOAT, INT,
         BoxVector, BoxFloat, BoxInt, ConstFloat, TargetToken, JitCellToken, Box,
-        BoxVectorAccum, LoopVersion)
+        LoopVersion, Accum)
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method, Renamer
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
         MemoryRef, Node, IndexVar)
 from rpython.jit.metainterp.optimizeopt.schedule import (VecScheduleData,
-        Scheduler, Pack, Pair, AccumPair, Accum, vectorbox_outof_box, getpackopnum,
+        Scheduler, Pack, Pair, AccumPair, vectorbox_outof_box, getpackopnum,
         getunpackopnum, PackType, determine_input_output_types)
 from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
@@ -36,7 +36,6 @@
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, False)
     orig_ops = loop.operations
-    orig_version = LoopVersion(loop)
     if len(orig_ops) >= 75:
         # if more than 75 operations are present in this loop,
         # it won't be possible to vectorize. There are too many
@@ -47,29 +46,27 @@
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "pre vectorize")
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZE_TRY)
         #
-        #XXXstart = time.clock()
-        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, cost_threshold, orig_version)
+        start = time.clock()
+        #
+        #
+        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, cost_threshold)
         opt.propagate_all_forward()
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
         gso.propagate_all_forward(opt.loop)
-        #XXXend = time.clock()
+        # loop versioning
+        loop.versions = [LoopVersion(orig_ops, loop.operations)]
         #
-        loop.versions = [orig_version]
-
+        #
+        end = time.clock()
+        #
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "post vectorize")
+        #
+        nano = int((end-start)*10.0**9)
+        debug_print("# vecopt factor: %d opcount: (%d -> %d) took %dns" % \
+                      (opt.unroll_count+1, len(orig_ops), len(loop.operations), nano))
         debug_stop("vec-opt-loop")
         #
-        # XXX
-        #XXXns = int((end-start)*10.0**9)
-        #XXXdebug_start("xxx-clock")
-        #XXXdebug_print("vecopt unroll: %d gso count: %d opcount: (%d -> %d) took %dns" % \
-        #XXX              (opt.unroll_count+1,
-        #XXX               gso.strength_reduced,
-        #XXX               len(orig_ops),
-        #XXX               len(loop.operations),
-        #XXX               ns))
-        #XXXdebug_stop("xxx-clock")
     except NotAVectorizeableLoop:
         debug_stop("vec-opt-loop")
         # vectorization is not possible
@@ -93,12 +90,25 @@
     return a.left.getindex() < b.left.getindex()
 packsort = listsort.make_timsort_class(lt=cmp_pack_lt)
 
+def copy_fail_descr(op, optimizer):
+    olddescr = op.getdescr()
+    exits_early = olddescr.guard_opnum == rop.GUARD_EARLY_EXIT
+    if exits_early:
+        if isinstance(olddescr, CompileLoopVersionDescr):
+            descr = CompileLoopVersionDescr()
+        else:
+            descr = ResumeAtLoopHeaderDescr()
+    else:
+        descr = invent_fail_descr_for_op(op.getopnum(), optimizer)
+    if olddescr:
+        descr.copy_all_attributes_from(olddescr)
+    return descr
+
 class VectorizingOptimizer(Optimizer):
     """ Try to unroll the loop and find instructions to group """
 
-    def __init__(self, metainterp_sd, jitdriver_sd, loop, cost_threshold, orig_loop_version):
+    def __init__(self, metainterp_sd, jitdriver_sd, loop, cost_threshold):
         Optimizer.__init__(self, metainterp_sd, jitdriver_sd, loop, [])
-        self.orig_loop_version = orig_loop_version
         self.dependency_graph = None
         self.packset = None
         self.unroll_count = 0
@@ -234,15 +244,10 @@
                 if copied_op.is_guard():
                     assert isinstance(copied_op, GuardResOp)
                     target_guard = copied_op
+                    copied_op.setdescr(copy_fail_descr(copied_op, self))
                     descr = target_guard.getdescr()
                     exits_early = descr.guard_opnum == rop.GUARD_EARLY_EXIT
-                    # early exits already have the right failargs set
                     if not exits_early:
-                        descr = invent_fail_descr_for_op(copied_op.getopnum(), self)
-                        olddescr = copied_op.getdescr()
-                        if olddescr:
-                            descr.copy_all_attributes_from(olddescr)
-                        copied_op.setdescr(descr)
                         # copy failargs/snapshot
                         copied_op.rd_snapshot = \
                           renamer.rename_rd_snapshot(copied_op.rd_snapshot,
@@ -472,11 +477,9 @@
                 op = guard_node.getoperation()
                 failargs = op.getfailargs()
                 for i,arg in enumerate(failargs):
-                    if isinstance(arg, BoxVectorAccum):
-                        descr = op.getdescr()
-                        assert isinstance(descr,ResumeGuardDescr)
-                        ai = AccumInfo(descr.rd_accum_list, i, arg.operator, arg.scalar_var)
-                        descr.rd_accum_list = ai
+                    accum = arg.getaccum()
+                    if accum:
+                        accum.save_to_descr(op.getdescr(),i)
         self.loop.operations = \
             sched_data.prepend_invariant_operations(self._newoperations)
         self.clear_newoperations()
@@ -516,10 +519,9 @@
         return arg
 
     def analyse_index_calculations(self):
-        ee_pos = 1
-        ops = self.loop.operations
-        if len(ops) <= 2 or ops[ee_pos].getopnum() != rop.GUARD_EARLY_EXIT:
-            return
+        ee_pos = self.loop.find_first_index(rop.GUARD_EARLY_EXIT)
+        if len(self.loop.operations) <= 2 or ee_pos == -1:
+            raise NotAVectorizeableLoop()
         self.dependency_graph = graph = DependencyGraph(self.loop)
         label_node = graph.getnode(0)
         ee_guard_node = graph.getnode(ee_pos)
@@ -569,9 +571,9 @@
                         label_node.edge_to(last_but_one, label='pullup')
                 # only the last guard needs a connection
                 guard_node.edge_to(ee_guard_node, label='pullup-last-guard')
-                self.relax_guard_to(guard_node, ee_guard_node, label_node)
+                self.relax_guard_to(guard_node, ee_guard_node)
 
-    def relax_guard_to(self, guard_node, other_node, label_node):
+    def relax_guard_to(self, guard_node, other_node):
         """ Relaxes a guard operation to an earlier guard. """
         # clone this operation object. if the vectorizer is
         # not able to relax guards, it won't leave behind a modified operation
@@ -594,10 +596,6 @@
         tgt_op.setdescr(descr)
         tgt_op.rd_snapshot = op.rd_snapshot
         tgt_op.setfailargs(op.getfailargs())
-        if guard_true_false:
-            self.orig_loop_version.adddescr(tgt_op, descr)
-            tgt_op.setfailargs(label_node.getoperation().getarglist()[:])
-            tgt_op.rd_snapshot = None
 
 
 class CostModel(object):
@@ -840,6 +838,7 @@
             if not pack.is_accumulating():
                 continue
             accum = pack.accum
+            pack.accum = None
             # create a new vector box for the parameters
             box = pack.input_type.new_vector_box()
             size = vec_reg_size // pack.input_type.getsize()
@@ -857,12 +856,14 @@
                 sched_data.invariant_oplist.append(op)
             else:
                 raise NotImplementedError("can only handle + and *")
-            result = BoxVectorAccum(box, accum.var, accum.operator)
+            result = box.clonebox()
+            assert isinstance(result, BoxVector)
+            result.accum = accum
             # pack the scalar value
             op = ResOperation(getpackopnum(box.gettype()),
                               [box, accum.var, ConstInt(0), ConstInt(1)], result)
             sched_data.invariant_oplist.append(op)
             # rename the variable with the box
-            sched_data.setvector_of_box(accum.var, 0, result) # prevent it from expansion
-            renamer.start_renaming(accum.var, result)
+            sched_data.setvector_of_box(accum.getoriginalbox(), 0, result) # prevent it from expansion
+            renamer.start_renaming(accum.getoriginalbox(), result)
 

From noreply at buildbot.pypy.org  Thu Jul  9 16:43:45 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  9 Jul 2015 16:43:45 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: expansion misses some adjustments after
	refactoring
Message-ID: <20150709144345.6E4231C048F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78509:9ab23edf0d53
Date: 2015-07-09 15:42 +0200
http://bitbucket.org/pypy/pypy/changeset/9ab23edf0d53/

Log:	expansion misses some adjustments after refactoring

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -307,29 +307,32 @@
     def locs_for_fail(self, guard_op):
         faillocs = []
         descr = guard_op.getdescr()
-        for v in guard_op.getfailargs():
-            if v is None:
+        for arg in guard_op.getfailargs():
+            if arg is None:
+                faillocs.append(None)
                 continue
-            accum = v.getaccum()
+            accum = arg.getaccum()
             if accum:
-                loc = self.loc(accum.getvar())
-                self.update_accumulation_loc(v, descr)
+                loc = self.loc(accum.getoriginalbox())
                 faillocs.append(loc)
+                self.update_accumulation_loc(arg, accum, descr)
             else:
-                faillocs.append(self.loc(v))
+                faillocs.append(self.loc(arg))
 
         return faillocs
 
-    def update_accumulation_loc(self, box, accum, descr):
-        """ Saves the location to the AccumInfo object.
-        Necessary to reconstruct the values at a guard exit.
+    def update_accumulation_loc(self, arg, accum, descr):
         """
-        box = accumbox.scalar_var
+        Faillocs saved on the guard can only represent one value.
+        Accumulation has the accumulation box which need to updated uppon
+        guard exit. The fail descr saves where (regloc) the accumulator
+        is located.
+        """
         assert isinstance(descr, ResumeGuardDescr)
         accum_info = descr.rd_accum_list
         while accum_info:
-            if accum_info.box is box:
-                accum_info.loc = self.loc(accumbox)
+            if accum_info.box is accum.getoriginalbox():
+                accum_info.loc = self.loc(arg)
                 break
             accum_info = accum_info.prev
         else:
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -551,6 +551,8 @@
         return True
 
     def save_to_descr(self, descr, position):
+        from rpython.jit.metainterp.compile import ResumeGuardDescr
+        from rpython.jit.metainterp.resume import AccumInfo
         assert isinstance(descr,ResumeGuardDescr)
         ai = AccumInfo(descr.rd_accum_list, position, self.operator, self.var)
         descr.rd_accum_list = ai
@@ -732,6 +734,16 @@
     def repr_of_descr(self):
         return 'TargetToken(%d)' % compute_unique_id(self)
 
+def index_of_first(opnum, operations):
+    """ returns the position of the first operation matching the opnum.
+    Or -1 if non is found
+    """
+    for i,op in enumerate(operations):
+        if op.getopnum() == opnum:
+            return i
+    return -1
+
+
 class LoopVersion(object):
 
     def __init__(self, operations, opt_ops, aligned=False):
@@ -739,16 +751,14 @@
         self.aligned = aligned
         self.faildescrs = []
         #
-        i = 0
-        label = self.operations[i]
-        while i < len(self.operations):
-            label = self.operations[i]
-            if label.getopnum() == rop.LABEL:
-                break
-            i += 1
-        assert label.getopnum() == rop.LABEL
-        self.label_pos = i
+        idx = index_of_first(rop.LABEL, operations)
+        assert idx >= 0
+        label = operations[idx]
+        self.label_pos = idx
         self.inputargs = label.getarglist()
+        idx = index_of_first(rop.LABEL, opt_ops)
+        assert idx >= 0
+        version_failargs = opt_ops[idx].getarglist()
         for op in opt_ops:
             if op.is_guard():
                 descr = op.getdescr()
@@ -758,12 +768,9 @@
                     # if there are more possibilites, let the descr
                     # know which loop version he preferes
                     self.faildescrs.append(descr)
-                    op.setfailargs(self.inputargs)
+                    op.setfailargs(version_failargs)
                     op.rd_snapshot = None
 
-    def adddescr(self, op, descr):
-        self.faildescrs.append((op, descr))
-
     def update_token(self, jitcell_token):
         label = self.operations[self.label_pos]
         jump = self.operations[-1]
@@ -820,10 +827,7 @@
 
     def find_first_index(self, opnum):
         """ return the first operation having the same opnum or -1 """
-        for i,op in enumerate(self.operations):
-            if op.getopnum() == opnum:
-                return i
-        return -1
+        return index_of_first(opnum, self.operations)
 
     def get_display_text(self):    # for graphpage.py
         return self.name + '\n' + repr(self.inputargs)
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -213,15 +213,13 @@
         return True
 
     def rename_failargs(self, guard, clone=False):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
         if guard.getfailargs() is not None:
             if clone:
                 args = guard.getfailargs()[:]
             else:
                 args = guard.getfailargs()
             for i,arg in enumerate(args):
-                value = self.rename_map.get(arg,arg)
-                args[i] = value
+                args[i] = self.rename_map.get(arg,arg)
             return args
         return None
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -8,14 +8,14 @@
 import py
 import time
 
-from rpython.jit.metainterp.resume import Snapshot, AccumInfo
+from rpython.jit.metainterp.resume import Snapshot
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
 from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
 from rpython.jit.metainterp.compile import (ResumeAtLoopHeaderDescr,
         CompileLoopVersionDescr, invent_fail_descr_for_op, ResumeGuardDescr)
 from rpython.jit.metainterp.history import (ConstInt, VECTOR, FLOAT, INT,
         BoxVector, BoxFloat, BoxInt, ConstFloat, TargetToken, JitCellToken, Box,
-        LoopVersion, Accum)
+        LoopVersion, Accum, AbstractFailDescr)
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method, Renamer
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
@@ -246,15 +246,13 @@
                     target_guard = copied_op
                     copied_op.setdescr(copy_fail_descr(copied_op, self))
                     descr = target_guard.getdescr()
-                    exits_early = descr.guard_opnum == rop.GUARD_EARLY_EXIT
-                    if not exits_early:
-                        # copy failargs/snapshot
-                        copied_op.rd_snapshot = \
-                          renamer.rename_rd_snapshot(copied_op.rd_snapshot,
-                                                     clone=True)
-                        renamed_failargs = \
-                            renamer.rename_failargs(copied_op, clone=True)
-                        copied_op.setfailargs(renamed_failargs)
+                    # copy failargs/snapshot
+                    copied_op.rd_snapshot = \
+                      renamer.rename_rd_snapshot(copied_op.rd_snapshot,
+                                                 clone=True)
+                    renamed_failargs = \
+                        renamer.rename_failargs(copied_op, clone=True)
+                    copied_op.setfailargs(renamed_failargs)
                 #
                 self.emit_unrolled_operation(copied_op)
 
@@ -449,7 +447,6 @@
                 assert False
 
     def schedule(self, vector=False):
-        self.guard_early_exit = -1
         self.clear_newoperations()
         sched_data = VecScheduleData(self.cpu.vector_register_size, self.costmodel)
         scheduler = Scheduler(self.dependency_graph, sched_data)
@@ -838,7 +835,6 @@
             if not pack.is_accumulating():
                 continue
             accum = pack.accum
-            pack.accum = None
             # create a new vector box for the parameters
             box = pack.input_type.new_vector_box()
             size = vec_reg_size // pack.input_type.getsize()
@@ -866,4 +862,6 @@
             # rename the variable with the box
             sched_data.setvector_of_box(accum.getoriginalbox(), 0, result) # prevent it from expansion
             renamer.start_renaming(accum.getoriginalbox(), result)
+            if not we_are_translated():
+                print "renaming accum", accum.getoriginalbox(), "->", result
 

From noreply at buildbot.pypy.org  Thu Jul  9 16:43:46 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu,  9 Jul 2015 16:43:46 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: all tests zjit test passing again
Message-ID: <20150709144346.8EDAB1C048F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78510:a3f009cd3dc4
Date: 2015-07-09 16:43 +0200
http://bitbucket.org/pypy/pypy/changeset/a3f009cd3dc4/

Log:	all tests zjit test passing again

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -307,7 +307,7 @@
     def locs_for_fail(self, guard_op):
         faillocs = []
         descr = guard_op.getdescr()
-        for arg in guard_op.getfailargs():
+        for i,arg in enumerate(guard_op.getfailargs()):
             if arg is None:
                 faillocs.append(None)
                 continue
@@ -315,13 +315,13 @@
             if accum:
                 loc = self.loc(accum.getoriginalbox())
                 faillocs.append(loc)
-                self.update_accumulation_loc(arg, accum, descr)
+                self.update_accumulation_loc(arg, accum, descr, i)
             else:
                 faillocs.append(self.loc(arg))
 
         return faillocs
 
-    def update_accumulation_loc(self, arg, accum, descr):
+    def update_accumulation_loc(self, arg, accum, descr, pos):
         """
         Faillocs saved on the guard can only represent one value.
         Accumulation has the accumulation box which need to updated uppon
@@ -333,6 +333,7 @@
         while accum_info:
             if accum_info.box is accum.getoriginalbox():
                 accum_info.loc = self.loc(arg)
+                accum_info.position = pos
                 break
             accum_info = accum_info.prev
         else:
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -746,7 +746,7 @@
 
 class LoopVersion(object):
 
-    def __init__(self, operations, opt_ops, aligned=False):
+    def __init__(self, operations, opt_ops, invariant_arg_count=0, aligned=False):
         self.operations = operations
         self.aligned = aligned
         self.faildescrs = []
@@ -759,6 +759,14 @@
         idx = index_of_first(rop.LABEL, opt_ops)
         assert idx >= 0
         version_failargs = opt_ops[idx].getarglist()
+        if invariant_arg_count > 0:
+            # constant/variable expansion append arguments to the label
+            # if they are not removed, the register allocator cannot
+            # reconstruct the binding if len(inputargs) != len(faillocs)
+            to = len(version_failargs) - invariant_arg_count
+            assert to >= 0
+            version_failargs = version_failargs[:to]
+
         for op in opt_ops:
             if op.is_guard():
                 descr = op.getdescr()
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -54,7 +54,7 @@
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
         gso.propagate_all_forward(opt.loop)
         # loop versioning
-        loop.versions = [LoopVersion(orig_ops, loop.operations)]
+        loop.versions = [LoopVersion(orig_ops, loop.operations, opt.appended_arg_count)]
         #
         #
         end = time.clock()
@@ -116,6 +116,7 @@
         self.sched_data = None
         self.cpu = metainterp_sd.cpu
         self.costmodel = X86_CostModel(cost_threshold, self.cpu.vector_register_size)
+        self.appended_arg_count = 0
 
     def propagate_all_forward(self, clear=True):
         self.clear_newoperations()
@@ -470,6 +471,7 @@
             return
         if vector:
             # add accumulation info to the descriptor
+            self.appended_arg_count = len(sched_data.invariant_vector_vars)
             for guard_node in self.dependency_graph.guards:
                 op = guard_node.getoperation()
                 failargs = op.getfailargs()

From noreply at buildbot.pypy.org  Thu Jul  9 17:10:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 17:10:21 +0200 (CEST)
Subject: [pypy-commit] cffi default: Drop the ".. versionchanged" and "..
 versionadded", which was not very
Message-ID: <20150709151021.E59E31C06B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2218:1223303164c1
Date: 2015-07-09 16:35 +0200
http://bitbucket.org/cffi/cffi/changeset/1223303164c1/

Log:	Drop the ".. versionchanged" and ".. versionadded", which was not
	very essential and seems not to work on bitbucket's wiki

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -180,8 +180,6 @@
 
 .. _`common Windows types`: http://msdn.microsoft.com/en-us/library/windows/desktop/aa383751%28v=vs.85%29.aspx
 
-.. "versionadded:: 0.9.3": intmax_t etc.
-
 The declarations can also contain "``...``" at various places; these are
 placeholders that will be completed by the compiler.  More information
 about it below in `Letting the C compiler fill the gaps`_.
@@ -198,17 +196,16 @@
 slow to call ``ffi.cdef()`` a lot of times, a consideration that is
 important mainly in in-line mode.
 
-.. versionadded:: 0.8.2
-   The ``ffi.cdef()`` call takes an optional
-   argument ``packed``: if True, then all structs declared within
-   this cdef are "packed".  If you need both packed and non-packed
-   structs, use several cdefs in sequence.)  This
-   has a meaning similar to ``__attribute__((packed))`` in GCC.  It
-   specifies that all structure fields should have an alignment of one
-   byte.  (Note that the packed attribute has no effect on bit fields so
-   far, which mean that they may be packed differently than on GCC.
-   Also, this has no effect on structs declared with ``"...;"``---next
-   section.)
+The ``ffi.cdef()`` call takes an optional
+argument ``packed``: if True, then all structs declared within
+this cdef are "packed".  If you need both packed and non-packed
+structs, use several cdefs in sequence.)  This
+has a meaning similar to ``__attribute__((packed))`` in GCC.  It
+specifies that all structure fields should have an alignment of one
+byte.  (Note that the packed attribute has no effect on bit fields so
+far, which mean that they may be packed differently than on GCC.
+Also, this has no effect on structs declared with ``"...;"``---next
+section.)
 
 .. _`ffi.set_unicode()`:
 
@@ -233,8 +230,6 @@
 ``TCHAR`` and friends where hard-coded as unicode, but ``UNICODE`` was,
 inconsistently, not defined by default.)
 
-.. "versionadded:: 0.9" --- inlined in the previous paragraph
-
 
 ffi.dlopen(): loading libraries in ABI mode
 -------------------------------------------
@@ -565,7 +560,7 @@
 * Function pointers with non-default calling conventions (e.g. on
   Windows, "stdcall").
 
-Note that since version 0.8, declarations like ``int field[];`` in
+Note that declarations like ``int field[];`` in
 structures are interpreted as variable-length structures.  Declarations
 like ``int field[...];`` on the other hand are arrays whose length is
 going to be completed by the compiler.  You can use ``int field[];``
@@ -575,10 +570,10 @@
 get reduced safety checks: for example, you risk overwriting the
 following fields by passing too many array items in the constructor.
 
-.. versionadded:: 1.2
-   Thread-local variables (``__thread``) can be accessed, as well as
-   variables defined as dynamic macros (``#define myvar  (*fetchme())``).
-   Before version 1.2, you need to write getter/setter functions.
+*New in version 1.2:*
+Thread-local variables (``__thread``) can be accessed, as well as
+variables defined as dynamic macros (``#define myvar  (*fetchme())``).
+Before version 1.2, you need to write getter/setter functions.
 
 
 Debugging dlopen'ed C libraries
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -41,13 +41,16 @@
 
 Example::
 
-    >>> ffi.new("char *")
-    <cdata 'char *' owning 1 bytes>
     >>> ffi.new("int *")
     <cdata 'int *' owning 4 bytes>
     >>> ffi.new("int[10]")
     <cdata 'int[10]' owning 40 bytes>
 
+    >>> ffi.new("char *")          # allocates only one char---not a C string!
+    <cdata 'char *' owning 1 bytes>
+    >>> ffi.new("char[]", "foobar")  # this allocates a C string, ending in \0
+    <cdata 'char[]' owning 7 bytes>
+
 Unlike C, the returned pointer object has *ownership* on the allocated
 memory: when this exact object is garbage-collected, then the memory is
 freed.  If, at the level of C, you store a pointer to the memory
@@ -514,34 +517,32 @@
 discouraged: using this a style, we are more likely to forget the
 callback object too early, when it is still in use.
 
-.. versionadded:: 1.2
+*New in version 1.2:* If you want to be sure to catch all exceptions, use
+``ffi.callback(..., onerror=func)``.  If an exception occurs and
+``onerror`` is specified, then ``onerror(exception, exc_value,
+traceback)`` is called.  This is useful in some situations where
+you cannot simply write ``try: except:`` in the main callback
+function, because it might not catch exceptions raised by signal
+handlers: if a signal occurs while in C, it will be called after
+entering the main callback function but before executing the
+``try:``.
 
-   If you want to be sure to catch all exceptions, use
-   ``ffi.callback(..., onerror=func)``.  If an exception occurs and
-   ``onerror`` is specified, then ``onerror(exception, exc_value,
-   traceback)`` is called.  This is useful in some situations where
-   you cannot simply write ``try: except:`` in the main callback
-   function, because it might not catch exceptions raised by signal
-   handlers: if a signal occurs while in C, it will be called after
-   entering the main callback function but before executing the
-   ``try:``.
+If ``onerror`` returns normally, then it is assumed that it handled
+the exception on its own and nothing is printed to stderr.  If
+``onerror`` raises, then both tracebacks are printed.  Finally,
+``onerror`` can itself provide the result value of the callback in
+C, but doesn't have to: if it simply returns None---or if
+``onerror`` itself fails---then the value of ``error`` will be
+used, if any.
 
-   If ``onerror`` returns normally, then it is assumed that it handled
-   the exception on its own and nothing is printed to stderr.  If
-   ``onerror`` raises, then both tracebacks are printed.  Finally,
-   ``onerror`` can itself provide the result value of the callback in
-   C, but doesn't have to: if it simply returns None---or if
-   ``onerror`` itself fails---then the value of ``error`` will be
-   used, if any.
-
-   Note the following hack: in ``onerror``, you can access the original
-   callback arguments as follows.  First check if ``traceback`` is not
-   None (it is None e.g. if the whole function ran successfully but
-   there was an error converting the value returned: this occurs after
-   the call).  If ``traceback`` is not None, then ``traceback.tb_frame``
-   is the frame of the outermost function, i.e. directly the one invoked
-   by the callback handler.  So you can get the value of ``argname`` in
-   that frame by reading ``traceback.tb_frame.f_locals['argname']``.
+Note the following hack: in ``onerror``, you can access the original
+callback arguments as follows.  First check if ``traceback`` is not
+None (it is None e.g. if the whole function ran successfully but
+there was an error converting the value returned: this occurs after
+the call).  If ``traceback`` is not None, then ``traceback.tb_frame``
+is the frame of the outermost function, i.e. directly the one invoked
+by the callback handler.  So you can get the value of ``argname`` in
+that frame by reading ``traceback.tb_frame.f_locals['argname']``.
 
 
 FFI Interface
@@ -580,12 +581,10 @@
 calls.  This function returns this error code as a tuple ``(code,
 message)``, adding a readable message like Python does when raising
 WindowsError.  If the argument ``code`` is given, format that code into
-a message instead of using ``GetLastError()``.  *New in version 0.8.*
+a message instead of using ``GetLastError()``.
 (Note that it is also possible to declare and call the ``GetLastError()``
 function as usual.)
 
-.. "versionadded:: 0.8" --- inlined in the previous paragraph
-
 **ffi.string(cdata, [maxlen])**: return a Python string (or unicode
 string) from the 'cdata'.
 
@@ -637,14 +636,10 @@
 ``cdata`` object: if it was originally an owning cdata, then its
 owned memory will not be freed as long as the buffer is alive.
 
-.. versionchanged:: 0.8.2
-   Before version 0.8.2, ``bytes(buf)`` was supported in Python 3 to get
-   the content of the buffer, but on Python 2 it would return the repr
-   ``<_cffi_backend.buffer object>``.  This has been fixed.  But you
-   should avoid using ``str(buf)``: it gives inconsistent results
-   between Python 2 and Python 3 (this is similar to how ``str()``
-   gives inconsistent results on regular byte strings).  Use ``buf[:]``
-   instead.
+Python 2/3 compatibility note: you should avoid using ``str(buf)``,
+because it gives inconsistent results between Python 2 and Python 3.
+This is similar to how ``str()`` gives inconsistent results on regular
+byte strings).  Use ``buf[:]`` instead.
 
 **ffi.from_buffer(python_buffer)**: return a ``<cdata 'char[]'>`` that
 points to the data of the given Python object, which must support the
@@ -659,8 +654,6 @@
 of memoryview, locked) as long as the cdata object returned by
 ``ffi.from_buffer()`` is alive.  *New in version 0.9.*
 
-.. "versionadded:: 0.9" --- inlined in the previous paragraph
-
 
 **ffi.typeof("C type" or cdata object)**: return an object of type
 ``<ctype>`` corresponding to the parsed string, or to the C type of the
@@ -711,11 +704,11 @@
 offset within the struct of the given field.  Corresponds to ``offsetof()``
 in C.
 
-.. versionchanged:: 0.9
-   You can give several field names in case of nested structures.  You
-   can also give numeric values which correspond to array items, in case
-   of a pointer or array type.  For example, ``ffi.offsetof("int[5]", 2)``
-   is equal to the size of two integers, as is ``ffi.offsetof("int *", 2)``.
+*New in version 0.9:*
+You can give several field names in case of nested structures.  You
+can also give numeric values which correspond to array items, in case
+of a pointer or array type.  For example, ``ffi.offsetof("int[5]", 2)``
+is equal to the size of two integers, as is ``ffi.offsetof("int *", 2)``.
 
    
 **ffi.getctype("C type" or <ctype>, extra="")**: return the string
@@ -828,8 +821,6 @@
 ``ffi.new()`` but uses the provided low-level ``alloc`` and ``free``
 functions.  *New in version 1.2.*
 
-.. "versionadded:: 1.2" --- inlined in the previous paragraph
-
 ``alloc()`` is invoked with the size as sole argument.  If it returns
 NULL, a MemoryError is raised.  Later, if ``free`` is not None, it will
 be called with the result of ``alloc()`` as argument.  Both can be either

From noreply at buildbot.pypy.org  Thu Jul  9 17:10:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 17:10:22 +0200 (CEST)
Subject: [pypy-commit] cffi default: Doc fixes
Message-ID: <20150709151022.EFD801C06B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2219:1ff67de0da1e
Date: 2015-07-09 16:45 +0200
http://bitbucket.org/cffi/cffi/changeset/1ff67de0da1e/

Log:	Doc fixes

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -239,7 +239,7 @@
 the limitations of ABI-level access to the system.  In case of doubt, read
 again `ABI versus API`_ in the overview.
 
-.. _`ABI versus API`: overflow.html#abi-versus-api
+.. _`ABI versus API`: overview.html#abi-versus-api
 
 You can use the library object to call the functions previously
 declared by ``ffi.cdef()``, to read constants, and to read or write
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -574,7 +574,8 @@
 confuse it with ``ffi.errno``.)
         
 **ffi.errno**: the value of ``errno`` received from the most recent C call
-in this thread, and passed to the following C call.  (This is a property.)
+in this thread, and passed to the following C call.  (This is a read-write
+property.)
 
 **ffi.getwinerror(code=-1)**: on Windows, in addition to ``errno`` we
 also save and restore the ``GetLastError()`` value across function

From noreply at buildbot.pypy.org  Thu Jul  9 17:25:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 17:25:37 +0200 (CEST)
Subject: [pypy-commit] cffi default: Tweaks
Message-ID: <20150709152537.AD2181C101F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2220:6f057232892d
Date: 2015-07-09 17:25 +0200
http://bitbucket.org/cffi/cffi/changeset/6f057232892d/

Log:	Tweaks

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -231,6 +231,8 @@
 inconsistently, not defined by default.)
 
 
+.. _loading-libraries:
+
 ffi.dlopen(): loading libraries in ABI mode
 -------------------------------------------
 
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,21 +53,19 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.2.0.tar.gz
 
-   - Or grab the most current version by following the instructions below.
-
    - MD5: ...
 
    - SHA: ...
 
-* Or get it from the `Bitbucket page`_:
+* Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
 
 * ``python setup.py install`` or ``python setup_base.py install``
   (should work out of the box on Linux or Windows; see below for
   `MacOS X`_ or `Windows 64`_.)
 
-* running the tests: ``py.test c/ _cffi1/ testing/`` (if you didn't
-  install cffi yet, you may need ``python setup_base.py build_ext -f
+* running the tests: ``py.test  c/  testing/`` (if you didn't
+  install cffi yet, you need first ``python setup_base.py build_ext -f
   -i``)
 
 .. _`Bitbucket page`: https://bitbucket.org/cffi/cffi
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -33,8 +33,8 @@
     ... """)                                  
     >>> C = ffi.dlopen(None)                     # loads the entire C namespace
     >>> arg = ffi.new("char[]", "world")         # equivalent to C code: char arg[] = "world";
-    >>> C.printf("hi there, %s!\n", arg)         # call printf
-    hi there, world!
+    >>> C.printf("hi there, %s.\n", arg)         # call printf
+    hi there, world.
     17                                           # this is the return value
     >>>
 

From noreply at buildbot.pypy.org  Thu Jul  9 18:13:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 18:13:49 +0200 (CEST)
Subject: [pypy-commit] cffi default: Point to the bitbucket wiki.
Message-ID: <20150709161349.37DED1C101F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2221:d8df9b34d5c5
Date: 2015-07-09 17:35 +0200
http://bitbucket.org/cffi/cffi/changeset/d8df9b34d5c5/

Log:	Point to the bitbucket wiki.

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -2,8 +2,8 @@
 ====
 
 Foreign Function Interface for Python calling C code.
-Please see the [Documentation](http://cffi.readthedocs.org/) or uncompiled
-in the doc/ subdirectory.
+Please see the [Documentation](http://bitbucket.org/cffi/cffi/wiki/)
+(sources in the doc/ directory of the main repo).
 
 Download
 --------
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -137,7 +137,8 @@
 ====
 
 Foreign Function Interface for Python calling C code.
-Please see the `Documentation <http://cffi.readthedocs.org/>`_.
+Please see the `Documentation <http://bitbucket.org/cffi/cffi/wiki/>`_
+(sources in the doc/ directory of the main repo).
 
 Contact
 -------
@@ -150,7 +151,7 @@
                      if cpython else {},
         zip_safe=False,
 
-        url='http://cffi.readthedocs.org',
+        url='http://bitbucket.org/cffi/cffi/wiki/',
         author='Armin Rigo, Maciej Fijalkowski',
         author_email='python-cffi at googlegroups.com',
 

From noreply at buildbot.pypy.org  Thu Jul  9 18:33:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 18:33:21 +0200 (CEST)
Subject: [pypy-commit] cffi default: Add "content" directives to the main
	pages
Message-ID: <20150709163321.BE8991C13EE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2222:06e330ded4d0
Date: 2015-07-09 18:34 +0200
http://bitbucket.org/cffi/cffi/changeset/06e330ded4d0/

Log:	Add "content" directives to the main pages

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -2,6 +2,8 @@
 Preparing and Distributing modules
 ======================================
 
+.. contents::
+
 There are three or four different ways to use CFFI in a project.
 In order of complexity:
 
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -2,6 +2,8 @@
 Overview
 =======================================================
 
+.. contents::
+
 CFFI can be used in one of four modes: "ABI" versus "API" level,
 each with "in-line" or "out-of-line" preparation (or compilation).
 
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -2,6 +2,8 @@
 Using the ffi/lib objects
 ================================
 
+.. contents::
+
 Keep this page under your pillow.
 
 

From noreply at buildbot.pypy.org  Thu Jul  9 18:54:17 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  9 Jul 2015 18:54:17 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: remove unused import
Message-ID: <20150709165417.D78161C06B9@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78511:9724e26ff3e0
Date: 2015-07-07 14:15 +0200
http://bitbucket.org/pypy/pypy/changeset/9724e26ff3e0/

Log:	remove unused import

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -1,5 +1,5 @@
 
-from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers
+from rpython.jit.metainterp.resoperation import ResOperation, OpHelpers
 from rpython.jit.metainterp.history import Const
 
 

From noreply at buildbot.pypy.org  Thu Jul  9 18:54:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  9 Jul 2015 18:54:19 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: improve NotVirtualStateInfo
Message-ID: <20150709165419.00E601C06B9@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78512:94c5660db711
Date: 2015-07-07 14:34 +0200
http://bitbucket.org/pypy/pypy/changeset/94c5660db711/

Log:	improve NotVirtualStateInfo

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -19,6 +19,9 @@
     def force_box(self, op, optforce):
         return op
 
+    def getconst(self):
+        raise Exception("not a constant")
+
     
 class PtrInfo(AbstractInfo):
     _attrs_ = ()
@@ -32,6 +35,9 @@
     def is_virtual(self):
         return False
 
+    def get_known_class(self, cpu):
+        return None
+
     def getnullness(self):
         if self.is_null():
             return INFO_NULL
@@ -408,6 +414,9 @@
     def __init__(self, const):
         self._const = const
 
+    def getconst(self):
+        return self._const
+
     def _get_info(self, descr, optheap):
         ref = self._const.getref_base()
         info = optheap.const_infos.get(ref, None)
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -269,6 +269,11 @@
     def make_bool(self):
         self.intersect(IntBound(0, 1))
 
+    def getconst(self):
+        if not self.is_constant():
+            raise Exception("not a constant")
+        return ConstInt(self.getint())
+
     def getnullness(self):
         if self.known_gt(IntBound(0, 0)) or \
            self.known_lt(IntBound(0, 0)):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -9,7 +9,8 @@
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.metainterp.compile import LoopCompileData
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
-     NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN
+     NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN, LEVEL_KNOWNCLASS,\
+     LEVEL_NONNULL
 from rpython.jit.codewriter import heaptracker
 
 class FakeOptimizer(object):
@@ -81,7 +82,8 @@
         """
         es, loop, preamble = self.optimize(loop)
         p0 = preamble.inputargs[0]
-        assert (heaptracker.adr2int(self.node_vtable_adr) ==
-                es.exported_infos[p0]._known_class.getint())
-
-
+        expected_class = heaptracker.adr2int(self.node_vtable_adr)
+        assert expected_class ==es.exported_infos[p0]._known_class.getint()
+        vs = es.virtual_state
+        assert vs.state[0].level == LEVEL_KNOWNCLASS
+        assert vs.state[0].known_class.getint() == expected_class
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -82,7 +82,7 @@
     def test_make_inputargs(self):
         optimizer = FakeOptimizer()
         args = [InputArgInt()]
-        info0 = NotVirtualStateInfo(args[0], None)
+        info0 = NotVirtualStateInfo(self.cpu, args[0].type, None)
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == args
         info0.level = LEVEL_CONSTANT
@@ -109,8 +109,8 @@
             assert info1 in state.bad and info2 in state.bad
 
         for BoxType in (InputArgInt, InputArgFloat, InputArgRef):
-            info1 = NotVirtualStateInfo(BoxType(), None)
-            info2 = NotVirtualStateInfo(BoxType(), None)
+            info1 = NotVirtualStateInfo(self.cpu, BoxType.type, None)
+            info2 = NotVirtualStateInfo(self.cpu, BoxType.type, None)
             postest(info1, info2)
 
         info1, info2 = VArrayStateInfo(42), VArrayStateInfo(42)
@@ -127,9 +127,9 @@
 
     def test_NotVirtualStateInfo_generalization(self):
         def isgeneral(tp1, info1, tp2, info2):
-            info1 = NotVirtualStateInfo(tp1, info1)
+            info1 = NotVirtualStateInfo(self.cpu, tp1, info1)
             info1.position = 0
-            info2 = NotVirtualStateInfo(tp2, info2)
+            info2 = NotVirtualStateInfo(self.cpu, tp2, info2)
             info2.position = 0
             return VirtualState([info1]).generalization_of(VirtualState([info2]), cpu=self.cpu)
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -282,14 +282,20 @@
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     lenbound = None
     intbound = None
+    level = LEVEL_UNKNOWN
+    constbox = None
+    known_class = None
     
-    def __init__(self, type, info):
+    def __init__(self, cpu, type, info):
         if info and info.is_constant():
+            self.constbox = info.getconst()
             self.level = LEVEL_CONSTANT
-        elif type == 'r' and info and info.is_nonnull():
-            self.level = LEVEL_NONNULL
-        else:
-            self.level = LEVEL_UNKNOWN
+        elif type == 'r' and info:
+            if info.get_known_class(cpu):
+                self.known_class = info.get_known_class(cpu)
+                self.level = LEVEL_KNOWNCLASS
+            elif info.is_nonnull():
+                self.level = LEVEL_NONNULL
         return
         yyy
         self.level = LEVEL_UNKNOWN
@@ -602,7 +608,8 @@
 
     def visit_not_virtual(self, box):
         is_opaque = box in self.optimizer.opaque_pointers
-        return NotVirtualStateInfo(box, self.optimizer.getinfo(box))
+        return NotVirtualStateInfo(self.optimizer.cpu, box.type,
+                                   self.optimizer.getinfo(box))
 
     def visit_virtual(self, known_class, fielddescrs):
         return VirtualStateInfo(known_class, fielddescrs)

From noreply at buildbot.pypy.org  Thu Jul  9 18:54:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  9 Jul 2015 18:54:20 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start writing enum_forced_boxes
Message-ID: <20150709165420.2F3631C06B9@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78513:7e74715499f8
Date: 2015-07-09 18:54 +0200
http://bitbucket.org/pypy/pypy/changeset/7e74715499f8/

Log:	start writing enum_forced_boxes

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -77,6 +77,11 @@
         self.last_guard_pos = len(optimizer._newoperations) - 1
         assert self.get_last_guard(optimizer).is_guard()
 
+    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+        if visitor.already_seen_virtual(instbox):
+            return
+        return self._visitor_walk_recursive(instbox, visitor, optimizer)
+
 class AbstractVirtualPtrInfo(NonNullPtrInfo):
     _attrs_ = ('_cached_vinfo', 'vdescr')
     # XXX merge _cached_vinfo with vdescr
@@ -141,9 +146,7 @@
                 if optforce.optheap is not None:
                     optforce.optheap.register_dirty_field(flddescr, self)
 
-    def visitor_walk_recursive(self, instbox, visitor, optimizer):
-        if visitor.already_seen_virtual(instbox):
-            return
+    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         lst = self.vdescr.get_all_fielddescrs()
         assert self.is_virtual()
         visitor.register_virtual_fields(instbox,
@@ -174,6 +177,7 @@
         assert self.is_virtual()
         return visitor.visit_virtual(self.vdescr, fielddescrs)
 
+
 class StructPtrInfo(AbstractStructPtrInfo):
     def __init__(self, vdescr=None):
         self.vdescr = vdescr
@@ -185,7 +189,7 @@
         return visitor.visit_vstruct(self.vdescr, fielddescrs)
 
 class AbstractRawPtrInfo(AbstractVirtualPtrInfo):
-    def visitor_walk_recursive(self, op, visitor, optimizer):
+    def _visitor_walk_recursive(self, op, visitor, optimizer):
         raise NotImplementedError("abstract")
 
     @specialize.argtype(1)
@@ -228,7 +232,7 @@
                               [op, ConstInt(offset), itembox], descr=descr)
             optforce.emit_operation(op)
 
-    def visitor_walk_recursive(self, op, visitor, optimizer):
+    def _visitor_walk_recursive(self, op, visitor, optimizer):
         itemboxes = self.buffer.values
         visitor.register_virtual_fields(op, itemboxes)
         # there can be no virtuals stored in raw buffer
@@ -263,7 +267,7 @@
     def _force_elements(self, op, optforce, descr):
         raise Exception("implement me")
 
-    def visitor_walk_recursive(self, op, visitor, optimizer):
+    def _visitor_walk_recursive(self, op, visitor, optimizer):
         source_op = optimizer.get_box_replacement(op.getarg(0))
         visitor.register_virtual_fields(op, [source_op])
         self.parent.visitor_walk_recursive(source_op, visitor, optimizer)
@@ -331,7 +335,7 @@
     def getlength(self):
         return self.length
 
-    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         itemops = [optimizer.get_box_replacement(item)
                    for item in self._items]
         visitor.register_virtual_fields(instbox, itemops)
@@ -385,7 +389,7 @@
                     # if it does, we would need a fix here
                 i += 1
 
-    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         itemops = [optimizer.get_box_replacement(item)
                    for item in self._items]
         visitor.register_virtual_fields(instbox, itemops)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -2,19 +2,29 @@
 """ More direct tests for unrolling
 """
 
+import py
+
 from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest,\
      LLtypeMixin
 from rpython.jit.metainterp.history import (TreeLoop, ConstInt,
                                             JitCellToken, TargetToken)
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+     InputArgRef
 from rpython.jit.metainterp.compile import LoopCompileData
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN, LEVEL_KNOWNCLASS,\
-     LEVEL_NONNULL
+     VirtualStateInfo, BadVirtualState
+from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.codewriter import heaptracker
 
 class FakeOptimizer(object):
     optearlyforce = None
+
+    def getptrinfo(self, box):
+        return box.get_forwarded()
+
+    def get_box_replacement(self, box):
+        return box
      
 class BaseTestUnroll(BaseTest, LLtypeMixin):
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
@@ -83,7 +93,33 @@
         es, loop, preamble = self.optimize(loop)
         p0 = preamble.inputargs[0]
         expected_class = heaptracker.adr2int(self.node_vtable_adr)
-        assert expected_class ==es.exported_infos[p0]._known_class.getint()
+        assert expected_class == es.exported_infos[p0]._known_class.getint()
         vs = es.virtual_state
         assert vs.state[0].level == LEVEL_KNOWNCLASS
         assert vs.state[0].known_class.getint() == expected_class
+
+    def test_virtual(self):
+        loop = """
+        [p1, p2]
+        p0 = new_with_vtable(descr=nodesize)
+        setfield_gc(p0, 1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=nextdescr)
+        jump(p0, p0)
+        """
+        es, loop, preamble = self.optimize(loop)
+        vs = es.virtual_state
+        assert vs.state[0] is vs.state[1]
+        assert isinstance(vs.state[0], VirtualStateInfo)
+        assert isinstance(vs.state[0].fieldstate[0], NotVirtualStateInfo)
+        assert vs.state[0].fieldstate[0].level == LEVEL_CONSTANT
+        assert isinstance(vs.state[0].fieldstate[3], NotVirtualStateInfo)
+        assert vs.state[0].fieldstate[3].level == LEVEL_UNKNOWN
+        assert vs.numnotvirtuals == 1
+        p = InputArgRef()
+        py.test.raises(BadVirtualState, vs.make_inputargs, [p, p],
+                       FakeOptimizer())
+        ptrinfo = info.StructPtrInfo(self.nodesize)
+        p2 = InputArgRef()
+        ptrinfo._fields = [None, None, None, p2]
+        p.set_forwarded(ptrinfo)
+        vs.make_inputargs([p, p], FakeOptimizer())
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -10,7 +10,8 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \
                                                            equaloplists
-from rpython.jit.metainterp.optimizeopt.intutils import IntBound, ConstIntBound
+from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
+     ConstIntBound, IntLowerBound, IntUpperBound, IntUnbounded
 from rpython.jit.metainterp.history import TreeLoop, JitCellToken
 from rpython.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeMetaInterpStaticData
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
@@ -86,6 +87,7 @@
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == args
         info0.level = LEVEL_CONSTANT
+        vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == []
 
     def test_position_generalization(self):
@@ -148,11 +150,10 @@
                 if i != j:
                     assert not isgeneral('r', inorder[j], 'r', inorder[i])
 
-        value1 = IntOptValue(BoxInt())
-        value2 = IntOptValue(BoxInt())
-        value2.intbound.make_lt(IntBound(10, 10))
-        assert isgeneral(value1, value2)
-        assert not isgeneral(value2, value1)
+        i1 = IntLowerBound(10)
+        i2 = IntUnbounded()
+        assert isgeneral('i', i1, 'i', i2)
+        assert not isgeneral('i', i2, 'i', i1)
 
         assert isgeneral(OptValue(ConstInt(7)), OptValue(ConstInt(7)))
         S = lltype.GcStruct('S')
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -1,7 +1,6 @@
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
 from rpython.jit.metainterp.history import (ConstInt, Const,
         ConstPtr, ConstFloat)
-from rpython.jit.metainterp.optimizeopt import virtualize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractInputArg
@@ -133,23 +132,23 @@
     def _generalization_of_structpart(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value, optimizer):
-        if not isinstance(value, virtualize.AbstractVirtualStructValue):
-            raise BadVirtualState
-        if not value.is_virtual():
-            raise BadVirtualState
+    def enum_forced_boxes(self, boxes, box, optimizer):
+        info = optimizer.getptrinfo(box)
+        box = optimizer.get_box_replacement(box)
+        if info is None or not info.is_virtual():
+            raise BadVirtualState()
         for i in range(len(self.fielddescrs)):
-            try:
-                v = value._fields[self.fielddescrs[i]]
-            except KeyError:
-                raise BadVirtualState
-            s = self.fieldstate[i]
-            if s.position > self.position:
-                s.enum_forced_boxes(boxes, v, optimizer)
+            state = self.fieldstate[i]
+            if not state:
+                continue
+            if state.position > self.position:
+                fieldbox = info._fields[i]
+                state.enum_forced_boxes(boxes, fieldbox, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
-            s.enum(virtual_state)
+            if s:
+                s.enum(virtual_state)
 
 
 class VirtualStateInfo(AbstractVirtualStructStateInfo):
@@ -200,6 +199,7 @@
                                                v, state)
 
     def enum_forced_boxes(self, boxes, value, optimizer):
+        xxx
         if not isinstance(value, virtualize.VArrayValue):
             raise BadVirtualState
         if not value.is_virtual():
@@ -258,6 +258,7 @@
             s.enum(virtual_state)
 
     def enum_forced_boxes(self, boxes, value, optimizer):
+        xxx
         if not isinstance(value, virtualize.VArrayStructValue):
             raise BadVirtualState
         if not value.is_virtual():
@@ -422,16 +423,18 @@
             return
         raise VirtualStatesCantMatch("intbounds don't match")
 
-    def enum_forced_boxes(self, boxes, value, optimizer):
+    def enum_forced_boxes(self, boxes, box, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals
-        if optimizer:
-            box = value.force_box(optimizer)
-        else:
-            if value.is_virtual():
+        #if optimizer:
+        #    box = value.force_box(optimizer)
+        #else:
+        box = optimizer.get_box_replacement(box)
+        if box.type == 'r':
+            info = optimizer.getptrinfo(box)
+            if info and info.is_virtual():
                 raise BadVirtualState
-            box = value.get_key_box()
         boxes[self.position_in_notvirtuals] = box
 
     def _enum(self, virtual_state):
@@ -505,34 +508,35 @@
         return state
 
     def make_inputargs(self, inputargs, optimizer, keyboxes=False):
-        if optimizer.optearlyforce:
-            optimizer = optimizer.optearlyforce
         assert len(inputargs) == len(self.state)
-        inpargs = []
-        for i, state in enumerate(self.state):
-            if state.level != LEVEL_CONSTANT:
-                inpargs.append(inputargs[i])
-        return inpargs
-        inputargs = [None] * self.numnotvirtuals
+        #inpargs = []
+        #for i, state in enumerate(self.state):
+        #    if not isinstance(state, NotVirtualStateInfo) or state.level != LEVEL_CONSTANT:
+        #        inpargs.append(inputargs[i])
+        #return inpargs
+        boxes = [None] * self.numnotvirtuals
 
+        # XXX no longer correct as far as I can tell, maybe we should
+        #     make the forcing more explicit somewhere else
         # We try twice. The first time around we allow boxes to be forced
         # which might change the virtual state if the box appear in more
         # than one place among the inputargs.
-        for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
-        for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i], None)
+        for i in range(len(inputargs)):
+            self.state[i].enum_forced_boxes(boxes, inputargs[i], optimizer)
+        #for i in range(len(values)):
+        #    self.state[i].enum_forced_boxes(inputargs, values[i], None)
 
-        if keyboxes:
-            for i in range(len(values)):
-                if not isinstance(self.state[i], NotVirtualStateInfo):
-                    box = values[i].get_key_box()
-                    assert not isinstance(box, Const)
-                    inputargs.append(box)
+        # not sure what are these guys doing
+        #if keyboxes:
+        #    for i in range(len(values)):
+        #        if not isinstance(self.state[i], NotVirtualStateInfo):
+        #            box = values[i].get_key_box()
+        #            assert not isinstance(box, Const)
+        #            inputargs.append(box)
 
-        assert None not in inputargs
+        assert None not in boxes
 
-        return inputargs
+        return boxes
 
     def debug_print(self, hdr='', bad=None, metainterp_sd=None):
         if bad is None:
@@ -556,54 +560,63 @@
     def already_seen_virtual(self, keybox):
         return keybox in self.fieldboxes
 
-    def getvalue(self, box):
-        return self.optimizer.getvalue(box)
+    #def state(self, box):
+    #    xxx
+    #    value = self.getvalue(box)
+    #    box = value.get_key_box()
+    #    try:
+    #        info = self.info[box]
+    #    except KeyError:
+    #        self.info[box] = info = value.visitor_dispatch_virtual_type(self)
+    #        if value.is_virtual():
+    #            flds = self.fieldboxes[box]
+    #            info.fieldstate = [self.state_or_none(b, value) for b in flds]
+    #    return info
 
-    def state(self, box):
+    #def state_or_none(self, box, value):
+    #    if box is None:
+    #        box = value.get_missing_null_value().box
+    #    return self.state(box)
+
+    def create_state_or_none(self, box, opt):
+        if box is None:
+            return None
+        return self.create_state(box, opt)
+
+    def create_state(self, box, opt):
+        box = opt.get_box_replacement(box)
+        try:
+            return self.info[box]
+        except KeyError:
+            pass
         if box.type == 'r':
-            xxxx
-        return None
-        value = self.getvalue(box)
-        box = value.get_key_box()
-        try:
-            info = self.info[box]
-        except KeyError:
-            self.info[box] = info = value.visitor_dispatch_virtual_type(self)
-            if value.is_virtual():
-                flds = self.fieldboxes[box]
-                info.fieldstate = [self.state_or_none(b, value) for b in flds]
-        return info
-
-    def state_or_none(self, box, value):
-        if box is None:
-            box = value.get_missing_null_value().box
-        return self.state(box)
+            info = opt.getptrinfo(box)
+            if info is not None and info.is_virtual():
+                result = info.visitor_dispatch_virtual_type(self)
+                self.info[box] = result
+                info.visitor_walk_recursive(box, self, opt)
+                result.fieldstate = [self.create_state_or_none(b, opt)
+                                     for b in self.fieldboxes[box]]
+            else:
+                result = self.visit_not_virtual(box)
+                self.info[box] = result
+        elif box.type == 'i' or box.type == 'f':
+            result = self.visit_not_virtual(box)
+            self.info[box] = result
+        else:
+            assert False
+        return result
 
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
-        already_forced = {}
         if self.optimizer.optearlyforce:
             opt = self.optimizer.optearlyforce
         else:
             opt = self.optimizer
         state = []
+        self.info = {}
         for box in jump_args:
-            box = opt.get_box_replacement(box)
-            if box.type == 'r':
-                info = opt.getptrinfo(box)
-                if info is not None and info.is_virtual():
-                    xxx
-                else:
-                    state.append(self.visit_not_virtual(box))
-            elif box.type == 'i':
-                intbound = opt.getintbound(box)
-                state.append(self.visit_not_virtual(box))
-            else:
-                xxx
-        #values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
-        #                                                      opt)
-        #          for box in jump_args]
-
+            state.append(self.create_state(box, opt))
         return VirtualState(state)
 
     def visit_not_virtual(self, box):
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -204,7 +204,7 @@
             offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
-    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         visitor.register_virtual_fields(instbox, self._chars)
 
     @specialize.argtype(1)
@@ -248,7 +248,7 @@
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         return self.lgtop
 
-    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         boxes = [self.s, self.start, self.lgtop]
         visitor.register_virtual_fields(instbox, boxes)
         opinfo = optimizer.getptrinfo(self.s)
@@ -309,7 +309,7 @@
                                              targetbox, offsetbox, mode)
         return offsetbox
 
-    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         # we don't store the lengthvalue in guards, because the
         # guard-failed code starts with a regular STR_CONCAT again
         leftbox = self.vleft

From noreply at buildbot.pypy.org  Thu Jul  9 20:08:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 20:08:28 +0200 (CEST)
Subject: [pypy-commit] cffi default: Silence a gcc warning
Message-ID: <20150709180828.565561C048F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2223:d0e1c599a102
Date: 2015-07-09 19:48 +0200
http://bitbucket.org/cffi/cffi/changeset/d0e1c599a102/

Log:	Silence a gcc warning

diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -397,8 +397,9 @@
         PyTuple_SET_ITEM(allocator, 2, my_free);
     }
     if (!should_clear_after_alloc) {
-        Py_INCREF(Py_True);
-        PyTuple_SET_ITEM(allocator, 3, Py_True);  /* dont_clear_after_alloc */
+        PyObject *my_true = Py_True;
+        Py_INCREF(my_true);
+        PyTuple_SET_ITEM(allocator, 3, my_true);  /* dont_clear_after_alloc */
     }
 
     {

From noreply at buildbot.pypy.org  Thu Jul  9 20:08:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 20:08:29 +0200 (CEST)
Subject: [pypy-commit] cffi default: In non-C++ mode,
 we need a warning to be turned into an error
Message-ID: <20150709180829.6FDB81C06B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2224:f93c1bd0d9a0
Date: 2015-07-09 20:08 +0200
http://bitbucket.org/cffi/cffi/changeset/f93c1bd0d9a0/

Log:	In non-C++ mode, we need a warning to be turned into an error

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -24,6 +24,9 @@
     if 1:     # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
+    else:
+        kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
+                                      ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
 

From noreply at buildbot.pypy.org  Thu Jul  9 20:29:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Jul 2015 20:29:52 +0200 (CEST)
Subject: [pypy-commit] cffi default: backout d8df9b34d5c and keep using
 readthedocs as the default doc
Message-ID: <20150709182952.9FAD41C120E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2225:6df6d705bac2
Date: 2015-07-09 20:30 +0200
http://bitbucket.org/cffi/cffi/changeset/6df6d705bac2/

Log:	backout d8df9b34d5c and keep using readthedocs as the default doc

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -2,8 +2,8 @@
 ====
 
 Foreign Function Interface for Python calling C code.
-Please see the [Documentation](http://bitbucket.org/cffi/cffi/wiki/)
-(sources in the doc/ directory of the main repo).
+Please see the [Documentation](http://cffi.readthedocs.org/) or uncompiled
+in the doc/ subdirectory.
 
 Download
 --------
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -137,8 +137,7 @@
 ====
 
 Foreign Function Interface for Python calling C code.
-Please see the `Documentation <http://bitbucket.org/cffi/cffi/wiki/>`_
-(sources in the doc/ directory of the main repo).
+Please see the `Documentation <http://cffi.readthedocs.org/>`_.
 
 Contact
 -------
@@ -151,7 +150,7 @@
                      if cpython else {},
         zip_safe=False,
 
-        url='http://bitbucket.org/cffi/cffi/wiki/',
+        url='http://cffi.readthedocs.org',
         author='Armin Rigo, Maciej Fijalkowski',
         author_email='python-cffi at googlegroups.com',
 

From noreply at buildbot.pypy.org  Thu Jul  9 22:48:23 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  9 Jul 2015 22:48:23 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: invent new names for newXXX,
 so we don't run into clashes when unrolling
Message-ID: <20150709204823.F1BA01C06B9@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78514:dc4e5a98335d
Date: 2015-07-09 22:48 +0200
http://bitbucket.org/pypy/pypy/changeset/dc4e5a98335d/

Log:	invent new names for newXXX, so we don't run into clashes when
	unrolling

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -258,15 +258,14 @@
                 self.pure_from_args(rop.INT_SUB, [args[0], result], args[1])
             #elif opnum == rop.INT_MUL_OVF:
             #    self.pure(rop.INT_MUL, args[:], result)
-        self.emit_operation(op)
+            self.emit_operation(op)
 
     def optimize_GUARD_OVERFLOW(self, op):
         # If INT_xxx_OVF was replaced by INT_xxx, *but* we still see
         # GUARD_OVERFLOW, then the loop is invalid.
         lastop = self.last_emitted_operation
         if lastop is None:
-            raise InvalidLoop('An INT_xxx_OVF was proven not to overflow but' +
-                              'guarded with GUARD_OVERFLOW')
+            return # e.g. beginning of the loop
         opnum = lastop.getopnum()
         if opnum not in (rop.INT_ADD_OVF, rop.INT_SUB_OVF, rop.INT_MUL_OVF):
             raise InvalidLoop('An INT_xxx_OVF was proven not to overflow but' +
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -803,7 +803,7 @@
         escape_n(i3)
         p1 = new_with_vtable(descr=nodesize)
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
+        p1sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -821,9 +821,9 @@
         i3 = getfield_gc_i(p2, descr=valuedescr)
         escape_n(i3)
         p4 = new_with_vtable(descr=nodesize)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
+        setfield_gc(p4, i1, descr=valuedescr)
+        p1sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p1sub, i1, descr=valuedescr)
-        setfield_gc(p4, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
         """
@@ -836,7 +836,7 @@
         i3 = getfield_gc_i(p3sub, descr=valuedescr)
         escape_n(i3)
         p1 = new_with_vtable(descr=nodesize)
-        p2sub = new_with_vtable(ConstClass(node_vtable2))
+        p2sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -848,7 +848,7 @@
         p3sub = getfield_gc_r(p3, descr=nextdescr)
         i3 = getfield_gc_i(p3sub, descr=valuedescr)
         escape_n(i3)
-        p2sub = new_with_vtable(ConstClass(node_vtable2))
+        p2sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
         jump(i1, p2, p2sub)
@@ -858,7 +858,7 @@
         i3 = getfield_gc_i(p2sub, descr=valuedescr)
         escape_n(i3)
         p1 = new_with_vtable(descr=nodesize)
-        p3sub = new_with_vtable(ConstClass(node_vtable2))
+        p3sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
         jump(i1, p1, p3sub)
@@ -8093,7 +8093,7 @@
         """
         self.optimize_loop(ops, expected)
 
-    def test_exploding_duplicatipon(self):
+    def test_exploding_duplication(self):
         ops = """
         [i1, i2]
         i3 = int_add(i1, i1)
@@ -8342,13 +8342,13 @@
         ops = """
         [i0, i1]
         p0 = escape_r()
-        setfield_gc(p0, p0, descr=arraydescr)
+        setfield_gc(p0, p0, descr=nextdescr)
         jump(i0, i1)
         """
         expected = """
         [i0, i1]
         p0 = escape_r()
-        setfield_gc(p0, p0, descr=arraydescr)
+        setfield_gc(p0, p0, descr=nextdescr)
         jump(i0, i1)
         """
         self.optimize_loop(ops, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -85,7 +85,8 @@
         jump_args = [self.get_box_replacement(op)
                      for op in end_jump.getarglist()]
         jump_args = state.virtual_state.make_inputargs(jump_args,
-                                                       self.optimizer)
+                                                       self.optimizer,
+                                                       force_boxes=True)
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer._newoperations.append(jump_op)
         return (UnrollInfo(self.make_short_preamble(start_label.getarglist())),
@@ -188,7 +189,6 @@
         assert isinstance(start_target, TargetToken)
         return stop_target.targeting_jitcell_token is start_target.targeting_jitcell_token
 
-
     def export_state(self, start_label, end_label, renamed_inputargs):
         original_label_args = end_label.getarglist()
         end_args = [self.get_box_replacement(a) for a in original_label_args]
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -18,7 +18,8 @@
     def make_virtual(self, known_class, source_op, descr):
         opinfo = info.InstancePtrInfo(known_class, vdescr=descr)
         opinfo.init_fields(descr, 0)
-        source_op.set_forwarded(opinfo)
+        newop = self.replace_op_with(source_op, source_op.getopnum())
+        newop.set_forwarded(opinfo)
         return opinfo
 
     def make_varray(self, arraydescr, size, source_op, clear=False):
@@ -28,23 +29,27 @@
         else:
             const = self.new_const_item(arraydescr)
             opinfo = info.ArrayPtrInfo(const, size, clear, vdescr=arraydescr)
-        source_op.set_forwarded(opinfo)
+        newop = self.replace_op_with(source_op, source_op.getopnum())
+        newop.set_forwarded(opinfo)
         return opinfo
 
     def make_vstruct(self, structdescr, source_op):
         opinfo = info.StructPtrInfo(vdescr=structdescr)
         opinfo.init_fields(structdescr, 0)
-        source_op.set_forwarded(opinfo)
+        newop = self.replace_op_with(source_op, source_op.getopnum())
+        newop.set_forwarded(opinfo)
         return opinfo
 
     def make_virtual_raw_memory(self, size, source_op):
         opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, size)
-        source_op.set_forwarded(opinfo)
+        newop = self.replace_op_with(source_op, source_op.getopnum())
+        newop.set_forwarded(opinfo)
         return opinfo
 
     def make_virtual_raw_slice(self, offset, parent, source_op):
         opinfo = info.RawSlicePtrInfo(offset, parent)
-        source_op.set_forwarded(opinfo)
+        newop = self.replace_op_with(source_op, source_op.getopnum())
+        newop.set_forwarded(opinfo)
         return opinfo
 
     def optimize_GUARD_NO_EXCEPTION(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -132,7 +132,7 @@
     def _generalization_of_structpart(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, box, optimizer):
+    def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         info = optimizer.getptrinfo(box)
         box = optimizer.get_box_replacement(box)
         if info is None or not info.is_virtual():
@@ -143,7 +143,7 @@
                 continue
             if state.position > self.position:
                 fieldbox = info._fields[i]
-                state.enum_forced_boxes(boxes, fieldbox, optimizer)
+                state.enum_forced_boxes(boxes, fieldbox, optimizer, force_boxes)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -423,18 +423,18 @@
             return
         raise VirtualStatesCantMatch("intbounds don't match")
 
-    def enum_forced_boxes(self, boxes, box, optimizer):
+    def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals
-        #if optimizer:
-        #    box = value.force_box(optimizer)
-        #else:
         box = optimizer.get_box_replacement(box)
         if box.type == 'r':
             info = optimizer.getptrinfo(box)
             if info and info.is_virtual():
-                raise BadVirtualState
+                if force_boxes:
+                    info.force_box(box, optimizer)
+                else:
+                    raise BadVirtualState
         boxes[self.position_in_notvirtuals] = box
 
     def _enum(self, virtual_state):
@@ -507,7 +507,8 @@
                                           state)
         return state
 
-    def make_inputargs(self, inputargs, optimizer, keyboxes=False):
+    def make_inputargs(self, inputargs, optimizer, keyboxes=False,
+                       force_boxes=False):
         assert len(inputargs) == len(self.state)
         #inpargs = []
         #for i, state in enumerate(self.state):
@@ -516,11 +517,13 @@
         #return inpargs
         boxes = [None] * self.numnotvirtuals
 
-        # XXX no longer correct as far as I can tell, maybe we should
-        #     make the forcing more explicit somewhere else
         # We try twice. The first time around we allow boxes to be forced
         # which might change the virtual state if the box appear in more
         # than one place among the inputargs.
+        if force_boxes:
+            for i in range(len(inputargs)):
+                self.state[i].enum_forced_boxes(boxes, inputargs[i], optimizer,
+                                                True)
         for i in range(len(inputargs)):
             self.state[i].enum_forced_boxes(boxes, inputargs[i], optimizer)
         #for i in range(len(values)):

From noreply at buildbot.pypy.org  Fri Jul 10 09:10:11 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 10 Jul 2015 09:10:11 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: fix maxalign calclulation,
	apply to offsets
Message-ID: <20150710071011.09FE41C053A@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78516:34d98db0437f
Date: 2015-07-02 22:53 -0400
http://bitbucket.org/pypy/pypy/changeset/34d98db0437f/

Log:	fix maxalign calclulation, apply to offsets

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -604,8 +604,10 @@
         fields[fldname] = offsets[i], subdtype
         if title is not None:
             fields[title] = offsets[i], subdtype
-        maxalign = max(subdtype.elsize, maxalign)
-        delta = subdtype.elsize
+        maxalign = max(subdtype.alignment, maxalign)
+        if not subdtype.is_record():
+            maxalign = max(subdtype.elsize, maxalign)
+        delta = subdtype.alignment
         if  i + 1 < len(offsets) and offsets[i + 1] == 0:
             if alignment >= 0:
                 # Set offset to the next power-of-two above delta
@@ -613,15 +615,12 @@
                 if delta > offsets[i]:
                     for j in range(i):
                         offsets[j+1] = delta + offsets[j]
-            offsets[i + 1] = offsets[i] + delta
-        print maxalign, delta, subdtype, subdtype.alignment, alignment
+            offsets[i + 1] = offsets[i] + max(delta, subdtype.elsize)
         names.append((fldname, title))
-        if alignment >= 0:
-            total = len(offsets) * maxalign
-        else:
-            total += subdtype.elsize
+    total = offsets[-1] + max(maxalign, fields[names[-1][0]][1].elsize)
     retval = W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
                    names=names, fields=fields, elsize=total)
+    retval.alignment = maxalign
     retval.flags |= NPY.NEEDS_PYAPI
     return retval
 
@@ -722,8 +721,11 @@
             'numpy.core._internal', '_commastring', Arguments(space, [w_spec]))
     else:
         # handle only simple cases for testing
-        spec = [s.strip() for s in space.str_w(w_spec).split(',')]
-        w_lst = space.newlist([space.wrap(s) for s in spec]) 
+        if space.isinstance_w(w_spec, space.w_str):
+            spec = [s.strip() for s in space.str_w(w_spec).split(',')]
+            w_lst = space.newlist([space.wrap(s) for s in spec]) 
+        elif space.isinstance_w(w_spec, space.w_list):
+            w_lst = w_spec
     if not space.isinstance_w(w_lst, space.w_list) or space.len_w(w_lst) < 1:
         raise oefmt(space.w_RuntimeError,
                     "_commastring is not returning a list with len >= 1")
@@ -835,7 +837,7 @@
     elif space.isinstance_w(w_dtype, space.w_tuple):
         w_dtype0 = space.getitem(w_dtype, space.wrap(0))
         w_dtype1 = space.getitem(w_dtype, space.wrap(1))
-        subdtype = make_new_dtype(space, w_subtype, w_dtype0, align, copy)
+        subdtype = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy)
         assert isinstance(subdtype, W_Dtype)
         if subdtype.elsize == 0:
             name = "%s%d" % (subdtype.kind, space.int_w(w_dtype1))

From noreply at buildbot.pypy.org  Fri Jul 10 09:10:09 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 10 Jul 2015 09:10:09 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: use integer alignment
 rather than boolean align
Message-ID: <20150710071009.C7E5E1C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78515:622b559dd108
Date: 2015-07-02 22:15 -0400
http://bitbucket.org/pypy/pypy/changeset/622b559dd108/

Log:	use integer alignment rather than boolean align

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -561,7 +561,7 @@
 
 
 @specialize.arg(2)
-def dtype_from_list(space, w_lst, simple, align=False, offsets=None):
+def dtype_from_list(space, w_lst, simple, alignment, offsets=None):
     lst_w = space.listview(w_lst)
     fields = {}
     if offsets is None:
@@ -573,8 +573,8 @@
         w_elem = lst_w[i]
         title = None
         if simple:
-            subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_elem,
-                                    align=align)
+            subdtype = make_new_dtype(space, space.gettypefor(W_Dtype), w_elem,
+                                    alignment)
             fldname = 'f%d' % i
         else:
             w_shape = space.newtuple([])
@@ -584,8 +584,8 @@
                     w_shape = space.newtuple([w_shape])
             else:
                 w_fldname, w_flddesc = space.fixedview(w_elem, 2)
-            subdtype = descr__new__(space, space.gettypefor(W_Dtype),
-                                    w_flddesc, w_shape=w_shape, align=align)
+            subdtype = make_new_dtype(space, space.gettypefor(W_Dtype),
+                                    w_flddesc, alignment, w_shape=w_shape)
             if space.isinstance_w(w_fldname, space.w_tuple):
                 fldlist = space.listview(w_fldname)
                 fldname = space.str_w(fldlist[0])
@@ -605,21 +605,21 @@
         if title is not None:
             fields[title] = offsets[i], subdtype
         maxalign = max(subdtype.elsize, maxalign)
+        delta = subdtype.elsize
         if  i + 1 < len(offsets) and offsets[i + 1] == 0:
-            delta = subdtype.elsize
-            if align:
+            if alignment >= 0:
                 # Set offset to the next power-of-two above delta
                 delta = (delta + maxalign -1) & (-maxalign)
                 if delta > offsets[i]:
                     for j in range(i):
                         offsets[j+1] = delta + offsets[j]
             offsets[i + 1] = offsets[i] + delta
-            print maxalign, delta, subdtype, subdtype.elsize
-        if align:
+        print maxalign, delta, subdtype, subdtype.alignment, alignment
+        names.append((fldname, title))
+        if alignment >= 0:
             total = len(offsets) * maxalign
         else:
             total += subdtype.elsize
-        names.append((fldname, title))
     retval = W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
                    names=names, fields=fields, elsize=total)
     retval.flags |= NPY.NEEDS_PYAPI
@@ -648,7 +648,7 @@
     for fname in w_dict.iterkeys().iterator:
         obj = _get_list_or_none(space, w_dict, fname)
         num = space.int_w(obj[1])
-        format = dtype_from_spec(space, obj[0], align=align)
+        format = dtype_from_spec(space, obj[0], alignment=-1)
         if len(obj) > 2:
             title = obj[2]
         else:
@@ -660,11 +660,15 @@
     offsets = [x[2] for x in allfields]
     titles  = [x[3] for x in allfields]
     aslist = []
+    if align:
+        alignment = 0
+    else:
+        alignment = -1
     for i in range(len(names)):
         aslist.append(space.newtuple([space.wrap(names[i]), space.wrap(formats[i])]))
-    return dtype_from_list(space, space.newlist(aslist), False, align=align, offsets=offsets)
+    return dtype_from_list(space, space.newlist(aslist), False, alignment, offsets=offsets)
     
-def dtype_from_dict(space, w_dict, align):
+def dtype_from_dict(space, w_dict, alignment):
     from pypy.objspace.std.dictmultiobject import W_DictMultiObject
     assert isinstance(w_dict, W_DictMultiObject)
     names_w = _get_list_or_none(space, w_dict, 'names')
@@ -676,18 +680,20 @@
     if names_w is None or formats_w is None:
         if we_are_translated():
             return get_appbridge_cache(space).call_method(space,
-                'numpy.core._internal', '_usefields', Arguments(space, [w_dict, space.wrap(align)]))
+                'numpy.core._internal', '_usefields', Arguments(space, 
+                                [w_dict, space.wrap(alignment >= 0)]))
         else:
-            return _usefields(space, w_dict, align)
+            return _usefields(space, w_dict, alignment >= 0)
     n = len(names_w)
     if (n != len(formats_w) or 
         (offsets_w is not None and n != len(offsets_w)) or
         (titles_w is not None and n != len(titles_w))):
         raise oefmt(space.w_ValueError, "'names', 'formats', 'offsets', and "
-            "'titles' dicct entries must have the same length") 
+            "'titles' dicct entries must have the same length")
     if aligned_w is not None:
         if space.isinstance_w(aligned_w, space.w_bool) and space.is_true(aligned_w):
-            align = True
+            if alignment < 0:
+                alignment = 0 
         else:
             raise oefmt(space.w_ValueError,
                     "NumPy dtype descriptor includes 'aligned' entry, "
@@ -703,13 +709,13 @@
     aslist = []
     for i in range(min(len(names_w), len(formats_w))):
         aslist.append(space.newtuple([names_w[i], formats_w[i]]))
-    retval = dtype_from_list(space, space.newlist(aslist), False, align=align, offsets=offsets)
+    retval = dtype_from_list(space, space.newlist(aslist), False, alignment, offsets=offsets)
     if metadata_w is not None:
         retval.descr_set_metadata(space, metadata_w)
     retval.flags |= NPY.NEEDS_PYAPI
     return retval 
 
-def dtype_from_spec(space, w_spec, align):
+def dtype_from_spec(space, w_spec, alignment):
 
     if we_are_translated():
         w_lst = get_appbridge_cache(space).call_method(space,
@@ -723,9 +729,9 @@
                     "_commastring is not returning a list with len >= 1")
     if space.len_w(w_lst) == 1:
         return descr__new__(space, space.gettypefor(W_Dtype),
-                            space.getitem(w_lst, space.wrap(0)), align=align)
+                            space.getitem(w_lst, space.wrap(0)), align=alignment>0)
     else:
-        return dtype_from_list(space, w_lst, True, align=align)
+        return dtype_from_list(space, w_lst, True, alignment)
 
 
 def _check_for_commastring(s):
@@ -757,11 +763,17 @@
 @unwrap_spec(align=bool, copy=bool)
 def descr__new__(space, w_subtype, w_dtype, align=False, copy=False,
                  w_shape=None, w_metadata=None):
+    if align:
+        alignment = 0
+    else:
+        alignment = -1
+    return make_new_dtype(space, w_subtype, w_dtype, alignment, copy=copy, w_shape=w_shape, w_metadata=None)
 
+def make_new_dtype(space, w_subtype, w_dtype, alignment, copy=False, w_shape=None, w_metadata=None):
     cache = get_dtype_cache(space)
     if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or
                                 space.len_w(w_shape) > 0):
-        subdtype = descr__new__(space, w_subtype, w_dtype, align, copy, w_metadata=w_metadata)
+        subdtype = make_new_dtype(space, w_subtype, w_dtype, alignment, copy, w_metadata=w_metadata)
         assert isinstance(subdtype, W_Dtype)
         size = 1
         if space.isinstance_w(w_shape, space.w_int):
@@ -804,7 +816,7 @@
     if space.isinstance_w(w_dtype, space.w_str):
         name = space.str_w(w_dtype)
         if _check_for_commastring(name):
-            return _set_metadata_and_copy(dtype_from_spec(space, w_dtype, align), space, w_metadata)
+            return _set_metadata_and_copy(dtype_from_spec(space, w_dtype, alignment), space, w_metadata)
         cname = name[1:] if name[0] == NPY.OPPBYTE else name
         try:
             dtype = cache.dtypes_by_name[cname]
@@ -818,21 +830,21 @@
             return variable_dtype(space, name)
         raise oefmt(space.w_TypeError, 'data type "%s" not understood', name)
     elif space.isinstance_w(w_dtype, space.w_list):
-        return _set_metadata_and_copy(dtype_from_list(space, w_dtype, False, align=align),
+        return _set_metadata_and_copy(dtype_from_list(space, w_dtype, False, alignment),
                     space, w_metadata, copy)
     elif space.isinstance_w(w_dtype, space.w_tuple):
         w_dtype0 = space.getitem(w_dtype, space.wrap(0))
         w_dtype1 = space.getitem(w_dtype, space.wrap(1))
-        subdtype = descr__new__(space, w_subtype, w_dtype0, align, copy)
+        subdtype = make_new_dtype(space, w_subtype, w_dtype0, align, copy)
         assert isinstance(subdtype, W_Dtype)
         if subdtype.elsize == 0:
             name = "%s%d" % (subdtype.kind, space.int_w(w_dtype1))
-            retval = descr__new__(space, w_subtype, space.wrap(name), align, copy)
+            retval = make_new_dtype(space, w_subtype, space.wrap(name), alignment, copy)
         else:
-            retval = descr__new__(space, w_subtype, w_dtype0, align, copy, w_shape=w_dtype1)
+            retval = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy, w_shape=w_dtype1)
         return _set_metadata_and_copy(retval, space, w_metadata, copy)
     elif space.isinstance_w(w_dtype, space.w_dict):
-        return _set_metadata_and_copy(dtype_from_dict(space, w_dtype, align),
+        return _set_metadata_and_copy(dtype_from_dict(space, w_dtype, alignment),
                     space, w_metadata, copy)
     for dtype in cache.builtin_dtypes:
         if dtype.num in cache.alternate_constructors and \
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1323,10 +1323,13 @@
         assert dt.itemsize == 8
         dt = np.dtype({'f0': ('i4', 0), 'f1':('u1', 4)}, align=True)
         assert dt.itemsize == 8
+        assert dt.alignment == 4
         # Nesting should preserve that alignment
+        print '......'
         dt1 = np.dtype([('f0', 'i4'),
                        ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
                        ('f2', 'i1')], align=True)
+        assert dt.alignment == 4
         assert dt1['f1'].itemsize == 12
         assert dt1.itemsize == 20
         dt2 = np.dtype({'names':['f0', 'f1', 'f2'],

From noreply at buildbot.pypy.org  Fri Jul 10 09:10:12 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 10 Jul 2015 09:10:12 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: create dtype from nested
	lists
Message-ID: <20150710071012.559951C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78517:b207cd06e2b1
Date: 2015-07-08 06:50 +0300
http://bitbucket.org/pypy/pypy/changeset/b207cd06e2b1/

Log:	create dtype from nested lists

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -733,8 +733,12 @@
         return descr__new__(space, space.gettypefor(W_Dtype),
                             space.getitem(w_lst, space.wrap(0)), align=alignment>0)
     else:
-        return dtype_from_list(space, w_lst, True, alignment)
-
+        try:
+            return dtype_from_list(space, w_lst, True, alignment)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return dtype_from_list(space, w_lst, False, alignment)
+            raise
 
 def _check_for_commastring(s):
     if s[0] in string.digits or s[0] in '<>=|' and s[1] in string.digits:
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1325,7 +1325,6 @@
         assert dt.itemsize == 8
         assert dt.alignment == 4
         # Nesting should preserve that alignment
-        print '......'
         dt1 = np.dtype([('f0', 'i4'),
                        ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
                        ('f2', 'i1')], align=True)
@@ -1343,6 +1342,7 @@
                        'f2': ('i1', 16)}, align=True)
         assert dt3.itemsize == 20
         assert dt1 == dt2
+        print '+++++++++++++++++'
         assert dt2 == dt3
         # Nesting should preserve packing
         dt1 = np.dtype([('f0', 'i4'),

From noreply at buildbot.pypy.org  Fri Jul 10 09:10:13 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 10 Jul 2015 09:10:13 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: str(dtype), repr(dtype),
 dtype.descr all use different formatting, add failing test
Message-ID: <20150710071013.83B861C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78518:46fc142d78df
Date: 2015-07-09 00:32 +0300
http://bitbucket.org/pypy/pypy/changeset/46fc142d78df/

Log:	str(dtype), repr(dtype), dtype.descr all use different formatting,
	add failing test

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -198,17 +198,61 @@
             size >>= 2
         return space.wrap("%s%s%s" % (endian, basic, size))
 
-    def descr_get_descr(self, space):
+    def descr_get_descr(self, space, style='descr'):
         if not self.is_record():
             return space.newlist([space.newtuple([space.wrap(""),
                                                   self.descr_get_str(space)])])
+        elif self.alignment >= 0 and style != 'descr':
+            # we need to force a sorting order for the keys,
+            # so return a string instead of a dict
+            names = "'names':["
+            formats = "'formats':["
+            offsets = "'offsets':["
+            titles = "'titles':["
+            use_titles = False
+            for name, title in self.names:
+                offset, subdtype = self.fields[name]
+                if subdtype.is_record():
+                    substr = space.str_w(subdtype.descr_get_descr(space, style))
+                elif subdtype.subdtype is not None:
+                    substr = space.str_w(
+                        subdtype.subdtype.descr_get_str(space)).replace('|', '')
+                else:
+                    substr = space.str_w(
+                        subdtype.descr_get_str(space)).replace('|', '')
+                offsets += str(offset)
+                names += "'" + name + "'"
+                titles += "'" + str(title) + "'"
+                if title is not None:
+                    use_titles = True
+                formats += "'" + substr + "',"
+                offsets += ','
+                names += ','
+                titles += ','
+            formats = formats[:-1] + ']'
+            offsets = offsets[:-1] + ']'
+            names = names[:-1] + ']'
+            titles = titles[:-1] + ']'
+            if style == 'str':
+                suffix = ", 'aligned':True}"
+            else:
+                suffix = "}, align=True"
+            if use_titles: 
+                return space.wrap('{' + names + ', ' + formats + ', ' +
+                            offsets + ', ' + "'itemsize':" + str(self.elsize) +
+                            titles + ', ' + suffix)
+            else:
+                return space.wrap('{' + names + ', ' + formats + ', ' +
+                            offsets + ', ' + "'itemsize':" + str(self.elsize) +
+                            suffix)
+            
         else:
             descr = []
             for name, title in self.names:
                 subdtype = self.fields[name][1]
                 subdescr = [space.wrap(name)]
                 if subdtype.is_record():
-                    subdescr.append(subdtype.descr_get_descr(space))
+                    subdescr.append(subdtype.descr_get_descr(space, style))
                 elif subdtype.subdtype is not None:
                     subdescr.append(subdtype.subdtype.descr_get_str(space))
                 else:
@@ -382,7 +426,7 @@
 
     def descr_str(self, space):
         if self.fields:
-            return space.str(self.descr_get_descr(space))
+            return space.str(self.descr_get_descr(space, style='str'))
         elif self.subdtype is not None:
             return space.str(space.newtuple([
                 self.subdtype.descr_get_str(space),
@@ -395,7 +439,7 @@
 
     def descr_repr(self, space):
         if self.fields:
-            r = self.descr_get_descr(space)
+            r = self.descr_get_descr(space, style='repr')
         elif self.subdtype is not None:
             r = space.newtuple([self.subdtype.descr_get_str(space),
                                 self.descr_get_shape(space)])
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1324,11 +1324,14 @@
         dt = np.dtype({'f0': ('i4', 0), 'f1':('u1', 4)}, align=True)
         assert dt.itemsize == 8
         assert dt.alignment == 4
+        assert str(dt) == "{'names':['f0','f1'], 'formats':['<i4','u1'], 'offsets':[0,4], 'itemsize':8, 'aligned':True}"
+        dt = np.dtype([('f1', 'u1'), ('f0', 'i4')], align=True)
+        assert str(dt) == "{'names':['f1','f0'], 'formats':['u1','<i4'], 'offsets':[0,4], 'itemsize':8, 'aligned':True}"
         # Nesting should preserve that alignment
         dt1 = np.dtype([('f0', 'i4'),
                        ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
                        ('f2', 'i1')], align=True)
-        assert dt.alignment == 4
+        assert dt1.alignment == 4
         assert dt1['f1'].itemsize == 12
         assert dt1.itemsize == 20
         dt2 = np.dtype({'names':['f0', 'f1', 'f2'],
@@ -1342,6 +1345,12 @@
                        'f2': ('i1', 16)}, align=True)
         assert dt3.itemsize == 20
         assert dt1 == dt2
+        assert str(dt3) == "{'names':['f0','f1','f2'], " + \
+                            "'formats':['<i4',{'names':['f1','f2','f3'], " + \
+                                              "'formats':['i1','<i4','i1'], " + \
+                                              "'offsets':[0,4,8], 'itemsize':12}," + \
+                                         "'i1'], " + \
+                            "'offsets':[0,4,16], 'itemsize':20, 'aligned':True}"
         print '+++++++++++++++++'
         assert dt2 == dt3
         # Nesting should preserve packing

From noreply at buildbot.pypy.org  Fri Jul 10 09:10:14 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 10 Jul 2015 09:10:14 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: simplify, fixes translation
Message-ID: <20150710071014.BBFCE1C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78519:0dfdcb222bf9
Date: 2015-07-09 01:00 +0300
http://bitbucket.org/pypy/pypy/changeset/0dfdcb222bf9/

Log:	simplify, fixes translation

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -189,14 +189,19 @@
         return space.wrap(name)
 
     def descr_get_str(self, space):
+        return space.wrap(self.get_str())
+
+    def get_str(self, ignore='|'):
         basic = self.kind
         endian = self.byteorder
         size = self.elsize
         if endian == NPY.NATIVE:
             endian = NPY.NATBYTE
+        elif endian == NPY.IGNORE:
+            endian = ignore
         if self.num == NPY.UNICODE:
             size >>= 2
-        return space.wrap("%s%s%s" % (endian, basic, size))
+        return "%s%s%s" % (endian, basic, size)
 
     def descr_get_descr(self, space, style='descr'):
         if not self.is_record():
@@ -215,20 +220,15 @@
                 if subdtype.is_record():
                     substr = space.str_w(subdtype.descr_get_descr(space, style))
                 elif subdtype.subdtype is not None:
-                    substr = space.str_w(
-                        subdtype.subdtype.descr_get_str(space)).replace('|', '')
+                    substr = subdtype.subdtype.get_str(ignore='')
                 else:
-                    substr = space.str_w(
-                        subdtype.descr_get_str(space)).replace('|', '')
-                offsets += str(offset)
-                names += "'" + name + "'"
-                titles += "'" + str(title) + "'"
+                    substr = subdtype.get_str(ignore='')
+                formats += "'" + substr + "',"
+                offsets += str(offset) + ','
+                names += "'" + name + "',"
+                titles += "'" + str(title) + "',"
                 if title is not None:
                     use_titles = True
-                formats += "'" + substr + "',"
-                offsets += ','
-                names += ','
-                titles += ','
             formats = formats[:-1] + ']'
             offsets = offsets[:-1] + ']'
             names = names[:-1] + ']'

From noreply at buildbot.pypy.org  Fri Jul 10 09:10:15 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 10 Jul 2015 09:10:15 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: whoops
Message-ID: <20150710071015.E8AD51C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78520:36b54ccd3df4
Date: 2015-07-09 01:25 +0300
http://bitbucket.org/pypy/pypy/changeset/36b54ccd3df4/

Log:	whoops

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -817,7 +817,8 @@
         alignment = 0
     else:
         alignment = -1
-    return make_new_dtype(space, w_subtype, w_dtype, alignment, copy=copy, w_shape=w_shape, w_metadata=None)
+    return make_new_dtype(space, w_subtype, w_dtype, alignment, copy=copy,
+                          w_shape=w_shape, w_metadata=w_metadata)
 
 def make_new_dtype(space, w_subtype, w_dtype, alignment, copy=False, w_shape=None, w_metadata=None):
     cache = get_dtype_cache(space)

From noreply at buildbot.pypy.org  Fri Jul 10 09:10:17 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 10 Jul 2015 09:10:17 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: rework alignment handling,
 marginally simplify from_list()
Message-ID: <20150710071017.1310E1C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78521:0f838ade799b
Date: 2015-07-10 00:44 +0300
http://bitbucket.org/pypy/pypy/changeset/0f838ade799b/

Log:	rework alignment handling, marginally simplify from_list()

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -209,7 +209,9 @@
                                                   self.descr_get_str(space)])])
         elif self.alignment >= 0 and style != 'descr':
             # we need to force a sorting order for the keys,
-            # so return a string instead of a dict
+            # so return a string instead of a dict. Also, numpy formats
+            # the lists without spaces between elements, so we cannot simply
+            # do str(names)
             names = "'names':["
             formats = "'formats':["
             offsets = "'offsets':["
@@ -513,7 +515,10 @@
         values = self.descr_get_fields(space)
         if self.is_flexible():
             w_size = space.wrap(self.elsize)
-            w_alignment = space.wrap(self.alignment)
+            if self.alignment > 2:
+                w_alignment = space.wrap(self.alignment)
+            else:
+                w_alignment = space.wrap(1)
         else:
             w_size = space.wrap(-1)
             w_alignment = space.wrap(-1)
@@ -542,6 +547,8 @@
         w_fields = space.getitem(w_data, space.wrap(4))
         size = space.int_w(space.getitem(w_data, space.wrap(5)))
         alignment = space.int_w(space.getitem(w_data, space.wrap(6)))
+        if alignment < 2:
+            alignment = -1
         flags = space.int_w(space.getitem(w_data, space.wrap(7)))
 
         if (w_names == space.w_None) != (w_fields == space.w_None):
@@ -610,16 +617,16 @@
     fields = {}
     if offsets is None:
         offsets = [0] * len(lst_w)
-    names = []
     maxalign = 0
-    total = 0
+    fldnames = [''] * len(lst_w)
+    subdtypes = [None] * len(lst_w)
+    titles = [None] * len(lst_w)
     for i in range(len(lst_w)):
         w_elem = lst_w[i]
-        title = None
         if simple:
             subdtype = make_new_dtype(space, space.gettypefor(W_Dtype), w_elem,
                                     alignment)
-            fldname = 'f%d' % i
+            fldnames[i] = 'f%d' % i
         else:
             w_shape = space.newtuple([])
             if space.len_w(w_elem) == 3:
@@ -632,39 +639,52 @@
                                     w_flddesc, alignment, w_shape=w_shape)
             if space.isinstance_w(w_fldname, space.w_tuple):
                 fldlist = space.listview(w_fldname)
-                fldname = space.str_w(fldlist[0])
-                title = space.str_w(fldlist[1])
+                fldnames[i] = space.str_w(fldlist[0])
+                titles[i] = space.str_w(fldlist[1])
                 if len(fldlist) != 2:
                     raise oefmt(space.w_TypeError, "data type not understood")
             elif space.isinstance_w(w_fldname, space.w_str): 
-                fldname = space.str_w(w_fldname)
+                fldnames[i] = space.str_w(w_fldname)
             else:
                 raise oefmt(space.w_TypeError, "data type not understood")
-            if fldname == '':
-                fldname = 'f%d' % i
-            if fldname in fields:
-                raise oefmt(space.w_ValueError, "two fields with the same name")
+            if fldnames[i] == '':
+                fldnames[i] = 'f%d' % i
         assert isinstance(subdtype, W_Dtype)
-        fields[fldname] = offsets[i], subdtype
-        if title is not None:
-            fields[title] = offsets[i], subdtype
-        maxalign = max(subdtype.alignment, maxalign)
-        if not subdtype.is_record():
-            maxalign = max(subdtype.elsize, maxalign)
-        delta = subdtype.alignment
+        if alignment >= 0:
+            maxalign = max(subdtype.alignment, maxalign)
+            if not subdtype.is_record():
+                maxalign = max(subdtype.elsize, maxalign)
         if  i + 1 < len(offsets) and offsets[i + 1] == 0:
             if alignment >= 0:
+                delta = subdtype.alignment
                 # Set offset to the next power-of-two above delta
                 delta = (delta + maxalign -1) & (-maxalign)
                 if delta > offsets[i]:
                     for j in range(i):
                         offsets[j+1] = delta + offsets[j]
-            offsets[i + 1] = offsets[i] + max(delta, subdtype.elsize)
-        names.append((fldname, title))
+                offsets[i + 1] = offsets[i] + max(delta, subdtype.elsize)
+            else:
+                offsets[i+1] = offsets[i] + subdtype.elsize
+        subdtypes[i] = subdtype
+    names = []
+    for i in range(len(subdtypes)):
+        subdtype = subdtypes[i]
+        assert isinstance(subdtype, W_Dtype)
+        if fldnames[i] in fields:
+            raise oefmt(space.w_ValueError, "two fields with the same name")
+        fields[fldnames[i]] = offsets[i], subdtype
+        if titles[i] is not None:
+            if titles[i] in fields:
+                raise oefmt(space.w_ValueError, "two fields with the same name")
+            fields[titles[i]] = offsets[i], subdtype
+        names.append((fldnames[i], titles[i]))
     total = offsets[-1] + max(maxalign, fields[names[-1][0]][1].elsize)
     retval = W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
                    names=names, fields=fields, elsize=total)
-    retval.alignment = maxalign
+    if alignment >=0:
+        retval.alignment = maxalign
+    else:
+        retval.alignment = -1
     retval.flags |= NPY.NEEDS_PYAPI
     return retval
 
@@ -746,9 +766,10 @@
     else:
         offsets = [space.int_w(i) for i in offsets_w]
     if titles_w is not None:
-        names_w = []
+        _names_w = []
         for i in range(min(len(names_w), len(titles_w))):
-            names_w.append(space.newtuple([names_w[i], titles_w[i]]))
+            _names_w.append(space.newtuple([names_w[i], titles_w[i]]))
+        names_w = _names_w
     aslist = []
     for i in range(min(len(names_w), len(formats_w))):
         aslist.append(space.newtuple([names_w[i], formats_w[i]]))

From noreply at buildbot.pypy.org  Fri Jul 10 09:10:18 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 10 Jul 2015 09:10:18 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: fine tune alignment and
 __str__(), all tests pass
Message-ID: <20150710071018.4251B1C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78522:10bff82523d3
Date: 2015-07-10 10:03 +0300
http://bitbucket.org/pypy/pypy/changeset/10bff82523d3/

Log:	fine tune alignment and __str__(), all tests pass

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -220,12 +220,13 @@
             for name, title in self.names:
                 offset, subdtype = self.fields[name]
                 if subdtype.is_record():
-                    substr = space.str_w(subdtype.descr_get_descr(space, style))
+                    substr = space.str_w(space.str(subdtype.descr_get_descr(
+                                                space, style='substr'))) + ","
                 elif subdtype.subdtype is not None:
-                    substr = subdtype.subdtype.get_str(ignore='')
+                    substr = "'" + subdtype.subdtype.get_str(ignore='') + "',"
                 else:
-                    substr = subdtype.get_str(ignore='')
-                formats += "'" + substr + "',"
+                    substr = "'" + subdtype.get_str(ignore='') + "',"
+                formats += substr
                 offsets += str(offset) + ','
                 names += "'" + name + "',"
                 titles += "'" + str(title) + "',"
@@ -237,6 +238,8 @@
             titles = titles[:-1] + ']'
             if style == 'str':
                 suffix = ", 'aligned':True}"
+            elif style == 'substr':
+                suffix = '}'
             else:
                 suffix = "}, align=True"
             if use_titles: 
@@ -617,7 +620,7 @@
     fields = {}
     if offsets is None:
         offsets = [0] * len(lst_w)
-    maxalign = 0
+    maxalign = alignment 
     fldnames = [''] * len(lst_w)
     subdtypes = [None] * len(lst_w)
     titles = [None] * len(lst_w)
@@ -625,7 +628,7 @@
         w_elem = lst_w[i]
         if simple:
             subdtype = make_new_dtype(space, space.gettypefor(W_Dtype), w_elem,
-                                    alignment)
+                                    maxalign)
             fldnames[i] = 'f%d' % i
         else:
             w_shape = space.newtuple([])
@@ -636,7 +639,7 @@
             else:
                 w_fldname, w_flddesc = space.fixedview(w_elem, 2)
             subdtype = make_new_dtype(space, space.gettypefor(W_Dtype),
-                                    w_flddesc, alignment, w_shape=w_shape)
+                                    w_flddesc, maxalign, w_shape=w_shape)
             if space.isinstance_w(w_fldname, space.w_tuple):
                 fldlist = space.listview(w_fldname)
                 fldnames[i] = space.str_w(fldlist[0])
@@ -654,22 +657,24 @@
             maxalign = max(subdtype.alignment, maxalign)
             if not subdtype.is_record():
                 maxalign = max(subdtype.elsize, maxalign)
-        if  i + 1 < len(offsets) and offsets[i + 1] == 0:
-            if alignment >= 0:
-                delta = subdtype.alignment
-                # Set offset to the next power-of-two above delta
-                delta = (delta + maxalign -1) & (-maxalign)
-                if delta > offsets[i]:
-                    for j in range(i):
-                        offsets[j+1] = delta + offsets[j]
+            delta = subdtype.alignment
+            # Set offset to the next power-of-two above delta
+            delta = (delta + maxalign -1) & (-maxalign)
+            if delta > offsets[i]:
+                for j in range(i):
+                    offsets[j+1] = delta + offsets[j]
+            if  i + 1 < len(offsets) and offsets[i + 1] == 0:
                 offsets[i + 1] = offsets[i] + max(delta, subdtype.elsize)
-            else:
+        else:
+            if  i + 1 < len(offsets) and offsets[i + 1] == 0:
                 offsets[i+1] = offsets[i] + subdtype.elsize
         subdtypes[i] = subdtype
     names = []
     for i in range(len(subdtypes)):
         subdtype = subdtypes[i]
         assert isinstance(subdtype, W_Dtype)
+        if alignment >=0 and subdtype.is_record():
+            subdtype.alignment = maxalign
         if fldnames[i] in fields:
             raise oefmt(space.w_ValueError, "two fields with the same name")
         fields[fldnames[i]] = offsets[i], subdtype
@@ -711,7 +716,11 @@
     for fname in w_dict.iterkeys().iterator:
         obj = _get_list_or_none(space, w_dict, fname)
         num = space.int_w(obj[1])
-        format = dtype_from_spec(space, obj[0], alignment=-1)
+        if align:
+            alignment = 0
+        else:
+            alignment = -1
+        format = dtype_from_spec(space, obj[0], alignment=alignment)
         if len(obj) > 2:
             title = obj[2]
         else:
@@ -821,7 +830,7 @@
             sqbracket -= 1
     return False
 
-def _set_metadata_and_copy(dtype, space, w_metadata, copy=False):
+def _set_metadata_and_copy(space, w_metadata, dtype, copy=False):
     cache = get_dtype_cache(space)
     assert isinstance(dtype, W_Dtype)
     if copy or (dtype in cache.builtin_dtypes and w_metadata is not None):
@@ -874,10 +883,9 @@
         if size >= 2 ** 31:
             raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
                   "dtype size in bytes must fit into a C int.")
-        return _set_metadata_and_copy(W_Dtype(types.VoidType(space),
-                       space.gettypefor(boxes.W_VoidBox),
-                       shape=shape, subdtype=subdtype, elsize=size),
-                    space, w_metadata)
+        return _set_metadata_and_copy(space, w_metadata,
+               W_Dtype(types.VoidType(space), space.gettypefor(boxes.W_VoidBox),
+                       shape=shape, subdtype=subdtype, elsize=size))
 
     if space.is_none(w_dtype):
         return cache.w_float64dtype
@@ -888,7 +896,8 @@
     if space.isinstance_w(w_dtype, space.w_str):
         name = space.str_w(w_dtype)
         if _check_for_commastring(name):
-            return _set_metadata_and_copy(dtype_from_spec(space, w_dtype, alignment), space, w_metadata)
+            return _set_metadata_and_copy(space, w_metadata,
+                                dtype_from_spec(space, w_dtype, alignment))
         cname = name[1:] if name[0] == NPY.OPPBYTE else name
         try:
             dtype = cache.dtypes_by_name[cname]
@@ -902,8 +911,8 @@
             return variable_dtype(space, name)
         raise oefmt(space.w_TypeError, 'data type "%s" not understood', name)
     elif space.isinstance_w(w_dtype, space.w_list):
-        return _set_metadata_and_copy(dtype_from_list(space, w_dtype, False, alignment),
-                    space, w_metadata, copy)
+        return _set_metadata_and_copy( space, w_metadata,
+                        dtype_from_list(space, w_dtype, False, alignment), copy)
     elif space.isinstance_w(w_dtype, space.w_tuple):
         w_dtype0 = space.getitem(w_dtype, space.wrap(0))
         w_dtype1 = space.getitem(w_dtype, space.wrap(1))
@@ -914,22 +923,22 @@
             retval = make_new_dtype(space, w_subtype, space.wrap(name), alignment, copy)
         else:
             retval = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy, w_shape=w_dtype1)
-        return _set_metadata_and_copy(retval, space, w_metadata, copy)
+        return _set_metadata_and_copy(space, w_metadata, retval, copy)
     elif space.isinstance_w(w_dtype, space.w_dict):
-        return _set_metadata_and_copy(dtype_from_dict(space, w_dtype, alignment),
-                    space, w_metadata, copy)
+        return _set_metadata_and_copy(space, w_metadata,
+                dtype_from_dict(space, w_dtype, alignment), copy)
     for dtype in cache.builtin_dtypes:
         if dtype.num in cache.alternate_constructors and \
                 w_dtype in cache.alternate_constructors[dtype.num]:
-            return _set_metadata_and_copy(dtype, space, w_metadata, copy)
+            return _set_metadata_and_copy(space, w_metadata, dtype, copy)
         if w_dtype is dtype.w_box_type:
-            return _set_metadata_and_copy(dtype, space, w_metadata, copy)
+            return _set_metadata_and_copy(space, w_metadata, dtype, copy)
         if space.isinstance_w(w_dtype, space.w_type) and \
            space.is_true(space.issubtype(w_dtype, dtype.w_box_type)):
-            return _set_metadata_and_copy(W_Dtype(dtype.itemtype, w_dtype, elsize=0),
-                    space, w_metadata, copy)
+            return _set_metadata_and_copy( space, w_metadata,
+                            W_Dtype(dtype.itemtype, w_dtype, elsize=0), copy)
     if space.isinstance_w(w_dtype, space.w_type):
-        return _set_metadata_and_copy(cache.w_objectdtype, space, w_metadata, copy)
+        return _set_metadata_and_copy(space, w_metadata, cache.w_objectdtype, copy)
     raise oefmt(space.w_TypeError, "data type not understood")
 
 
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1321,7 +1321,7 @@
                        'formats':['i4', 'u1'],
                        'offsets':[0, 4]}, align=True)
         assert dt.itemsize == 8
-        dt = np.dtype({'f0': ('i4', 0), 'f1':('u1', 4)}, align=True)
+        dt = np.dtype([('f0', 'i4'), ('f1', 'u1')], align=True)
         assert dt.itemsize == 8
         assert dt.alignment == 4
         assert str(dt) == "{'names':['f0','f1'], 'formats':['<i4','u1'], 'offsets':[0,4], 'itemsize':8, 'aligned':True}"
@@ -1345,13 +1345,13 @@
                        'f2': ('i1', 16)}, align=True)
         assert dt3.itemsize == 20
         assert dt1 == dt2
-        assert str(dt3) == "{'names':['f0','f1','f2'], " + \
-                            "'formats':['<i4',{'names':['f1','f2','f3'], " + \
-                                              "'formats':['i1','<i4','i1'], " + \
-                                              "'offsets':[0,4,8], 'itemsize':12}," + \
-                                         "'i1'], " + \
-                            "'offsets':[0,4,16], 'itemsize':20, 'aligned':True}"
-        print '+++++++++++++++++'
+        answer = "{'names':['f0','f1','f2'], " + \
+                    "'formats':['<i4',{'names':['f1','f2','f3'], " + \
+                                      "'formats':['i1','<i4','i1'], " + \
+                                      "'offsets':[0,4,8], 'itemsize':12}," + \
+                                 "'i1'], " + \
+                    "'offsets':[0,4,16], 'itemsize':20, 'aligned':True}"
+        assert str(dt3) == answer
         assert dt2 == dt3
         # Nesting should preserve packing
         dt1 = np.dtype([('f0', 'i4'),

From noreply at buildbot.pypy.org  Fri Jul 10 09:10:23 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 10 Jul 2015 09:10:23 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: merge default into branch
Message-ID: <20150710071023.BAF0E1C033F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78523:29b32820d583
Date: 2015-07-10 10:04 +0300
http://bitbucket.org/pypy/pypy/changeset/29b32820d583/

Log:	merge default into branch

diff too long, truncating to 2000 out of 6164 lines

diff --git a/lib-python/2.7/test/test_urllib2.py b/lib-python/2.7/test/test_urllib2.py
--- a/lib-python/2.7/test/test_urllib2.py
+++ b/lib-python/2.7/test/test_urllib2.py
@@ -291,6 +291,7 @@
         self.req_headers = []
         self.data = None
         self.raise_on_endheaders = False
+        self.sock = None
         self._tunnel_headers = {}
 
     def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
diff --git a/lib-python/2.7/urllib2.py b/lib-python/2.7/urllib2.py
--- a/lib-python/2.7/urllib2.py
+++ b/lib-python/2.7/urllib2.py
@@ -1200,6 +1200,12 @@
                 r = h.getresponse(buffering=True)
             except TypeError: # buffering kw not supported
                 r = h.getresponse()
+            # If the server does not send us a 'Connection: close' header,
+            # HTTPConnection assumes the socket should be left open. Manually
+            # mark the socket to be closed when this response object goes away.
+            if h.sock:
+                h.sock.close()
+                h.sock = None
 
         # Pick apart the HTTPResponse object to get the addinfourl
         # object initialized properly.
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -108,6 +108,8 @@
         return value.internalRep.doubleValue
     if value.typePtr == typeCache.IntType:
         return value.internalRep.longValue
+    if value.typePtr == typeCache.WideIntType:
+        return FromWideIntObj(app, value)
     if value.typePtr == typeCache.BigNumType and tklib.HAVE_LIBTOMMATH:
         return FromBignumObj(app, value)
     if value.typePtr == typeCache.ListType:
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -179,6 +179,7 @@
 typedef int... Tcl_WideInt;
 
 int Tcl_GetWideIntFromObj(Tcl_Interp *interp, Tcl_Obj *obj, Tcl_WideInt *value);
+Tcl_Obj *Tcl_NewWideIntObj(Tcl_WideInt value);
 """)
 
 if HAVE_LIBTOMMATH:
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.1.2
+Version: 1.2.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.2"
-__version_info__ = (1, 1, 2)
+__version__ = "1.2.0"
+__version_info__ = (1, 2, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -236,6 +236,30 @@
             cdecl = self._typeof(cdecl)
         return self._backend.newp(cdecl, init)
 
+    def new_allocator(self, alloc=None, free=None,
+                      should_clear_after_alloc=True):
+        """Return a new allocator, i.e. a function that behaves like ffi.new()
+        but uses the provided low-level 'alloc' and 'free' functions.
+
+        'alloc' is called with the size as argument.  If it returns NULL, a
+        MemoryError is raised.  'free' is called with the result of 'alloc'
+        as argument.  Both can be either Python function or directly C
+        functions.  If 'free' is None, then no free function is called.
+        If both 'alloc' and 'free' are None, the default is used.
+
+        If 'should_clear_after_alloc' is set to False, then the memory
+        returned by 'alloc' is assumed to be already cleared (or you are
+        fine with garbage); otherwise CFFI will clear it.
+        """
+        compiled_ffi = self._backend.FFI()
+        allocator = compiled_ffi.new_allocator(alloc, free,
+                                               should_clear_after_alloc)
+        def allocate(cdecl, init=None):
+            if isinstance(cdecl, basestring):
+                cdecl = self._typeof(cdecl)
+            return allocator(cdecl, init)
+        return allocate
+
     def cast(self, cdecl, source):
         """Similar to a C cast: returns an instance of the named C
         type initialized with the given 'source'.  The source is
@@ -286,7 +310,7 @@
         """
         return self._backend.from_buffer(self.BCharA, python_buffer)
 
-    def callback(self, cdecl, python_callable=None, error=None):
+    def callback(self, cdecl, python_callable=None, error=None, onerror=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
         The callback invokes the specified 'python_callable' (which may
@@ -298,7 +322,8 @@
             if not callable(python_callable):
                 raise TypeError("the 'python_callable' argument "
                                 "is not callable")
-            return self._backend.callback(cdecl, python_callable, error)
+            return self._backend.callback(cdecl, python_callable,
+                                          error, onerror)
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl, consider_function_as_funcptr=True)
         if python_callable is None:
@@ -327,6 +352,13 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
+        try:
+            gcp = self._backend.gcp
+        except AttributeError:
+            pass
+        else:
+            return gcp(cdata, destructor)
+        #
         with self._lock:
             try:
                 gc_weakrefs = self.gc_weakrefs
@@ -428,6 +460,8 @@
             raise TypeError("ffi.include() expects an argument that is also of"
                             " type cffi.FFI, not %r" % (
                                 type(ffi_to_include).__name__,))
+        if ffi_to_include is self:
+            raise ValueError("self.include(self)")
         with ffi_to_include._lock:
             with self._lock:
                 self._parser.include(ffi_to_include._parser)
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -989,7 +989,8 @@
     def cast(self, BType, source):
         return BType._cast_from(source)
 
-    def callback(self, BType, source, error):
+    def callback(self, BType, source, error, onerror):
+        assert onerror is None   # XXX not implemented
         return BType(source, error)
 
     typeof = type
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -633,6 +633,8 @@
 
     def include(self, other):
         for name, tp in other._declarations.items():
+            if name.startswith('anonymous $enum_$'):
+                continue   # fix for test_anonymous_enum_include
             kind = name.split(' ', 1)[0]
             if kind in ('struct', 'union', 'enum', 'anonymous'):
                 self._declare(name, tp, included=True)
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -35,9 +35,6 @@
     def is_integer_type(self):
         return False
 
-    def sizeof_enabled(self):
-        return False
-
     def get_cached_btype(self, ffi, finishlist, can_delay=False):
         try:
             BType = ffi._cached_btypes[self]
@@ -80,8 +77,7 @@
 
 
 class BasePrimitiveType(BaseType):
-    def sizeof_enabled(self):
-        return True
+    pass
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -205,9 +201,6 @@
 class FunctionPtrType(BaseFunctionType):
     _base_pattern = '(*&)(%s)'
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         result = self.result.get_cached_btype(ffi, finishlist)
         args = []
@@ -233,9 +226,6 @@
             extra = self._base_pattern
         self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra)
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True)
         return global_cache(self, ffi, 'new_pointer_type', BItem)
@@ -276,9 +266,6 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def sizeof_enabled(self):
-        return self.item.sizeof_enabled() and self.length is not None
-
     def resolve_length(self, newlength):
         return ArrayType(self.item, newlength)
 
@@ -433,9 +420,6 @@
             from . import ffiplatform
             raise ffiplatform.VerificationMissing(self._get_c_name())
 
-    def sizeof_enabled(self):
-        return self.fldtypes is not None
-
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
@@ -464,9 +448,6 @@
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
-    def sizeof_enabled(self):
-        return True     # not strictly true, but external enums are obscure
-
     def force_the_name(self, forcename):
         StructOrUnionOrEnum.force_the_name(self, forcename)
         if self.forcename is None:
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -26,6 +26,7 @@
 #define _CFFI_OP_GLOBAL_VAR     33
 #define _CFFI_OP_DLOPEN_FUNC    35
 #define _CFFI_OP_DLOPEN_CONST   37
+#define _CFFI_OP_GLOBAL_VAR_F   39
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -981,10 +981,6 @@
         if not self.target_is_python and tp.is_integer_type():
             type_op = CffiOp(OP_CONSTANT_INT, -1)
         else:
-            if not tp.sizeof_enabled():
-                raise ffiplatform.VerificationError(
-                    "constant '%s' is of type '%s', whose size is not known"
-                    % (name, tp._get_c_name()))
             if self.target_is_python:
                 const_kind = OP_DLOPEN_CONST
             else:
@@ -1069,18 +1065,36 @@
         self._do_collect_type(self._global_type(tp, name))
 
     def _generate_cpy_variable_decl(self, tp, name):
-        pass
+        prnt = self._prnt
+        tp = self._global_type(tp, name)
+        if isinstance(tp, model.ArrayType) and tp.length is None:
+            tp = tp.item
+            ampersand = ''
+        else:
+            ampersand = '&'
+        # This code assumes that casts from "tp *" to "void *" is a
+        # no-op, i.e. a function that returns a "tp *" can be called
+        # as if it returned a "void *".  This should be generally true
+        # on any modern machine.  The only exception to that rule (on
+        # uncommon architectures, and as far as I can tell) might be
+        # if 'tp' were a function type, but that is not possible here.
+        # (If 'tp' is a function _pointer_ type, then casts from "fn_t
+        # **" to "void *" are again no-ops, as far as I can tell.)
+        prnt('static ' + tp.get_c_name('*_cffi_var_%s(void)' % (name,)))
+        prnt('{')
+        prnt('  return %s(%s);' % (ampersand, name))
+        prnt('}')
+        prnt()
 
     def _generate_cpy_variable_ctx(self, tp, name):
         tp = self._global_type(tp, name)
         type_index = self._typesdict[tp]
-        type_op = CffiOp(OP_GLOBAL_VAR, type_index)
-        if tp.sizeof_enabled():
-            size = "sizeof(%s)" % (name,)
+        if self.target_is_python:
+            op = OP_GLOBAL_VAR
         else:
-            size = 0
+            op = OP_GLOBAL_VAR_F
         self._lsts["global"].append(
-            GlobalExpr(name, '&%s' % name, type_op, size))
+            GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index)))
 
     # ----------
     # emitting the opcodes for individual types
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -23,14 +23,14 @@
 default_modules.update([
     "_codecs", "gc", "_weakref", "marshal", "errno", "imp", "math", "cmath",
     "_sre", "_pickle_support", "operator", "parser", "symbol", "token", "_ast",
-    "_io", "_random", "__pypy__", "_testing"
+    "_io", "_random", "__pypy__", "_testing", "time"
 ])
 
 
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -135,7 +135,7 @@
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
 is not reliable in PyPy (nor Jython nor IronPython).  It also means that
-weak references may stay alive for a bit longer than expected.  This
+**weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
 suddenly they will really be dead, raising a ``ReferenceError`` on the
@@ -143,6 +143,24 @@
 ``ReferenceError`` at any place that uses them.  (Or, better yet, don't use
 ``weakref.proxy()`` at all; use ``weakref.ref()``.)
 
+Note a detail in the `documentation for weakref callbacks`__:
+
+    If callback is provided and not None, *and the returned weakref
+    object is still alive,* the callback will be called when the object
+    is about to be finalized.
+
+There are cases where, due to CPython's refcount semantics, a weakref
+dies immediately before or after the objects it points to (typically
+with some circular reference).  If it happens to die just after, then
+the callback will be invoked.  In a similar case in PyPy, both the
+object and the weakref will be considered as dead at the same time,
+and the callback will not be invoked.  (Issue `#2030`__)
+
+.. __: https://docs.python.org/2/library/weakref.html
+.. __: https://bitbucket.org/pypy/pypy/issue/2030/
+
+---------------------------------
+
 There are a few extra implications from the difference in the GC.  Most
 notably, if an object has a ``__del__``, the ``__del__`` is never called more
 than once in PyPy; but CPython will call the same ``__del__`` several times
@@ -321,9 +339,8 @@
 Miscellaneous
 -------------
 
-* Hash randomization (``-R``) is ignored in PyPy.  As documented in
-  http://bugs.python.org/issue14621, some of us believe it has no
-  purpose in CPython either.
+* Hash randomization (``-R``) `is ignored in PyPy`_.  In CPython
+  before 3.4 it has `little point`_.
 
 * You can't store non-string keys in type objects.  For example::
 
@@ -338,7 +355,8 @@
   for about 1400 calls.
 
 * since the implementation of dictionary is different, the exact number
-  which ``__hash__`` and ``__eq__`` are called is different. Since CPython
+  of times that ``__hash__`` and ``__eq__`` are called is different. 
+  Since CPython
   does not give any specific guarantees either, don't rely on it.
 
 * assignment to ``__class__`` is limited to the cases where it
@@ -395,3 +413,12 @@
   interactive mode. In a released version, this behaviour is suppressed, but
   setting the environment variable PYPY_IRC_TOPIC will bring it back. Note that
   downstream package providers have been known to totally disable this feature.
+
+* PyPy's readline module was rewritten from scratch: it is not GNU's
+  readline.  It should be mostly compatible, and it adds multiline
+  support (see ``multiline_input()``).  On the other hand,
+  ``parse_and_bind()`` calls are ignored (issue `#2072`_).
+
+.. _`is ignored in PyPy`: http://bugs.python.org/issue14621
+.. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
+.. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,15 +6,9 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
-**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
-on linux, linux64 and windows. We will make it the default on all platforms
-by the time of the next release.
-
-The first thing that you need is to compile PyPy yourself with the option
-``--shared``. We plan to make ``--shared`` the default in the future. Consult
-the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
-or ``pypy.dll`` file or something similar, depending on your platform. Consult
-your platform specification for details.
+**NOTE**: You need a PyPy compiled with the option ``--shared``, i.e.
+with a ``libpypy-c.so`` or ``pypy-c.dll`` file.  This is the default in
+recent versions of PyPy.
 
 The resulting shared library exports very few functions, however they are
 enough to accomplish everything you need, provided you follow a few principles.
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -70,6 +70,20 @@
 .. _`use virtualenv (as documented here)`: getting-started.html#installing-using-virtualenv
 
 
+Module xyz does not work in the sandboxed PyPy?
+-----------------------------------------------
+
+You cannot import *any* extension module in a `sandboxed PyPy`_,
+sorry.  Even the built-in modules available are very limited.
+Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
+it is only a proof of concept.  It seriously requires someone working
+on it.  Before this occurs, it can only be used it for "pure Python"
+examples: programs that import mostly nothing (or only pure Python
+modules, recursively).
+
+.. _`sandboxed PyPy`: sandbox.html
+
+
 .. _`See below.`:
 
 Do CPython Extension modules work with PyPy?
diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -103,12 +103,15 @@
 Howto
 -----
 
-In pypy/goal::
+Grab a copy of the pypy repository_.  In the directory pypy/goal, run::
 
    ../../rpython/bin/rpython -O2 --sandbox targetpypystandalone.py
 
 If you don't have a regular PyPy installed, you should, because it's
-faster to translate, but you can also run ``python translate.py`` instead.
+faster to translate; but you can also run the same line with ``python``
+in front.
+
+.. _repository: https://bitbucket.org/pypy/pypy
 
 
 To run it, use the tools in the pypy/sandbox directory::
@@ -136,8 +139,6 @@
 Not all operations are supported; e.g. if you type os.readlink('...'),
 the controller crashes with an exception and the subprocess is killed.
 Other operations make the subprocess die directly with a "Fatal RPython
-error".  None of this is a security hole; it just means that if you try
-to run some random program, it risks getting killed depending on the
-Python built-in functions it tries to call.  This is a matter of the
-sandboxing layer being incomplete so far, but it should not really be
-a problem in practice.
+error".  None of this is a security hole.  More importantly, *most other
+built-in modules are not enabled.  Please read all the warnings in this
+page before complaining about this.  Contributions welcome.*
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -11,3 +11,29 @@
 .. branch: stdlib-2.7.10
 
 Update stdlib to version 2.7.10
+
+.. branch: issue2062
+
+.. branch: disable-unroll-for-short-loops
+The JIT no longer performs loop unrolling if the loop compiles to too much code.
+
+.. branch: run-create_cffi_imports
+
+Build cffi import libraries as part of translation by monkey-patching an 
+additional task into translation
+
+.. branch: int-float-list-strategy
+
+Use a compact strategy for Python lists that mix integers and floats,
+at least if the integers fit inside 32 bits.  These lists are now
+stored as an array of floats, like lists that contain only floats; the
+difference is that integers are stored as tagged NaNs.  (This should
+have no visible effect!  After ``lst = [42, 42.5]``, the value of
+``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
+
+.. branch: cffi-callback-onerror
+.. branch: cffi-new-allocator
+
+.. branch: unicode-dtype
+
+Partial implementation of unicode dtype and unicode scalars.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -1,6 +1,6 @@
 import py
 
-import os, sys
+import os, sys, subprocess
 
 import pypy
 from pypy.interpreter import gateway
@@ -297,8 +297,49 @@
         options = make_dict(config)
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
+        if config.objspace.usemodules._cffi_backend:
+            self.hack_for_cffi_modules(driver)
 
         return self.get_entry_point(config)
+    
+    def hack_for_cffi_modules(self, driver):
+        # HACKHACKHACK
+        # ugly hack to modify target goal from compile_c to build_cffi_imports
+        # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.translator.driver import taskdef
+        import types
+
+        class Options(object):
+            pass
+
+
+        def mkexename(name):
+            if sys.platform == 'win32':
+                name = name.new(ext='exe')
+            return name
+
+        @taskdef(['compile_c'], "Create cffi bindings for modules")
+        def task_build_cffi_imports(self):
+            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
+            ''' Use cffi to compile cffi interfaces to modules'''
+            exename = mkexename(driver.compute_exe_name())
+            basedir = exename
+            while not basedir.join('include').exists():
+                _basedir = basedir.dirpath()
+                if _basedir == basedir:
+                    raise ValueError('interpreter %s not inside pypy repo', 
+                                     str(exename))
+                basedir = _basedir
+            modules = self.config.objspace.usemodules.getpaths()
+            options = Options()
+            # XXX possibly adapt options using modules
+            failures = create_cffi_import_libraries(exename, options, basedir)
+            # if failures, they were already printed
+            print  >> sys.stderr, str(exename),'successfully built, but errors while building the above modules will be ignored'
+        driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
+        driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, ['compile_c']
+        driver.default_goal = 'build_cffi_imports'
+        # HACKHACKHACK end
 
     def jitpolicy(self, driver):
         from pypy.module.pypyjit.policy import PyPyJitPolicy
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -252,7 +252,8 @@
                                w_t, w_v, w_tb],
                 """(where, objrepr, extra_line, t, v, tb):
                     import sys, traceback
-                    sys.stderr.write('From %s%s:\\n' % (where, objrepr))
+                    if where or objrepr:
+                        sys.stderr.write('From %s%s:\\n' % (where, objrepr))
                     if extra_line:
                         sys.stderr.write(extra_line)
                     traceback.print_exception(t, v, tb)
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -15,7 +15,10 @@
         self.running = False
 
     def descr__repr__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = '<finished>'
+        else:
+            code_name = self.pycode.co_name
         addrstring = self.getaddrstring(space)
         return space.wrap("<generator object %s at 0x%s>" %
                           (code_name, addrstring))
@@ -45,6 +48,8 @@
         w_framestate, w_running = args_w
         if space.is_w(w_framestate, space.w_None):
             self.frame = None
+            self.space = space
+            self.pycode = None
         else:
             frame = instantiate(space.FrameClass)   # XXX fish
             frame.descr__setstate__(space, w_framestate)
@@ -62,9 +67,10 @@
 
     def send_ex(self, w_arg, operr=None):
         pycode = self.pycode
-        if jit.we_are_jitted() and should_not_inline(pycode):
-            generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
-                                                  operr=operr, pycode=pycode)
+        if pycode is not None:
+            if jit.we_are_jitted() and should_not_inline(pycode):
+                generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
+                                                    operr=operr, pycode=pycode)
         return self._send_ex(w_arg, operr)
 
     def _send_ex(self, w_arg, operr):
@@ -158,7 +164,10 @@
         return self.pycode
 
     def descr__name__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = '<finished>'
+        else:
+            code_name = self.pycode.co_name
         return space.wrap(code_name)
 
     # Results can be either an RPython list of W_Root, or it can be an
diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py
--- a/pypy/interpreter/pytraceback.py
+++ b/pypy/interpreter/pytraceback.py
@@ -60,7 +60,6 @@
 
 
 def check_traceback(space, w_tb, msg):
-    from pypy.interpreter.typedef import PyTraceback
     if w_tb is None or not space.isinstance_w(w_tb, space.gettypeobject(PyTraceback.typedef)):
         raise OperationError(space.w_TypeError, space.wrap(msg))
     return w_tb
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -491,6 +491,22 @@
         assert pack.mod is result
 
 
+    def test_pickle_generator_crash(self):
+        import pickle
+
+        def f():
+            yield 0
+
+        x = f()
+        x.next()
+        try:
+            x.next()
+        except StopIteration:
+            y = pickle.loads(pickle.dumps(x))
+        assert 'finished' in y.__name__
+        assert 'finished' in repr(y)
+        assert y.gi_code is None
+
 class AppTestGeneratorCloning:
 
     def setup_class(cls):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.2"
+VERSION = "1.2.0"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -0,0 +1,86 @@
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+class W_Allocator(W_Root):
+    _immutable_ = True
+
+    def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc):
+        self.ffi = ffi    # may be None
+        self.w_alloc = w_alloc
+        self.w_free = w_free
+        self.should_clear_after_alloc = should_clear_after_alloc
+
+    def allocate(self, space, datasize, ctype, length=-1):
+        from pypy.module._cffi_backend import cdataobj, ctypeptr
+        if self.w_alloc is None:
+            if self.should_clear_after_alloc:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=True)
+            else:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=False)
+            return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+        else:
+            w_raw_cdata = space.call_function(self.w_alloc,
+                                              space.wrap(datasize))
+            if not isinstance(w_raw_cdata, cdataobj.W_CData):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata object (got %T)",
+                            w_raw_cdata)
+            if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata pointer, not '%s'",
+                            w_raw_cdata.ctype.name)
+            #
+            ptr = w_raw_cdata.unsafe_escaping_ptr()
+            if not ptr:
+                raise oefmt(space.w_MemoryError, "alloc() returned NULL")
+            #
+            if self.should_clear_after_alloc:
+                rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0,
+                              rffi.cast(rffi.SIZE_T, datasize))
+            #
+            if self.w_free is None:
+                # use this class which does not have a __del__, but still
+                # keeps alive w_raw_cdata
+                res = cdataobj.W_CDataNewNonStdNoFree(space, ptr, ctype, length)
+            else:
+                res = cdataobj.W_CDataNewNonStdFree(space, ptr, ctype, length)
+                res.w_free = self.w_free
+            res.w_raw_cdata = w_raw_cdata
+            return res
+
+    @unwrap_spec(w_init=WrappedDefault(None))
+    def descr_call(self, space, w_arg, w_init):
+        ffi = self.ffi
+        assert ffi is not None
+        w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING | ffi.ACCEPT_CTYPE)
+        return w_ctype.newp(w_init, self)
+
+
+W_Allocator.typedef = TypeDef(
+        'FFIAllocator',
+        __call__ = interp2app(W_Allocator.descr_call),
+        )
+W_Allocator.typedef.acceptable_as_base_class = False
+
+
+def new_allocator(ffi, w_alloc, w_free, should_clear_after_alloc):
+    space = ffi.space
+    if space.is_none(w_alloc):
+        w_alloc = None
+    if space.is_none(w_free):
+        w_free = None
+    if w_alloc is None and w_free is not None:
+        raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'")
+    alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc))
+    return space.wrap(alloc)
+
+
+default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True)
+nonzero_allocator = W_Allocator(None, None, None,should_clear_after_alloc=False)
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -22,8 +22,9 @@
 class W_CDataCallback(W_CData):
     #_immutable_fields_ = ...
     ll_error = lltype.nullptr(rffi.CCHARP.TO)
+    w_onerror = None
 
-    def __init__(self, space, ctype, w_callable, w_error):
+    def __init__(self, space, ctype, w_callable, w_error, w_onerror):
         raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc())
         W_CData.__init__(self, space, raw_closure, ctype)
         #
@@ -31,6 +32,12 @@
             raise oefmt(space.w_TypeError,
                         "expected a callable object, not %T", w_callable)
         self.w_callable = w_callable
+        if not space.is_none(w_onerror):
+            if not space.is_true(space.callable(w_onerror)):
+                raise oefmt(space.w_TypeError,
+                            "expected a callable object for 'onerror', not %T",
+                            w_onerror)
+            self.w_onerror = w_onerror
         #
         fresult = self.getfunctype().ctitem
         size = fresult.size
@@ -161,6 +168,29 @@
 STDERR = 2
 
 
+ at jit.dont_look_inside
+def _handle_applevel_exception(space, callback, e, ll_res, extra_line):
+    callback.write_error_return_value(ll_res)
+    if callback.w_onerror is None:
+        callback.print_error(e, extra_line)
+    else:
+        try:
+            e.normalize_exception(space)
+            w_t = e.w_type
+            w_v = e.get_w_value(space)
+            w_tb = space.wrap(e.get_traceback())
+            w_res = space.call_function(callback.w_onerror,
+                                        w_t, w_v, w_tb)
+            if not space.is_none(w_res):
+                callback.convert_result(ll_res, w_res)
+        except OperationError, e2:
+            # double exception! print a double-traceback...
+            callback.print_error(e, extra_line)    # original traceback
+            e2.write_unraisable(space, '', with_traceback=True,
+                                extra_line="\nDuring the call to 'onerror', "
+                                           "another exception occurred:\n\n")
+
+
 @jit.jit_callback("CFFI")
 def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
     """ Callback specification.
@@ -178,7 +208,7 @@
         try:
             os.write(STDERR, "SystemError: invoking a callback "
                              "that was already freed\n")
-        except OSError:
+        except:
             pass
         # In this case, we don't even know how big ll_res is.  Let's assume
         # it is just a 'ffi_arg', and store 0 there.
@@ -195,9 +225,7 @@
             extra_line = "Trying to convert the result back to C:\n"
             callback.convert_result(ll_res, w_res)
         except OperationError, e:
-            # got an app-level exception
-            callback.print_error(e, extra_line)
-            callback.write_error_return_value(ll_res)
+            _handle_applevel_exception(space, callback, e, ll_res, extra_line)
         #
     except Exception, e:
         # oups! last-level attempt to recover.
@@ -205,7 +233,7 @@
             os.write(STDERR, "SystemError: callback raised ")
             os.write(STDERR, str(e))
             os.write(STDERR, "\n")
-        except OSError:
+        except:
             pass
         callback.write_error_return_value(ll_res)
     if must_leave:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -363,16 +363,19 @@
     def _sizeof(self):
         return self.ctype.size
 
+    def with_gc(self, w_destructor):
+        with self as ptr:
+            return W_CDataGCP(self.space, ptr, self.ctype, self, w_destructor)
+
 
 class W_CDataMem(W_CData):
-    """This is the base class used for cdata objects that own and free
-    their memory.  Used directly by the results of cffi.cast('int', x)
-    or other primitive explicitly-casted types.  It is further subclassed
-    by W_CDataNewOwning."""
+    """This is used only by the results of cffi.cast('int', x)
+    or other primitive explicitly-casted types."""
     _attrs_ = []
 
-    def __init__(self, space, size, ctype):
-        cdata = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw', zero=True)
+    def __init__(self, space, ctype):
+        cdata = lltype.malloc(rffi.CCHARP.TO, ctype.size, flavor='raw',
+                              zero=False)
         W_CData.__init__(self, space, cdata, ctype)
 
     @rgc.must_be_light_finalizer
@@ -380,36 +383,65 @@
         lltype.free(self._ptr, flavor='raw')
 
 
-class W_CDataNewOwning(W_CDataMem):
-    """This is the class used for the cata objects created by newp()."""
-    _attrs_ = []
+class W_CDataNewOwning(W_CData):
+    """This is the abstract base class used for cdata objects created
+    by newp().  They create and free their own memory according to an
+    allocator."""
+
+    # the 'length' is either >= 0 for arrays, or -1 for pointers.
+    _attrs_ = ['length']
+    _immutable_fields_ = ['length']
+
+    def __init__(self, space, cdata, ctype, length=-1):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.length = length
 
     def _repr_extra(self):
         return self._repr_extra_owning()
 
-
-class W_CDataNewOwningLength(W_CDataNewOwning):
-    """Subclass with an explicit length, for allocated instances of
-    the C type 'foo[]'."""
-    _attrs_ = ['length']
-    _immutable_fields_ = ['length']
-
-    def __init__(self, space, size, ctype, length):
-        W_CDataNewOwning.__init__(self, space, size, ctype)
-        self.length = length
-
     def _sizeof(self):
-        from pypy.module._cffi_backend import ctypearray
         ctype = self.ctype
-        assert isinstance(ctype, ctypearray.W_CTypeArray)
-        return self.length * ctype.ctitem.size
+        if self.length >= 0:
+            from pypy.module._cffi_backend import ctypearray
+            assert isinstance(ctype, ctypearray.W_CTypeArray)
+            return self.length * ctype.ctitem.size
+        else:
+            return ctype.size
 
     def get_array_length(self):
         return self.length
 
 
+class W_CDataNewStd(W_CDataNewOwning):
+    """Subclass using the standard allocator, lltype.malloc()/lltype.free()"""
+    _attrs_ = []
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        lltype.free(self._ptr, flavor='raw')
+
+
+class W_CDataNewNonStdNoFree(W_CDataNewOwning):
+    """Subclass using a non-standard allocator, no free()"""
+    _attrs_ = ['w_raw_cdata']
+
+class W_CDataNewNonStdFree(W_CDataNewNonStdNoFree):
+    """Subclass using a non-standard allocator, with a free()"""
+    _attrs_ = ['w_free']
+
+    def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space,
+                                     W_CDataNewNonStdFree.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataNewNonStdFree)
+        self.space.call_function(self.w_free, self.w_raw_cdata)
+
+
 class W_CDataPtrToStructOrUnion(W_CData):
-    """This subclass is used for the pointer returned by new('struct foo').
+    """This subclass is used for the pointer returned by new('struct foo *').
     It has a strong reference to a W_CDataNewOwning that really owns the
     struct, which is the object returned by the app-level expression 'p[0]'.
     But it is not itself owning any memory, although its repr says so;
@@ -483,6 +515,26 @@
             self.length, self.space.type(self.w_keepalive).name)
 
 
+class W_CDataGCP(W_CData):
+    """For ffi.gc()."""
+    _attrs_ = ['w_original_cdata', 'w_destructor']
+    _immutable_fields_ = ['w_original_cdata', 'w_destructor']
+
+    def __init__(self, space, cdata, ctype, w_original_cdata, w_destructor):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.w_original_cdata = w_original_cdata
+        self.w_destructor = w_destructor
+
+    def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space, W_CDataGCP.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataGCP)
+        self.space.call_function(self.w_destructor, self.w_original_cdata)
+
+
 W_CData.typedef = TypeDef(
     '_cffi_backend.CData',
     __module__ = '_cffi_backend',
diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -36,7 +36,10 @@
                         self.libname)
         try:
             cdata = dlsym(self.libhandle, name)
+            found = bool(cdata)
         except KeyError:
+            found = False
+        if not found:
             raise oefmt(self.ffi.w_FFIError,
                         "symbol '%s' not found in library '%s'",
                         name, self.libname)
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/pypy/module/_cffi_backend/cgc.py b/pypy/module/_cffi_backend/cgc.py
deleted file mode 100644
--- a/pypy/module/_cffi_backend/cgc.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from rpython.rlib import jit
-
-
- at jit.dont_look_inside
-def gc_weakrefs_build(ffi, w_cdata, w_destructor):
-    from pypy.module._weakref import interp__weakref
-
-    space = ffi.space
-    if ffi.w_gc_wref_remove is None:
-        ffi.gc_wref_dict = {}
-        ffi.w_gc_wref_remove = space.getattr(space.wrap(ffi),
-                                             space.wrap("__gc_wref_remove"))
-
-    w_new_cdata = w_cdata.ctype.cast(w_cdata)
-    assert w_new_cdata is not w_cdata
-
-    w_ref = interp__weakref.make_weakref_with_callback(
-        space,
-        space.gettypefor(interp__weakref.W_Weakref),
-        w_new_cdata,
-        ffi.w_gc_wref_remove)
-
-    ffi.gc_wref_dict[w_ref] = (w_destructor, w_cdata)
-    return w_new_cdata
-
-
-def gc_wref_remove(ffi, w_ref):
-    (w_destructor, w_cdata) = ffi.gc_wref_dict.pop(w_ref)
-    ffi.space.call_function(w_destructor, w_cdata)
diff --git a/pypy/module/_cffi_backend/cglob.py b/pypy/module/_cffi_backend/cglob.py
--- a/pypy/module/_cffi_backend/cglob.py
+++ b/pypy/module/_cffi_backend/cglob.py
@@ -1,24 +1,66 @@
+from pypy.interpreter.error import oefmt
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend import newtype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 
 class W_GlobSupport(W_Root):
-    def __init__(self, space, w_ctype, ptr):
+    _immutable_fields_ = ['w_ctype', 'ptr', 'fetch_addr']
+
+    def __init__(self, space, name, w_ctype, ptr=lltype.nullptr(rffi.CCHARP.TO),
+                 fetch_addr=lltype.nullptr(rffi.VOIDP.TO)):
         self.space = space
+        self.name = name
         self.w_ctype = w_ctype
         self.ptr = ptr
+        self.fetch_addr = fetch_addr
+
+    def fetch_global_var_addr(self):
+        if self.ptr:
+            result = self.ptr
+        else:
+            if not we_are_translated():
+                FNPTR = rffi.CCallback([], rffi.VOIDP)
+                fetch_addr = rffi.cast(FNPTR, self.fetch_addr)
+                result = fetch_addr()
+            else:
+                # careful in translated versions: we need to call fetch_addr,
+                # but in a GIL-releasing way.  The easiest is to invoke a
+                # llexternal() helper.
+                result = pypy__cffi_fetch_var(self.fetch_addr)
+            result = rffi.cast(rffi.CCHARP, result)
+        if not result:
+            from pypy.module._cffi_backend import ffi_obj
+            ffierror = ffi_obj.get_ffi_error(self.space)
+            raise oefmt(ffierror, "global variable '%s' is at address NULL",
+                        self.name)
+        return result
 
     def read_global_var(self):
-        return self.w_ctype.convert_to_object(self.ptr)
+        return self.w_ctype.convert_to_object(self.fetch_global_var_addr())
 
     def write_global_var(self, w_newvalue):
-        self.w_ctype.convert_from_object(self.ptr, w_newvalue)
+        self.w_ctype.convert_from_object(self.fetch_global_var_addr(),
+                                         w_newvalue)
 
     def address(self):
         w_ctypeptr = newtype.new_pointer_type(self.space, self.w_ctype)
-        return W_CData(self.space, self.ptr, w_ctypeptr)
+        return W_CData(self.space, self.fetch_global_var_addr(), w_ctypeptr)
 
 W_GlobSupport.typedef = TypeDef("FFIGlobSupport")
 W_GlobSupport.typedef.acceptable_as_base_class = False
+
+
+eci = ExternalCompilationInfo(post_include_bits=["""
+static void *pypy__cffi_fetch_var(void *fetch_addr) {
+    return ((void*(*)(void))fetch_addr)();
+}
+"""])
+
+pypy__cffi_fetch_var = rffi.llexternal(
+    "pypy__cffi_fetch_var", [rffi.VOIDP], rffi.VOIDP,
+    compilation_info=eci)
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -28,7 +28,7 @@
     def _alignof(self):
         return self.ctitem.alignof()
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         datasize = self.size
         #
@@ -40,12 +40,10 @@
             except OverflowError:
                 raise OperationError(space.w_OverflowError,
                     space.wrap("array size would overflow a ssize_t"))
-            #
-            cdata = cdataobj.W_CDataNewOwningLength(space, datasize,
-                                                    self, length)
+        else:
+            length = self.length
         #
-        else:
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+        cdata = allocator.allocate(space, datasize, self, length)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -143,7 +143,7 @@
     @jit.unroll_safe
     def _call(self, funcaddr, args_w):
         space = self.space
-        cif_descr = self.cif_descr
+        cif_descr = self.cif_descr   # 'self' should have been promoted here
         size = cif_descr.exchange_size
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -55,7 +55,7 @@
     def pack_list_of_items(self, cdata, w_ob):
         return False
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         raise oefmt(space.w_TypeError,
                     "expected a pointer or array ctype, got '%s'", self.name)
@@ -90,6 +90,16 @@
     def _convert_error(self, expected, w_got):
         space = self.space
         if isinstance(w_got, cdataobj.W_CData):
+            if self.name == w_got.ctype.name:
+                # in case we'd give the error message "initializer for
+                # ctype 'A' must be a pointer to same type, not cdata
+                # 'B'", but with A=B, then give instead a different error
+                # message to try to clear up the confusion
+                return oefmt(space.w_TypeError,
+                             "initializer for ctype '%s' appears indeed to "
+                             "be '%s', but the types are different (check "
+                             "that you are not e.g. mixing up different ffi "
+                             "instances)", self.name, w_got.ctype.name)
             return oefmt(space.w_TypeError,
                          "initializer for ctype '%s' must be a %s, not cdata "
                          "'%s'", self.name, expected, w_got.ctype.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -63,7 +63,7 @@
             value = self._cast_result(value)
         else:
             value = self._cast_generic(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         self.write_raw_integer_data(w_cdata, value)
         return w_cdata
 
@@ -134,8 +134,7 @@
 
     def convert_to_object(self, cdata):
         unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        s = rffi.wcharpsize2unicode(unichardata, 1)
-        return self.space.wrap(s)
+        return self.space.wrap(unichardata[0])
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
@@ -354,7 +353,7 @@
             value = self.cast_unicode(w_ob)
         else:
             value = space.float_w(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         if not isinstance(self, W_CTypePrimitiveLongDouble):
             w_cdata.write_raw_float_data(value)
         else:
@@ -447,7 +446,7 @@
         return self.space.wrap(value)
 
     def convert_to_object(self, cdata):
-        w_cdata = cdataobj.W_CDataMem(self.space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(self.space, self)
         with w_cdata as ptr:
             self._copy_longdouble(cdata, ptr)
         return w_cdata
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -185,7 +185,7 @@
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
         space = self.space
         ctitem = self.ctitem
@@ -205,14 +205,14 @@
                     datasize = ctitem.convert_struct_from_object(
                         lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
             #
-            cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
+            cdatastruct = allocator.allocate(space, datasize, ctitem)
             ptr = cdatastruct.unsafe_escaping_ptr()
             cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
                                                        self, cdatastruct)
         else:
             if self.is_char_or_unichar_ptr_or_array():
                 datasize *= 2       # forcefully add a null character
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+            cdata = allocator.allocate(space, datasize, self)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
@@ -223,9 +223,13 @@
         if (isinstance(w_cdata, cdataobj.W_CDataNewOwning) or
             isinstance(w_cdata, cdataobj.W_CDataPtrToStructOrUnion)):
             if i != 0:
-                space = self.space
-                raise oefmt(space.w_IndexError,
+                raise oefmt(self.space.w_IndexError,
                             "cdata '%s' can only be indexed by 0", self.name)
+        else:
+            if not w_cdata.unsafe_escaping_ptr():
+                raise oefmt(self.space.w_RuntimeError,
+                            "cannot dereference null pointer from cdata '%s'",
+                            self.name)
         return self
 
     def _check_slice_index(self, w_cdata, start, stop):
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -81,10 +81,9 @@
     def copy_and_convert_to_object(self, source):
         space = self.space
         self.check_complete()
-        ob = cdataobj.W_CDataNewOwning(space, self.size, self)
-        with ob as target:
-            misc._raw_memcopy(source, target, self.size)
-        return ob
+        ptr = lltype.malloc(rffi.CCHARP.TO, self.size, flavor='raw', zero=False)
+        misc._raw_memcopy(source, ptr, self.size)
+        return cdataobj.W_CDataNewStd(space, ptr, self)
 
     def typeoffsetof_field(self, fieldname, following):
         self.force_lazy_struct()
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -10,8 +10,8 @@
 from pypy.module._cffi_backend import parse_c_type, realize_c_type
 from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray
 from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle
-from pypy.module._cffi_backend import cbuffer, func, cgc, wrapper
-from pypy.module._cffi_backend import cffi_opcode
+from pypy.module._cffi_backend import cbuffer, func, wrapper
+from pypy.module._cffi_backend import cffi_opcode, allocator
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend.cdataobj import W_CData
 
@@ -44,6 +44,10 @@
 
 
 class W_FFIObject(W_Root):
+    ACCEPT_STRING = ACCEPT_STRING
+    ACCEPT_CTYPE  = ACCEPT_CTYPE
+    ACCEPT_CDATA  = ACCEPT_CDATA
+
     w_gc_wref_remove = None
 
     @jit.dont_look_inside
@@ -276,8 +280,9 @@
 
 
     @unwrap_spec(w_python_callable=WrappedDefault(None),
-                 w_error=WrappedDefault(None))
-    def descr_callback(self, w_cdecl, w_python_callable, w_error):
+                 w_error=WrappedDefault(None),
+                 w_onerror=WrappedDefault(None))
+    def descr_callback(self, w_cdecl, w_python_callable, w_error, w_onerror):
         """\
 Return a callback object or a decorator making such a callback object.
 'cdecl' must name a C function pointer type.  The callback invokes the
@@ -290,14 +295,16 @@
         space = self.space
         if not space.is_none(w_python_callable):
             return ccallback.W_CDataCallback(space, w_ctype,
-                                             w_python_callable, w_error)
+                                             w_python_callable, w_error,
+                                             w_onerror)
         else:
             # decorator mode: returns a single-argument function
-            return space.appexec([w_ctype, w_error],
-            """(ctype, error):
+            return space.appexec([w_ctype, w_error, w_onerror],
+            """(ctype, error, onerror):
                 import _cffi_backend
                 return lambda python_callable: (
-                    _cffi_backend.callback(ctype, python_callable, error))""")
+                    _cffi_backend.callback(ctype, python_callable,
+                                           error, onerror))""")
 
 
     def descr_cast(self, w_arg, w_ob):
@@ -341,10 +348,7 @@
 Later, when this new cdata object is garbage-collected,
 'destructor(old_cdata_object)' will be called."""
         #
-        return cgc.gc_weakrefs_build(self, w_cdata, w_destructor)
-
-    def descr___gc_wref_remove(self, w_ref):
-        return cgc.gc_wref_remove(self, w_ref)
+        return w_cdata.with_gc(w_destructor)
 
 
     @unwrap_spec(replace_with=str)
@@ -411,7 +415,31 @@
 pointer to the memory somewhere else, e.g. into another structure."""
         #
         w_ctype = self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CTYPE)
-        return w_ctype.newp(w_init)
+        return w_ctype.newp(w_init, allocator.default_allocator)
+
+
+    @unwrap_spec(w_alloc=WrappedDefault(None),
+                 w_free=WrappedDefault(None),
+                 should_clear_after_alloc=int)
+    def descr_new_allocator(self, w_alloc, w_free,
+                            should_clear_after_alloc=1):
+        """\
+Return a new allocator, i.e. a function that behaves like ffi.new()
+but uses the provided low-level 'alloc' and 'free' functions.
+
+'alloc' is called with the size as argument.  If it returns NULL, a
+MemoryError is raised.  'free' is called with the result of 'alloc'
+as argument.  Both can be either Python function or directly C
+functions.  If 'free' is None, then no free function is called.
+If both 'alloc' and 'free' are None, the default is used.
+
+If 'should_clear_after_alloc' is set to False, then the memory
+returned by 'alloc' is assumed to be already cleared (or you are
+fine with garbage); otherwise CFFI will clear it.
+        """
+        #
+        return allocator.new_allocator(self, w_alloc, w_free,
+                                       should_clear_after_alloc)
 
 
     def descr_new_handle(self, w_arg):
@@ -539,12 +567,17 @@
 
 
 @jit.dont_look_inside
-def W_FFIObject___new__(space, w_subtype, __args__):
-    r = space.allocate_instance(W_FFIObject, w_subtype)
+def make_plain_ffi_object(space, w_ffitype=None):
+    if w_ffitype is None:
+        w_ffitype = space.gettypefor(W_FFIObject)
+    r = space.allocate_instance(W_FFIObject, w_ffitype)
     # get in 'src_ctx' a NULL which translation doesn't consider to be constant
     src_ctx = rffi.cast(parse_c_type.PCTX, 0)
     r.__init__(space, src_ctx)
-    return space.wrap(r)
+    return r
+
+def W_FFIObject___new__(space, w_subtype, __args__):
+    return space.wrap(make_plain_ffi_object(space, w_subtype))
 
 def make_CData(space):
     return space.gettypefor(W_CData)
@@ -578,7 +611,6 @@
                                      W_FFIObject.set_errno,
                                      doc=W_FFIObject.doc_errno,
                                      cls=W_FFIObject),
-        __gc_wref_remove = interp2app(W_FFIObject.descr___gc_wref_remove),
         addressof   = interp2app(W_FFIObject.descr_addressof),
         alignof     = interp2app(W_FFIObject.descr_alignof),
         buffer      = interp2app(W_FFIObject.descr_buffer),
@@ -592,6 +624,7 @@
         getctype    = interp2app(W_FFIObject.descr_getctype),
         integer_const = interp2app(W_FFIObject.descr_integer_const),
         new         = interp2app(W_FFIObject.descr_new),
+        new_allocator = interp2app(W_FFIObject.descr_new_allocator),
         new_handle  = interp2app(W_FFIObject.descr_new_handle),
         offsetof    = interp2app(W_FFIObject.descr_offsetof),
         sizeof      = interp2app(W_FFIObject.descr_sizeof),
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,13 +1,13 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.module._cffi_backend import ctypeobj, cdataobj
+from pypy.module._cffi_backend import ctypeobj, cdataobj, allocator
 
 
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_init=WrappedDefault(None))
 def newp(space, w_ctype, w_init):
-    return w_ctype.newp(w_init)
+    return w_ctype.newp(w_init, allocator.default_allocator)
 
 # ____________________________________________________________
 
@@ -18,9 +18,9 @@
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
-def callback(space, w_ctype, w_callable, w_error=None):
+def callback(space, w_ctype, w_callable, w_error=None, w_onerror=None):
     from pypy.module._cffi_backend.ccallback import W_CDataCallback
-    return W_CDataCallback(space, w_ctype, w_callable, w_error)
+    return W_CDataCallback(space, w_ctype, w_callable, w_error, w_onerror)
 
 # ____________________________________________________________
 
@@ -105,3 +105,9 @@
                     "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
+
+# ____________________________________________________________
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData)
+def gcp(space, w_cdata, w_destructor):
+    return w_cdata.with_gc(w_destructor)
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -60,12 +60,12 @@
             self.ffi, self.ctx.c_types, getarg(g.c_type_op))
         assert isinstance(rawfunctype, realize_c_type.W_RawFuncType)
         #
-        w_ct, locs = rawfunctype.unwrap_as_nostruct_fnptr(self.ffi)
+        rawfunctype.prepare_nostruct_fnptr(self.ffi)
         #
         ptr = rffi.cast(rffi.CCHARP, g.c_address)
         assert ptr
-        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn, w_ct,
-                                 locs, rawfunctype, fnname, self.libname)
+        return W_FunctionWrapper(self.space, ptr, g.c_size_or_direct_fn,
+                                 rawfunctype, fnname, self.libname)
 
     @jit.elidable_promote()
     def _get_attr_elidable(self, attr):
@@ -102,6 +102,8 @@
                 #
             elif op == cffi_opcode.OP_GLOBAL_VAR:
                 # A global variable of the exact type specified here
+                # (nowadays, only used by the ABI mode or backend
+                # compatibility; see OP_GLOBAL_F for the API mode
                 w_ct = realize_c_type.realize_c_type(
                     self.ffi, self.ctx.c_types, getarg(g.c_type_op))
                 g_size = rffi.cast(lltype.Signed, g.c_size_or_direct_fn)
@@ -113,7 +115,13 @@
                 ptr = rffi.cast(rffi.CCHARP, g.c_address)
                 if not ptr:   # for dlopen() style
                     ptr = self.cdlopen_fetch(attr)
-                w_result = cglob.W_GlobSupport(space, w_ct, ptr)
+                w_result = cglob.W_GlobSupport(space, attr, w_ct, ptr=ptr)
+                #
+            elif op == cffi_opcode.OP_GLOBAL_VAR_F:
+                w_ct = realize_c_type.realize_c_type(
+                    self.ffi, self.ctx.c_types, getarg(g.c_type_op))
+                w_result = cglob.W_GlobSupport(space, attr, w_ct,
+                                               fetch_addr=g.c_address)
                 #
             elif (op == cffi_opcode.OP_CONSTANT_INT or
                   op == cffi_opcode.OP_ENUM):
@@ -131,6 +139,9 @@
                     realize_c_type.FUNCPTR_FETCH_CHARP,
                     g.c_address)
                 if w_ct.size <= 0:
+                    raise oefmt(self.ffi.w_FFIError,
+                                "constant '%s' is of type '%s', "
+                                "whose size is not known", attr, w_ct.name)
                     raise oefmt(space.w_SystemError,
                                 "constant has no known size")
                 if not fetch_funcptr:   # for dlopen() style
@@ -172,7 +183,11 @@
             w_value = self._build_attr(attr)
             if w_value is None:
                 if is_getattr and attr == '__all__':
-                    return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
+                    return self.dir1(ignore_global_vars=True)
+                if is_getattr and attr == '__dict__':
+                    return self.full_dict_copy()
+                if is_getattr and attr == '__name__':
+                    return self.descr_repr()
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
@@ -202,16 +217,31 @@
     def descr_dir(self):
         return self.dir1()
 
-    def dir1(self, ignore_type=-1):
+    def dir1(self, ignore_global_vars=False):
         space = self.space
         total = rffi.getintfield(self.ctx, 'c_num_globals')
         g = self.ctx.c_globals
         names_w = []
         for i in range(total):
-            if getop(g[i].c_type_op) != ignore_type:
-                names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
+            if ignore_global_vars:
+                op = getop(g[i].c_type_op)
+                if (op == cffi_opcode.OP_GLOBAL_VAR or
+                    op == cffi_opcode.OP_GLOBAL_VAR_F):
+                    continue
+            names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
+    def full_dict_copy(self):
+        space = self.space
+        total = rffi.getintfield(self.ctx, 'c_num_globals')
+        g = self.ctx.c_globals
+        w_result = space.newdict()
+        for i in range(total):
+            w_attr = space.wrap(rffi.charp2str(g[i].c_name))
+            w_value = self._get_attr(w_attr)
+            space.setitem(w_result, w_attr, w_value)
+        return w_result
+
     def address_of_func_or_global_var(self, varname):
         # rebuild a string object from 'varname', to do typechecks and
         # to force a unicode back to a plain string
@@ -224,7 +254,8 @@
         if isinstance(w_value, W_FunctionWrapper):
             # '&func' returns a regular cdata pointer-to-function
             if w_value.directfnptr:
-                return W_CData(space, w_value.directfnptr, w_value.ctype)
+                ctype = w_value.typeof(self.ffi)
+                return W_CData(space, w_value.directfnptr, ctype)
             else:
                 return w_value    # backward compatibility
         #
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib import jit
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -135,8 +136,12 @@
 
 class W_RawFuncType(W_Root):
     """Temporary: represents a C function type (not a function pointer)"""
+
+    _immutable_fields_ = ['nostruct_ctype', 'nostruct_locs', 'nostruct_nargs']
     _ctfuncptr = None
-    _nostruct_ctfuncptr = (None, None)
+    nostruct_ctype = None
+    nostruct_locs = None
+    nostruct_nargs = 0
 
     def __init__(self, opcodes, base_index):
         self.opcodes = opcodes
@@ -168,14 +173,16 @@
         assert self._ctfuncptr is not None
         return self._ctfuncptr
 
-    def unwrap_as_nostruct_fnptr(self, ffi):
-        # tweaked version: instead of returning the ctfuncptr corresponding
-        # exactly to the OP_FUNCTION ... OP_FUNCTION_END opcodes, return
-        # another one in which the struct args are replaced with ptr-to-
-        # struct, and a struct return value is replaced with a hidden first
-        # arg of type ptr-to-struct.  This is how recompiler.py produces
+    @jit.dont_look_inside
+    def prepare_nostruct_fnptr(self, ffi):
+        # tweaked version: instead of returning the ctfuncptr
+        # corresponding exactly to the OP_FUNCTION ... OP_FUNCTION_END
+        # opcodes, this builds in self.nostruct_ctype another one in
+        # which the struct args are replaced with ptr-to- struct, and
+        # a struct return value is replaced with a hidden first arg of
+        # type ptr-to-struct.  This is how recompiler.py produces
         # trampoline functions for PyPy.
-        if self._nostruct_ctfuncptr[0] is None:
+        if self.nostruct_ctype is None:
             fargs, fret, ellipsis = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
             # containing 'A' where a struct argument was detected, and 'R'
@@ -198,8 +205,10 @@
                 locs = None
             else:
                 locs = ''.join(locs)
-            self._nostruct_ctfuncptr = (ctfuncptr, locs)
-        return self._nostruct_ctfuncptr
+            self.nostruct_ctype = ctfuncptr
+            self.nostruct_locs = locs
+            self.nostruct_nargs = len(ctfuncptr.fargs) - (locs is not None and
+                                                          locs[0] == 'R')
 
     def unexpected_fn_type(self, ffi):
         fargs, fret, ellipsis = self._unpack(ffi)
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -362,7 +362,7 @@
 
             case TOK_INTEGER:
                 errno = 0;
-#ifndef MS_WIN32
+#ifndef _MSC_VER
                 if (sizeof(length) > sizeof(unsigned long))
                     length = strtoull(tok->p, &endptr, 0);
                 else
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1170,6 +1170,14 @@
     BShort = new_primitive_type("short")
     BFunc = new_function_type((BShort,), BShort, False)
     f = callback(BFunc, Zcb1, -42)
+    #
+    seen = []
+    oops_result = None
+    def oops(*args):
+        seen.append(args)
+        return oops_result
+    ff = callback(BFunc, Zcb1, -42, oops)
+    #
     orig_stderr = sys.stderr
     orig_getline = linecache.getline
     try:
@@ -1195,6 +1203,59 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        assert len(seen) == 0
+        assert ff(bigvalue) == -42
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = 81
+        assert ff(bigvalue) == 81
+        oops_result = None
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = "xy"     # not None and not an int!
+        assert ff(bigvalue) == -42
+        oops_result = None
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+TypeError: $integer$
+""")
+        #
+        sys.stderr = cStringIO.StringIO()
+        seen = "not a list"    # this makes the oops() function crash
+        assert ff(bigvalue) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+Traceback (most recent call last):
+  File "$", line $, in oops
+    $
+AttributeError: 'str' object has no attribute 'append'
+""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline
@@ -2099,8 +2160,7 @@
     p = cast(BVoidP, 123456)
     py.test.raises(TypeError, "p[0]")
     p = cast(BVoidP, 0)
-    if 'PY_DOT_PY' in globals(): py.test.skip("NULL crashes early on py.py")
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises((TypeError, RuntimeError), "p[0]")
 
 def test_iter():
     BInt = new_primitive_type("int")
@@ -3333,6 +3393,38 @@
     check(4 | 8,  "CHB", "GTB")
     check(4 | 16, "CHB", "ROB")
 
+def test_dereference_null_ptr():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    p = cast(BIntPtr, 0)
+    py.test.raises(RuntimeError, "p[0]")
+    py.test.raises(RuntimeError, "p[0] = 42")
+    py.test.raises(RuntimeError, "p[42]")
+    py.test.raises(RuntimeError, "p[42] = -1")
+
+def test_mixup():
+    BStruct1 = new_struct_type("foo")
+    BStruct2 = new_struct_type("foo")   # <= same name as BStruct1
+    BStruct3 = new_struct_type("bar")
+    BStruct1Ptr = new_pointer_type(BStruct1)
+    BStruct2Ptr = new_pointer_type(BStruct2)
+    BStruct3Ptr = new_pointer_type(BStruct3)
+    BStruct1PtrPtr = new_pointer_type(BStruct1Ptr)
+    BStruct2PtrPtr = new_pointer_type(BStruct2Ptr)
+    BStruct3PtrPtr = new_pointer_type(BStruct3Ptr)
+    pp1 = newp(BStruct1PtrPtr)
+    pp2 = newp(BStruct2PtrPtr)
+    pp3 = newp(BStruct3PtrPtr)
+    pp1[0] = pp1[0]
+    e = py.test.raises(TypeError, "pp3[0] = pp1[0]")
+    assert str(e.value).startswith("initializer for ctype 'bar *' must be a ")
+    assert str(e.value).endswith(", not cdata 'foo *'")
+    e = py.test.raises(TypeError, "pp2[0] = pp1[0]")
+    assert str(e.value) == ("initializer for ctype 'foo *' appears indeed to "
+                            "be 'foo *', but the types are different (check "
+                            "that you are not e.g. mixing up different ffi "
+                            "instances)")
+
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.2"
+    assert __version__ == "1.2.0"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -114,6 +114,18 @@
         assert ffi.callback("int(int)", lambda x: x + "", -66)(10) == -66
         assert ffi.callback("int(int)", lambda x: x + "", error=-66)(10) == -66
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myerror(exc, val, tb):
+            seen.append(exc)
+        cb = ffi.callback("int(int)", lambda x: x + "", onerror=myerror)
+        assert cb(10) == 0
+        cb = ffi.callback("int(int)", lambda x:int(1E100), -66, onerror=myerror)
+        assert cb(10) == -66
+        assert seen == [TypeError, OverflowError]
+
     def test_ffi_callback_decorator(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
@@ -122,6 +134,37 @@
         assert deco(lambda x: x + "")(10) == -66
         assert deco(lambda x: x + 42)(10) == 52
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def oops(*args):
+            seen.append(args)
+
+        @ffi.callback("int(int)", onerror=oops)
+        def fn1(x):
+            return x + ""
+        assert fn1(10) == 0
+
+        @ffi.callback("int(int)", onerror=oops, error=-66)
+        def fn2(x):
+            return x + ""
+        assert fn2(10) == -66
+
+        assert len(seen) == 2
+        exc, val, tb = seen[0]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn1"
+        exc, val, tb = seen[1]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn2"
+        del seen[:]
+        #
+        raises(TypeError, ffi.callback, "int(int)",
+               lambda x: x, onerror=42)   # <- not callable
+
     def test_ffi_getctype(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
@@ -228,3 +271,99 @@
             import gc
             gc.collect()
         assert seen == [1]
+
+    def test_ffi_new_allocator_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        alloc1 = ffi.new_allocator()
+        alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
+        for retry in range(100):
+            p1 = alloc1("int[10]")
+            p2 = alloc2("int[10]")
+            combination = 0
+            for i in range(10):
+                assert p1[i] == 0
+                combination |= p2[i]
+                p1[i] = -42
+                p2[i] = -43
+            if combination != 0:
+                break
+            del p1, p2
+            import gc; gc.collect()
+        else:
+            raise AssertionError("cannot seem to get an int[10] not "
+                                 "completely cleared")
+
+    def test_ffi_new_allocator_2(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        def myfree(raw):
+            seen.append(raw)
+        alloc1 = ffi.new_allocator(myalloc, myfree)
+        alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+                                   should_clear_after_alloc=False)
+        p1 = alloc1("int[10]")
+        p2 = alloc2("int[]", 10)
+        assert seen == [40, 40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert ffi.typeof(p2) == ffi.typeof("int[]")
+        assert ffi.sizeof(p2) == 40
+        assert p1[5] == 0
+        assert p2[6] == ord('X') * 0x01010101
+        raw1 = ffi.cast("char *", p1)
+        raw2 = ffi.cast("char *", p2)
+        del p1, p2
+        retries = 0
+        while len(seen) != 4:
+            retries += 1
+            assert retries <= 5
+            import gc; gc.collect()
+        assert seen == [40, 40, raw1, raw2]
+        assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+        assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+    def test_ffi_new_allocator_3(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        alloc1 = ffi.new_allocator(myalloc)    # no 'free'
+        p1 = alloc1("int[10]")
+        assert seen == [40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert p1[5] == 0
+
+    def test_ffi_new_allocator_4(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(TypeError, ffi.new_allocator, free=lambda x: None)
+        #
+        def myalloc2(size):
+            raise LookupError
+        alloc2 = ffi.new_allocator(myalloc2)
+        raises(LookupError, alloc2, "int[5]")
+        #
+        def myalloc3(size):
+            return 42
+        alloc3 = ffi.new_allocator(myalloc3)
+        e = raises(TypeError, alloc3, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata object (got int)"
+        #
+        def myalloc4(size):
+            return ffi.cast("int", 42)
+        alloc4 = ffi.new_allocator(myalloc4)
+        e = raises(TypeError, alloc4, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+        #
+        def myalloc5(size):
+            return ffi.NULL
+        alloc5 = ffi.new_allocator(myalloc5)
+        raises(MemoryError, alloc5, "int[5]")
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -16,8 +16,8 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 0, 4):
-        py.test.skip("system cffi module needs to be at least 1.0.4")
+    if cffi.__version_info__ < (1, 2, 0):
+        py.test.skip("system cffi module needs to be at least 1.2.0")
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -276,6 +276,15 @@
         """)
         lib.aa = 5
         assert dir(lib) == ['aa', 'ff', 'my_constant']
+        #
+        aaobj = lib.__dict__['aa']
+        assert not isinstance(aaobj, int)    # some internal object instead
+        assert lib.__dict__ == {

From noreply at buildbot.pypy.org  Fri Jul 10 09:39:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Jul 2015 09:39:23 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.2: Make a release branch (but don't
	release right now)
Message-ID: <20150710073923.349241C03CA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.2
Changeset: r2226:6a2a331807e4
Date: 2015-07-10 09:39 +0200
http://bitbucket.org/cffi/cffi/changeset/6a2a331807e4/

Log:	Make a release branch (but don't release right now)


From noreply at buildbot.pypy.org  Fri Jul 10 12:02:12 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 10 Jul 2015 12:02:12 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: removed code comments for benchmarking
Message-ID: <20150710100212.F17A01C11C8@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78524:6494c7b82def
Date: 2015-07-10 11:42 +0200
http://bitbucket.org/pypy/pypy/changeset/6494c7b82def/

Log:	removed code comments for benchmarking

diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -28,42 +28,6 @@
 from rpython.rlib.entrypoint import all_jit_entrypoints,\
      annotated_jit_entrypoints
 
-from rpython.rlib.debug import debug_print, debug_start, debug_stop
-import time
-
-# XXX XXX XXX
-#class XXXBench(object):
-#    def __init__(self, name, uid, vec):
-#        self.t = []
-#        if name is None:
-#            name = "<unkown>"
-#        if uid is None:
-#            uid = 0
-#        self.name = str(name)
-#        self.unique_id = hex(uid)
-#        self.vec = vec
-#
-#    def xxx_clock_start(self):
-#        now = time.clock()
-#        self.t.append(now)
-#        debug_start("xxx-clock")
-#        debug_print("start name: %s id: %s clock: %f" % \
-#                (self.name, self.unique_id, now) )
-#        debug_stop("xxx-clock")
-#
-#    def xxx_clock_stop(self, fail=False):
-#        end = time.clock()
-#        assert len(self.t) > 0
-#        start = self.t[-1]
-#        del self.t[-1]
-#        ns = (end - start) * 10**9
-#        debug_start("xxx-clock")
-#        debug_print("stop name: %s id: %s clock: %f exe time: %dns fail? %d vec? %d" % \
-#                    (self.name, self.unique_id, end, int(ns), int(fail), int(self.vec)))
-#        debug_stop("xxx-clock")
-
-
-
 # ____________________________________________________________
 # Bootstrapping
 
@@ -438,9 +402,6 @@
         jd.portal_runner_ptr = "<not set so far>"
         jd.result_type = history.getkind(jd.portal_graph.getreturnvar()
                                          .concretetype)[0]
-        # XXX XXX XXX
-        #jd.xxxbench = XXXBench(jd.jitdriver.name, id(jd), jd.vectorize)
-        # XXX XXX XXX
         self.jitdrivers_sd.append(jd)
 
     def check_access_directly_sanity(self, graphs):
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -368,19 +368,12 @@
             if vinfo is not None:
                 virtualizable = args[index_of_virtualizable]
                 vinfo.clear_vable_token(virtualizable)
-
-            # XXX debug purpose only
-            #jitdriver_sd.xxxbench.xxx_clock_start()
-            # XXX debug purpose only end
-
+            #
             deadframe = func_execute_token(loop_token, *args)
             #
             # Record in the memmgr that we just ran this loop,
             # so that it will keep it alive for a longer time
             warmrunnerdesc.memory_manager.keep_loop_alive(loop_token)
-            # XXX debug purpose only
-            #jitdriver_sd.xxxbench.xxx_clock_stop(fail=True)
-            # XXX debug purpose only end
             #
             # Handle the failure
             fail_descr = cpu.get_latest_descr(deadframe)

From noreply at buildbot.pypy.org  Fri Jul 10 12:02:14 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 10 Jul 2015 12:02:14 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rpython hint
Message-ID: <20150710100214.4A86D1C11C8@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78525:a7b40371681b
Date: 2015-07-10 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/a7b40371681b/

Log:	rpython hint

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -6,7 +6,7 @@
 
 from rpython.conftest import option
 
-from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.metainterp.resoperation import ResOperation, GuardResOp, rop
 from rpython.jit.codewriter import heaptracker, longlong
 from rpython.rlib.objectmodel import compute_identity_hash
 import weakref
@@ -769,6 +769,7 @@
 
         for op in opt_ops:
             if op.is_guard():
+                assert isinstance(op, GuardResOp)
                 descr = op.getdescr()
                 if descr.loop_version():
                     # currently there is only ONE versioning,

From noreply at buildbot.pypy.org  Fri Jul 10 13:35:37 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 10 Jul 2015 13:35:37 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: copying operations if there are more
 versioned fail descrs, 
Message-ID: <20150710113537.8EBD71C1247@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78526:756c0b5ed2d7
Date: 2015-07-10 13:35 +0200
http://bitbucket.org/pypy/pypy/changeset/756c0b5ed2d7/

Log:	copying operations if there are more versioned fail descrs, added a
	test to check if array bounds checking can be improved

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -195,18 +195,18 @@
 
 def generate_pending_loop_versions(loop, jitdriver_sd, metainterp, jitcell_token):
     metainterp_sd = metainterp.staticdata
+    cpu = metainterp_sd.cpu
     if loop.versions is not None:
         token = jitcell_token
         for version in loop.versions:
-            for faildescr  in version.faildescrs:
+            for faildescr in version.faildescrs:
                 vl = create_empty_loop(metainterp)
                 vl.inputargs = version.inputargs
-                vl.operations = version.operations
+                vl.operations = version.copy_operations()
                 vl.original_jitcell_token = jitcell_token
                 send_bridge_to_backend(jitdriver_sd, metainterp_sd,
                                        faildescr, version.inputargs,
                                        version.operations, jitcell_token)
-                vl.original_jitcell_token = jitcell_token
                 record_loop_or_bridge(metainterp_sd, vl)
     loop.versions = None
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -780,6 +780,9 @@
                     op.setfailargs(version_failargs)
                     op.rd_snapshot = None
 
+    def copy_operations(self):
+        return [op.clone() for op in self.operations]
+
     def update_token(self, jitcell_token):
         label = self.operations[self.label_pos]
         jump = self.operations[-1]
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -171,6 +171,46 @@
         res = self.meta_interp(f, [60])
         assert res == f(60) == sum(range(60))
 
+    @py.test.mark.parametrize('i',[15])
+    def test_array_bounds_check_elimination(self,i):
+        myjitdriver = JitDriver(greens = [],
+                                reds = 'auto',
+                                vectorize=True)
+        T = lltype.Array(rffi.INT, hints={'nolength': True})
+        def f(d):
+            va = lltype.malloc(T, d, flavor='raw', zero=True)
+            vb = lltype.malloc(T, d, flavor='raw', zero=True)
+            for j in range(d):
+                va[j] = rffi.r_int(j)
+                vb[j] = rffi.r_int(j)
+            i = 0
+            while i < d:
+                myjitdriver.jit_merge_point()
+
+                if i < 0:
+                    raise IndexError
+                if i >= d:
+                    raise IndexError
+                a = va[i]
+                if i < 0:
+                    raise IndexError
+                if i >= d:
+                    raise IndexError
+                b = vb[i]
+                ec = intmask(a)+intmask(b)
+                if i < 0:
+                    raise IndexError
+                if i >= d:
+                    raise IndexError
+                va[i] = rffi.r_int(ec)
+
+                i += 1
+            lltype.free(va, flavor='raw')
+            lltype.free(vb, flavor='raw')
+            return 0
+        res = self.meta_interp(f, [i])
+        assert res == f(i)
+
 class VectorizeLLtypeTests(VectorizeTests):
     pass
 

From noreply at buildbot.pypy.org  Fri Jul 10 14:46:49 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 10 Jul 2015 14:46:49 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: Raise an error when indexing with 2
	ellipses
Message-ID: <20150710124649.26D691C033F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78527:a0c21224679b
Date: 2015-07-10 13:46 +0100
http://bitbucket.org/pypy/pypy/changeset/a0c21224679b/

Log:	Raise an error when indexing with 2 ellipses

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -9,7 +9,8 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray, \
     ArrayArgumentException, W_NumpyObject
 from pypy.module.micronumpy.iterators import ArrayIter
-from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
+from pypy.module.micronumpy.strides import (
+    Chunk, Chunks, NewAxisChunk, EllipsisChunk,
     RecordChunk, calc_strides, calc_new_strides, shape_agreement,
     calculate_broadcast_strides, calc_backstrides, calc_start, is_c_contiguous,
     is_f_contiguous)
@@ -204,6 +205,8 @@
                 if (isinstance(w_item, W_NDimArray) or
                     space.isinstance_w(w_item, space.w_list)):
                     raise ArrayArgumentException
+                elif space.is_w(w_item, space.w_Ellipsis):
+                    raise IndexError
             return self._lookup_by_index(space, view_w)
         if shape_len == 0:
             raise oefmt(space.w_IndexError, "too many indices for array")
@@ -240,8 +243,17 @@
             return Chunks([NewAxisChunk()])
         result = []
         i = 0
+        has_ellipsis = False
         for w_item in space.fixedview(w_idx):
-            if space.is_w(w_item, space.w_None):
+            if space.is_w(w_item, space.w_Ellipsis):
+                if has_ellipsis:
+                    # in CNumPy, this is only a deprecation warning
+                    raise oefmt(space.w_ValueError,
+                        "an index can only have a single Ellipsis (`...`); "
+                        "replace all but one with slices (`:`).")
+                result.append(EllipsisChunk())
+                has_ellipsis = True
+            elif space.is_w(w_item, space.w_None):
                 result.append(NewAxisChunk())
             else:
                 result.append(Chunk(*space.decode_index4(w_item,
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -82,6 +82,10 @@
     def __init__(self):
         pass
 
+class EllipsisChunk(BaseChunk):
+    def __init__(self):
+        pass
+
 
 class BaseTransform(object):
     pass
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2484,6 +2484,10 @@
         assert b.shape == b[...].shape
         assert (b == b[...]).all()
 
+        a = np.arange(6).reshape(2, 3)
+        if '__pypy__' in sys.builtin_module_names:
+            raises(ValueError, "a[..., ...]")
+
     def test_empty_indexing(self):
         import numpy as np
         r = np.ones(3)

From noreply at buildbot.pypy.org  Sat Jul 11 20:30:52 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 11 Jul 2015 20:30:52 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: progress
Message-ID: <20150711183052.72B3C1C1168@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78528:d702dcbd242c
Date: 2015-07-11 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/d702dcbd242c/

Log:	progress

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -34,10 +34,12 @@
         #      'cached_infos'.
         #
         self.cached_infos = []
+        self.cached_structs = []
         self._lazy_setfield = None
         self._lazy_setfield_registered = False
 
-    def register_dirty_field(self, info):
+    def register_dirty_field(self, structop, info):
+        self.cached_structs.append(structop)
         self.cached_infos.append(info)
 
     def invalidate(self, descr):
@@ -45,7 +47,50 @@
             assert isinstance(opinfo, info.AbstractStructPtrInfo)
             opinfo._fields[descr.get_index()] = None
         self.cached_infos = []
+        self.cached_structs = []
 
+    def produce_potential_short_preamble_ops(self, optimizer, shortboxes,
+                                             descr):
+        assert self._lazy_setfield is None
+        for i, info in enumerate(self.cached_infos):
+            structbox = self.cached_structs[i]
+            op = info._fields[descr.get_index()]
+            op = optimizer.get_box_replacement(op)
+            opnum = OpHelpers.getfield_for_descr(descr)
+            getfield_op = ResOperation(opnum, [structbox], descr=descr)
+            shortboxes.add_potential(op, getfield_op)
+        return
+        for structvalue in self._cached_fields_getfield_op.keys():
+            op = self._cached_fields_getfield_op[structvalue]
+            if not op:
+                continue
+            value = optimizer.getvalue(op.getarg(0))
+            if value in optimizer.opaque_pointers:
+                if value.getlevel() < LEVEL_KNOWNCLASS:
+                    continue
+                if op.getopnum() != rop.SETFIELD_GC and op.getopnum() != rop.GETFIELD_GC:
+                    continue
+            if structvalue in self._cached_fields:
+                if op.getopnum() == rop.SETFIELD_GC:
+                    result = op.getarg(1)
+                    if isinstance(result, Const):
+                        newresult = result.clonebox()
+                        optimizer.make_constant(newresult, result)
+                        result = newresult
+                    getop = ResOperation(rop.GETFIELD_GC, [op.getarg(0)],
+                                         result, op.getdescr())
+                    shortboxes.add_potential(getop, synthetic=True)
+                if op.getopnum() == rop.SETARRAYITEM_GC:
+                    result = op.getarg(2)
+                    if isinstance(result, Const):
+                        newresult = result.clonebox()
+                        optimizer.make_constant(newresult, result)
+                        result = newresult
+                    getop = ResOperation(rop.GETARRAYITEM_GC, [op.getarg(0), op.getarg(1)],
+                                         result, op.getdescr())
+                    shortboxes.add_potential(getop, synthetic=True)
+                elif op.result is not None:
+                    shortboxes.add_potential(op)
 
     def possible_aliasing(self, optheap, opinfo):
         # If lazy_setfield is set and contains a setfield on a different
@@ -135,7 +180,7 @@
 
     def _setfield(self, op, opinfo, optheap):
         arg = optheap.get_box_replacement(op.getarg(1))
-        opinfo.setfield(op.getdescr(), arg, optheap, self)
+        opinfo.setfield(op.getdescr(), op, arg, optheap, self)
 
 class ArrayCachedField(CachedField):
     def __init__(self, index):
@@ -150,13 +195,15 @@
 
     def _setfield(self, op, opinfo, optheap):
         arg = optheap.get_box_replacement(op.getarg(2))
-        opinfo.setitem(self.index, arg, self, optheap)
+        struct = optheap.get_box_replacement(op.getarg(0))
+        opinfo.setitem(self.index, struct, arg, self, optheap)
 
     def invalidate(self, descr):
         for opinfo in self.cached_infos:
             assert isinstance(opinfo, info.ArrayPtrInfo)
             opinfo._items = None
         self.cached_infos = []
+        self.cached_structs = []
 
 class OptHeap(Optimization):
     """Cache repeated heap accesses"""
@@ -203,7 +250,6 @@
             self.next_optimization.propagate_forward(postponed_op)
 
     def produce_potential_short_preamble_ops(self, sb):
-        return
         descrkeys = self.cached_fields.keys()
         if not we_are_translated():
             # XXX Pure operation of boxes that are cached in several places will
@@ -221,11 +267,11 @@
             for index, d in submap.items():
                 d.produce_potential_short_preamble_ops(self.optimizer, sb, descr)
 
-    def register_dirty_field(self, descr, info):
-        self.field_cache(descr).register_dirty_field(info)
+    def register_dirty_field(self, descr, op, info):
+        self.field_cache(descr).register_dirty_field(op, info)
 
-    def register_dirty_array_field(self, arraydescr, index, info):
-        self.arrayitem_cache(arraydescr, index).register_dirty_field(info)
+    def register_dirty_array_field(self, arraydescr, op, index, info):
+        self.arrayitem_cache(arraydescr, index).register_dirty_field(op, info)
 
     def clean_caches(self):
         del self._lazy_setfields_and_arrayitems[:]
@@ -467,7 +513,7 @@
         self.make_nonnull(op.getarg(0))
         self.emit_operation(op)
         # then remember the result of reading the field
-        structinfo.setfield(op.getdescr(), op, self, cf)
+        structinfo.setfield(op.getdescr(), op.getarg(0), op, self, cf)
     optimize_GETFIELD_GC_R = optimize_GETFIELD_GC_I
     optimize_GETFIELD_GC_F = optimize_GETFIELD_GC_I
 
@@ -518,7 +564,9 @@
         self.emit_operation(op)
         # the remember the result of reading the array item
         if cf is not None:
-            arrayinfo.setitem(indexb.getint(), self.get_box_replacement(op), cf,
+            arrayinfo.setitem(indexb.getint(),
+                              self.get_box_replacement(op.getarg(0)),
+                              self.get_box_replacement(op), cf,
                               self)
     optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I
     optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -122,12 +122,13 @@
         assert not self.is_virtual()
         self._fields = [None] * len(self._fields)
 
-    def setfield(self, descr, op, optheap=None, cf=None):
+    def setfield(self, descr, struct, op, optheap=None, cf=None):
         self.init_fields(descr.get_parent_descr(), descr.get_index())
+        assert isinstance(op, AbstractValue)
         self._fields[descr.get_index()] = op
         if cf is not None:
             assert not self.is_virtual()
-            cf.register_dirty_field(self)
+            cf.register_dirty_field(struct, self)
 
     def getfield(self, descr, optheap=None):
         self.init_fields(descr.get_parent_descr(), descr.get_index())
@@ -144,7 +145,7 @@
                                           descr=flddescr)
                 optforce._emit_operation(setfieldop)
                 if optforce.optheap is not None:
-                    optforce.optheap.register_dirty_field(flddescr, self)
+                    optforce.optheap.register_dirty_field(flddescr, op, self)
 
     def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         lst = self.vdescr.get_all_fielddescrs()
@@ -313,11 +314,11 @@
                                       descr=arraydescr)
                 optforce._emit_operation(setop)
                 if optforce.optheap is not None:
-                    optforce.optheap.register_dirty_array_field(
+                    optforce.optheap.register_dirty_array_field(op,
                         arraydescr, i, self)
         optforce.pure_from_args(rop.ARRAYLEN_GC, [op], ConstInt(len(self._items)))
 
-    def setitem(self, index, op, cf=None, optheap=None):
+    def setitem(self, index, struct_op, op, cf=None, optheap=None):
         if self._items is None:
             self._items = [None] * (index + 1)
         if index >= len(self._items):
@@ -325,7 +326,7 @@
         self._items[index] = op
         if cf is not None:
             assert not self.is_virtual()
-            cf.register_dirty_field(self)
+            cf.register_dirty_field(struct_op, self)
 
     def getitem(self, index, optheap=None):
         if self._items is None or index >= len(self._items):
@@ -450,9 +451,9 @@
         info = self._get_array_info(optheap)
         info.setitem(index, op, cf)
 
-    def setfield(self, descr, op, optheap=None, cf=None):
+    def setfield(self, descr, struct, op, optheap=None, cf=None):
         info = self._get_info(descr, optheap)
-        info.setfield(descr, op, optheap, cf)
+        info.setfield(descr, struct, op, optheap, cf)
 
     def is_null(self):
         return not bool(self._const.getref_base())
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -555,7 +555,9 @@
                 if val is None:
                     continue
                 if dest_info and dest_info.is_virtual():
-                    dest_info.setitem(index + dest_start, val)
+                    dest_info.setitem(index + dest_start,
+                                      self.get_box_replacement(op.getarg(2)),
+                                      val)
                 else:
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -123,3 +123,12 @@
         ptrinfo._fields = [None, None, None, p2]
         p.set_forwarded(ptrinfo)
         vs.make_inputargs([p, p], FakeOptimizer())
+
+    def test_short_boxes_heapcache(self):
+        loop = """
+        [p0, i1]
+        i0 = getfield_gc_i(p0, descr=valuedescr)
+        jump(p0, i0)
+        """
+        es, loop, preamble = self.optimize(loop)
+        assert es.short_boxes[preamble.operations[0]]
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -36,14 +36,19 @@
             preamble_op.info.make_guards(op, self.optunroll.short)
         return op
 
-    def setinfo_from_preamble(self, op, old_info):
+    def setinfo_from_preamble(self, op, preamble_info):
         op = self.get_box_replacement(op)
-        if isinstance(old_info, info.PtrInfo):
+        if isinstance(preamble_info, info.PtrInfo):
+            if preamble_info.is_virtual():
+                op.set_forwarded(preamble_info)
+                return
             if op.is_constant():
                 return # nothing we can learn
-            known_class = old_info.get_known_class(self.cpu)
+            known_class = preamble_info.get_known_class(self.cpu)
             if known_class:
                 self.make_constant_class(op, known_class, False)
+            if preamble_info.is_nonnull():
+                self.make_nonnull(op)
 
 
 class UnrollOptimizer(Optimization):
@@ -72,6 +77,7 @@
         self._check_no_forwarding([[start_label, end_label], ops])
         info, newops = self.optimizer.propagate_all_forward(
             start_label.getarglist()[:], ops)
+        self.flush()
         exported_state = self.export_state(start_label, end_label,
                                            info.inputargs)
         return exported_state, self.optimizer._newoperations
@@ -87,6 +93,7 @@
         jump_args = state.virtual_state.make_inputargs(jump_args,
                                                        self.optimizer,
                                                        force_boxes=True)
+        self.flush()
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer._newoperations.append(jump_op)
         return (UnrollInfo(self.make_short_preamble(start_label.getarglist())),
@@ -202,6 +209,10 @@
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
         label_args = virtual_state.make_inputargs(end_args, self.optimizer)
+        for arg in end_args:
+            if arg.get_forwarded() is not None:
+                arg.set_forwarded(None) # forget the optimization info
+                                        # (it's by infos exported)
         return ExportedState(label_args, inparg_mapping, virtual_state, infos,
                              sb.short_boxes, renamed_inputargs)
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -113,8 +113,9 @@
         newop.set_forwarded(vrefvalue)
         token = ResOperation(rop.FORCE_TOKEN, [])
         self.emit_operation(token)
-        vrefvalue.setfield(descr_virtual_token, token)
-        vrefvalue.setfield(descr_forced, self.optimizer.cpu.ts.CONST_NULLREF)
+        vrefvalue.setfield(descr_virtual_token, newop, token)
+        vrefvalue.setfield(descr_forced, newop,
+                           self.optimizer.cpu.ts.CONST_NULLREF)
 
     def optimize_VIRTUAL_REF_FINISH(self, op):
         # This operation is used in two cases.  In normal cases, it
@@ -198,12 +199,13 @@
     optimize_GETFIELD_GC_PURE_F = optimize_GETFIELD_GC_I
 
     def optimize_SETFIELD_GC(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        struct = op.getarg(0)
+        opinfo = self.getptrinfo(struct)
         if opinfo is not None and opinfo.is_virtual():
-            opinfo.setfield(op.getdescr(),
+            opinfo.setfield(op.getdescr(), struct,
                             self.get_box_replacement(op.getarg(1)))
         else:
-            self.make_nonnull(op.getarg(0))
+            self.make_nonnull(struct)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -309,6 +311,7 @@
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
                 opinfo.setitem(indexbox.getint(),
+                               self.get_box_replacement(op.getarg(0)),
                                self.get_box_replacement(op.getarg(2)))
                 return
         self.make_nonnull(op.getarg(0))

From noreply at buildbot.pypy.org  Sun Jul 12 20:11:17 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 12 Jul 2015 20:11:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a skipped failing test,
 maybe we want to implement it one day
Message-ID: <20150712181117.9B0EA1C026B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78529:356912fca370
Date: 2015-07-12 10:05 +0200
http://bitbucket.org/pypy/pypy/changeset/356912fca370/

Log:	Add a skipped failing test, maybe we want to implement it one day

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8637,5 +8637,27 @@
         """
         self.optimize_loop(ops, expected, preamble)
 
+    def test_getfield_proven_constant(self):
+        py.test.skip("not working")
+        ops = """
+        [p0]
+        i1 = getfield_gc(p0, descr=valuedescr)
+        guard_value(i1, 13) []
+        escape(i1)
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        escape(13)
+        jump(p0)
+        """
+        expected_short = """
+        [p0]
+        i1 = getfield_gc(p0, descr=valuedescr)
+        guard_value(i1, 13) []
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected, expected_short=expected_short)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Sun Jul 12 20:11:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 12 Jul 2015 20:11:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150712181119.8108E1C026B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78530:9f734aff75ff
Date: 2015-07-12 20:11 +0200
http://bitbucket.org/pypy/pypy/changeset/9f734aff75ff/

Log:	merge

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -23,14 +23,14 @@
 default_modules.update([
     "_codecs", "gc", "_weakref", "marshal", "errno", "imp", "math", "cmath",
     "_sre", "_pickle_support", "operator", "parser", "symbol", "token", "_ast",
-    "_io", "_random", "__pypy__", "_testing"
+    "_io", "_random", "__pypy__", "_testing", "time"
 ])
 
 
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -297,7 +297,12 @@
         options = make_dict(config)
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
+        if config.objspace.usemodules._cffi_backend:
+            self.hack_for_cffi_modules(driver)
 
+        return self.get_entry_point(config)
+    
+    def hack_for_cffi_modules(self, driver):
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_c to build_cffi_imports
         # this should probably get cleaned up and merged with driver.create_exe
@@ -336,8 +341,6 @@
         driver.default_goal = 'build_cffi_imports'
         # HACKHACKHACK end
 
-        return self.get_entry_point(config)
-
     def jitpolicy(self, driver):
         from pypy.module.pypyjit.policy import PyPyJitPolicy
         from pypy.module.pypyjit.hooks import pypy_hooks
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -90,6 +90,16 @@
     def _convert_error(self, expected, w_got):
         space = self.space
         if isinstance(w_got, cdataobj.W_CData):
+            if self.name == w_got.ctype.name:
+                # in case we'd give the error message "initializer for
+                # ctype 'A' must be a pointer to same type, not cdata
+                # 'B'", but with A=B, then give instead a different error
+                # message to try to clear up the confusion
+                return oefmt(space.w_TypeError,
+                             "initializer for ctype '%s' appears indeed to "
+                             "be '%s', but the types are different (check "
+                             "that you are not e.g. mixing up different ffi "
+                             "instances)", self.name, w_got.ctype.name)
             return oefmt(space.w_TypeError,
                          "initializer for ctype '%s' must be a %s, not cdata "
                          "'%s'", self.name, expected, w_got.ctype.name)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3402,6 +3402,29 @@
     py.test.raises(RuntimeError, "p[42]")
     py.test.raises(RuntimeError, "p[42] = -1")
 
+def test_mixup():
+    BStruct1 = new_struct_type("foo")
+    BStruct2 = new_struct_type("foo")   # <= same name as BStruct1
+    BStruct3 = new_struct_type("bar")
+    BStruct1Ptr = new_pointer_type(BStruct1)
+    BStruct2Ptr = new_pointer_type(BStruct2)
+    BStruct3Ptr = new_pointer_type(BStruct3)
+    BStruct1PtrPtr = new_pointer_type(BStruct1Ptr)
+    BStruct2PtrPtr = new_pointer_type(BStruct2Ptr)
+    BStruct3PtrPtr = new_pointer_type(BStruct3Ptr)
+    pp1 = newp(BStruct1PtrPtr)
+    pp2 = newp(BStruct2PtrPtr)
+    pp3 = newp(BStruct3PtrPtr)
+    pp1[0] = pp1[0]
+    e = py.test.raises(TypeError, "pp3[0] = pp1[0]")
+    assert str(e.value).startswith("initializer for ctype 'bar *' must be a ")
+    assert str(e.value).endswith(", not cdata 'foo *'")
+    e = py.test.raises(TypeError, "pp2[0] = pp1[0]")
+    assert str(e.value) == ("initializer for ctype 'foo *' appears indeed to "
+                            "be 'foo *', but the types are different (check "
+                            "that you are not e.g. mixing up different ffi "
+                            "instances)")
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "1.2.0"
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -23,11 +23,16 @@
 # running make inside the src dir
 DYNAMIC_VMPROF = False
 
+if sys.platform.startswith('linux'):
+    libs = ['dl']
+else:
+    libs = []
+
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['vmprof.h', 'trampoline.h'],
     separate_module_files = [SRC.join('trampoline.vmprof.s')],
-    libraries = ['dl'],
+    libraries = libs,
     
     post_include_bits=["""
         int pypy_vmprof_init(void);
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
--- a/pypy/module/_vmprof/src/config.h
+++ b/pypy/module/_vmprof/src/config.h
@@ -1,2 +1,6 @@
 #define HAVE_SYS_UCONTEXT_H
+#if defined(__FreeBSD__) || defined(__APPLE__)
+#define PC_FROM_UCONTEXT uc_mcontext.mc_rip
+#else
 #define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
+#endif
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -33,6 +33,9 @@
 //#include <libunwind.h>
 
 #include "vmprof.h"
+#if defined(__FreeBSD__) || defined(__APPLE__)
+#define sighandler_t sig_t
+#endif
 
 #define _unused(x) ((void)x)
 
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -540,6 +540,15 @@
         for v in [float('inf'), float('-inf'), float('nan'), float('-nan')]:
             assert math.isnan(fmod(v, 2))
 
+    def test_mod(self):
+        from numpy import mod
+        assert mod(5, 3) == 2
+        assert mod(5, -3) == -1
+        assert mod(-5, 3) == 1
+        assert mod(-5, -3) == -2
+        assert mod(2.5, 1) == 0.5
+        assert mod(-1.5, 2) == 0.5
+
     def test_minimum(self):
         from numpy import array, minimum, nan, isnan
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -759,7 +759,21 @@
 
     @simple_binary_op
     def mod(self, v1, v2):
-        return math.fmod(v1, v2)
+        # partial copy of pypy.objspace.std.floatobject.W_FloatObject.descr_mod
+        if v2 == 0.0:
+            return rfloat.NAN
+        mod = math.fmod(v1, v2)
+        if mod:
+            # ensure the remainder has the same sign as the denominator
+            if (v2 < 0.0) != (mod < 0.0):
+                mod += v2
+        else:
+            # the remainder is zero, and in the presence of signed zeroes
+            # fmod returns different results across platforms; ensure
+            # it has the same sign as the denominator; we'd like to do
+            # "mod = v2 * 0.0", but that may get optimized away
+            mod = rfloat.copysign(0.0, v2)
+        return mod
 
     @simple_binary_op
     def pow(self, v1, v2):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -238,7 +238,7 @@
         log = self.run("""
         def main(n):
             for i in xrange(n):
-                unicode('abc')
+                unicode(str(i))
             return i
         """, [1000])
         loop, = log.loops_by_filename(self.filepath)
@@ -248,10 +248,10 @@
         i50 = int_add(i47, 1)
         setfield_gc(p15, i50, descr=<FieldS pypy.module.__builtin__.functional.W_XRangeIterator.inst_current 8>)
         guard_not_invalidated(descr=...)
-        p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=<Callr . rii EF=5>)
+        p80 = call(ConstClass(ll_str__IntegerR_SignedConst_Signed), i47, descr=<Callr . i EF=3>)
         guard_no_exception(descr=...)
-        p53 = getfield_gc_pure(p52, descr=<FieldP tuple2.item0 .>)
-        guard_nonnull(p53, descr=...)
+        p53 = call(ConstClass(fast_str_decode_ascii), p80, descr=<Callr . r EF=4>)
+        guard_no_exception(descr=...)
         --TICK--
         jump(..., descr=...)
         """)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -6,7 +6,7 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
-    unicode_encode_ascii, unicode_encode_utf_8)
+    unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
 
 from pypy.interpreter import unicodehelper
 from pypy.interpreter.baseobjspace import W_Root
@@ -481,9 +481,13 @@
         if encoding == 'ascii':
             # XXX error handling
             s = space.charbuf_w(w_obj)
-            eh = unicodehelper.decode_error_handler(space)
-            return space.wrap(str_decode_ascii(
-                    s, len(s), None, final=True, errorhandler=eh)[0])
+            try:
+                u = fast_str_decode_ascii(s)
+            except ValueError:
+                eh = unicodehelper.decode_error_handler(space)
+                u = str_decode_ascii(     # try again, to get the error right
+                    s, len(s), None, final=True, errorhandler=eh)[0]
+            return space.wrap(u)
         if encoding == 'utf-8':
             s = space.charbuf_w(w_obj)
             eh = unicodehelper.decode_error_handler(space)
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1009,6 +1009,16 @@
             result.append(r)
     return result.build(), pos
 
+# An elidable version, for a subset of the cases
+ at jit.elidable
+def fast_str_decode_ascii(s):
+    result = UnicodeBuilder(len(s))
+    for c in s:
+        if ord(c) >= 128:
+            raise ValueError
+        result.append(unichr(ord(c)))
+    return result.build()
+
 
 # Specialize on the errorhandler when it's a constant
 @specialize.arg_or_var(3)
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -139,6 +139,12 @@
             for encoding in "utf-8 latin-1 ascii".split():
                 self.checkdecode(chr(i), encoding)
 
+    def test_fast_str_decode_ascii(self):
+        u = runicode.fast_str_decode_ascii("abc\x00\x7F")
+        assert type(u) is unicode
+        assert u == u"abc\x00\x7F"
+        py.test.raises(ValueError, runicode.fast_str_decode_ascii, "ab\x80")
+
     def test_all_first_256(self):
         for i in range(256):
             for encoding in ("utf-7 utf-8 latin-1 utf-16 utf-16-be utf-16-le "

From noreply at buildbot.pypy.org  Sun Jul 12 20:23:11 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 12 Jul 2015 20:23:11 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: a first go at adding proper
 operations to label args, not sure what I think about this go
Message-ID: <20150712182311.2466F1C101F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78531:078fc2be0b85
Date: 2015-07-12 20:23 +0200
http://bitbucket.org/pypy/pypy/changeset/078fc2be0b85/

Log:	a first go at adding proper operations to label args, not sure what
	I think about this go

diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -16,8 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
-            ('earlyforce', OptEarlyForce),
             ('pure', OptPure),
+            #('earlyforce', OptEarlyForce), # XXX why do we have this hack?
             ('heap', OptHeap),
             ('unroll', None)]
 # no direct instantiation of unroll
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -58,7 +58,9 @@
             op = optimizer.get_box_replacement(op)
             opnum = OpHelpers.getfield_for_descr(descr)
             getfield_op = ResOperation(opnum, [structbox], descr=descr)
-            shortboxes.add_potential(op, getfield_op)
+            if not op.is_constant():
+                # XXXX why?
+                shortboxes.add_potential(op, getfield_op)
         return
         for structvalue in self._cached_fields_getfield_op.keys():
             op = self._cached_fields_getfield_op[structvalue]
@@ -151,7 +153,13 @@
         return op.getarg(1)
 
     def _getfield(self, opinfo, descr, optheap):
-        return opinfo.getfield(descr, optheap)
+        from rpython.jit.metainterp.optimizeopt.unroll import PreambleOp
+        
+        res = opinfo.getfield(descr, optheap)
+        if isinstance(res, PreambleOp):
+            res = optheap.optimizer.force_op_from_preamble(res)
+            opinfo._fields[descr.get_index()] == res
+        return res
 
     def force_lazy_setfield(self, optheap, descr, can_cache=True):
         op = self._lazy_setfield
@@ -180,7 +188,8 @@
 
     def _setfield(self, op, opinfo, optheap):
         arg = optheap.get_box_replacement(op.getarg(1))
-        opinfo.setfield(op.getdescr(), op, arg, optheap, self)
+        struct = optheap.get_box_replacement(op.getarg(0))
+        opinfo.setfield(op.getdescr(), struct, arg, optheap, self)
 
 class ArrayCachedField(CachedField):
     def __init__(self, index):
@@ -531,6 +540,7 @@
     optimize_GETFIELD_GC_PURE_F = optimize_GETFIELD_GC_PURE_I
 
     def optimize_SETFIELD_GC(self, op):
+        self.setfield(op)
         #opnum = OpHelpers.getfield_pure_for_descr(op.getdescr())
         #if self.has_pure_result(opnum, [op.getarg(0)],
         #                        op.getdescr()):
@@ -538,6 +548,8 @@
         #             (op.getdescr().repr_of_descr()))
         #    raise BogusImmutableField
         #
+
+    def setfield(self, op):
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -51,6 +51,9 @@
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         return None
 
+    def make_guards(self, op, short):
+        pass # XXX
+
     
 class NonNullPtrInfo(PtrInfo):
     _attrs_ = ('last_guard_pos',)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -302,6 +302,17 @@
     def force_box(self, op):
         op = self.get_box_replacement(op)
         info = op.get_forwarded()
+        if self.optunroll and self.optunroll.ops_to_import:
+            # XXX hack, use stuff on info somehow, a bit on the hard side
+            #     but doable :-)
+            try:
+                preamble_op = self.optunroll.ops_to_import[op]
+            except KeyError:
+                pass
+            else:
+                self.optunroll.short.append(preamble_op)
+                self.optunroll.extra_label_args.append(op)
+                del self.optunroll.ops_to_import[op]
         if info is not None:
             return info.force_box(op, self)
         return op
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -1,5 +1,6 @@
 
-from rpython.jit.metainterp.resoperation import ResOperation, OpHelpers
+from rpython.jit.metainterp.resoperation import ResOperation, OpHelpers,\
+     AbstractInputArg
 from rpython.jit.metainterp.history import Const
 
 
@@ -29,13 +30,17 @@
         self.short_boxes[op] = short_op
 
     def produce_short_preamble_op(self, op, preamble_op):
-        for arg in op.getarglist():
-            if isinstance(arg, Const):
-                pass
-            elif arg in self.ops_used:
-                pass
-            else:
-                return # can't produce
+        if isinstance(op, AbstractInputArg):
+            if op not in self.ops_used:
+                return
+        else:
+            for arg in op.getarglist():
+                if isinstance(arg, Const):
+                    pass
+                elif arg in self.ops_used:
+                    pass
+                else:
+                    return # can't produce
         if op in self.short_boxes:
             opnum = OpHelpers.same_as_for_type(op.type)
             same_as_op = ResOperation(opnum, [op])
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -450,8 +450,9 @@
         preamble.inputargs = start_state.renamed_inputargs
         start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs)
         preamble.operations = [start_label] + preamble_ops
-        emit_end_label = ResOperation(rop.LABEL, start_state.end_args)
-        loop.inputargs = start_state.end_args
+        inputargs = start_state.end_args + loop_info.extra_label_args
+        emit_end_label = ResOperation(rop.LABEL, inputargs)
+        loop.inputargs = inputargs
         loop.operations = [emit_end_label] + ops
         return Info(preamble, loop_info.short_preamble)
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,6 +16,16 @@
 
 
 class PreambleOp(AbstractResOp):
+    """ An operations that's only found in preamble and not
+    in the list of constructed operations. When encountered (can be found
+    either in pure ops or heap ops), it must be put in inputargs as well
+    as short preamble (together with corresponding guards). Extra_ops is
+    for extra things to be found in the label, for now only inputargs
+    of the preamble that have to be propagated further.
+
+    See force_op_from_preamble for details how the extra things are put.
+    """
+    
     def __init__(self, op, preamble_op, info):
         self.op = op
         self.preamble_op = preamble_op
@@ -57,6 +67,7 @@
     distinction anymore)"""
 
     inline_short_preamble = True
+    ops_to_import = None
 
     def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
         self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
@@ -84,19 +95,23 @@
 
     def optimize_peeled_loop(self, start_label, end_jump, ops, state):
         self.short = []
+        self.extra_label_args = []
         self._check_no_forwarding([[start_label, end_jump], ops])
         self.import_state(start_label, state)
         self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops,
                                              rename_inputargs=False)
         jump_args = [self.get_box_replacement(op)
                      for op in end_jump.getarglist()]
+        args_from_extras = [self.get_box_replacement(op) for op in
+                            self.extra_label_args]
         jump_args = state.virtual_state.make_inputargs(jump_args,
-                                                       self.optimizer,
-                                                       force_boxes=True)
+                    self.optimizer, force_boxes=True) + args_from_extras
+        
         self.flush()
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer._newoperations.append(jump_op)
-        return (UnrollInfo(self.make_short_preamble(start_label.getarglist())),
+        return (UnrollInfo(self.make_short_preamble(start_label.getarglist()),
+                           self.extra_label_args),
                 self.optimizer._newoperations)
 
     def make_short_preamble(self, args):
@@ -264,17 +279,24 @@
     def import_state(self, targetop, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit
+        self.ops_to_import = {}
         for source, target in exported_state.inputarg_mapping:
             if source is not target:
                 source.set_forwarded(target)
         # import the optimizer state, starting from boxes that can be produced
         # by short preamble
         for op, preamble_op in exported_state.short_boxes.items():
+            self.ops_to_import[op] = preamble_op
             if preamble_op.is_always_pure():
-                self.pure(op.getopnum(), PreambleOp(op, preamble_op,
-                                                self.optimizer.getinfo(op)))
+                self.pure(op.getopnum(), op)
             else:
-                yyy
+                assert preamble_op.is_getfield()
+                optheap = self.optimizer.optheap
+                if optheap is None:
+                    continue
+                opinfo = self.optimizer.ensure_ptr_info_arg0(preamble_op)
+                assert not opinfo.is_virtual()
+                opinfo._fields[preamble_op.getdescr().get_index()] = op
 
         for op, info in exported_state.exported_infos.iteritems():
             self.optimizer.setinfo_from_preamble(op, info)
@@ -714,9 +736,11 @@
     """ A state after optimizing the peeled loop, contains the following:
 
     * short_preamble - list of operations that go into short preamble
+    * extra_label_args - list of extra operations that go into the label
     """
-    def __init__(self, short_preamble):
+    def __init__(self, short_preamble, extra_label_args):
         self.short_preamble = short_preamble
+        self.extra_label_args = extra_label_args
             
 class ExportedState(LoopInfo):
     """ Exported state consists of a few pieces of information:
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -535,8 +535,9 @@
 class JitHintError(Exception):
     """Inconsistency in the JIT hints."""
 
+# XXX earlyforce before pure
 ENABLE_ALL_OPTS = (
-    'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll')
+    'intbounds:rewrite:virtualize:string:pure:heap:unroll')
 
 PARAMETER_DOCS = {
     'threshold': 'number of times a loop has to run for it to become hot',

From noreply at buildbot.pypy.org  Sun Jul 12 20:45:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 12 Jul 2015 20:45:13 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: another go,
 do it in two steps. There is an open question whether we're getting
 better results or we have bugs
Message-ID: <20150712184513.0F5B21C0135@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78532:e127ddd8a9ec
Date: 2015-07-12 20:45 +0200
http://bitbucket.org/pypy/pypy/changeset/e127ddd8a9ec/

Log:	another go, do it in two steps. There is an open question whether
	we're getting better results or we have bugs

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -854,14 +854,13 @@
         jump(i1, p2, p2sub)
         """
         expected = """
-        [i1, p2, p2sub]
-        i3 = getfield_gc_i(p2sub, descr=valuedescr)
+        [i1, p2, i3]
         escape_n(i3)
         p1 = new_with_vtable(descr=nodesize)
         p3sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        jump(i1, p1, p3sub)
+        jump(i1, p1, i1)
         """
         self.optimize_loop(ops, expected, preamble)
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -223,6 +223,8 @@
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
+        for box in sb.short_boxes:
+            infos[box] = self.optimizer.getinfo(box)
         label_args = virtual_state.make_inputargs(end_args, self.optimizer)
         for arg in end_args:
             if arg.get_forwarded() is not None:
@@ -278,28 +280,32 @@
 
     def import_state(self, targetop, exported_state):
         # the mapping between input args (from old label) and what we need
-        # to actually emit
+        # to actually emit. Update the info
         self.ops_to_import = {}
         for source, target in exported_state.inputarg_mapping:
             if source is not target:
                 source.set_forwarded(target)
+            info = exported_state.exported_infos.get(target, None)
+            if info is not None:
+                self.optimizer.setinfo_from_preamble(source, info)
         # import the optimizer state, starting from boxes that can be produced
         # by short preamble
         for op, preamble_op in exported_state.short_boxes.items():
             self.ops_to_import[op] = preamble_op
             if preamble_op.is_always_pure():
-                self.pure(op.getopnum(), op)
+                self.pure(op.getopnum(), PreambleOp(op, preamble_op,
+                                exported_state.exported_infos.get(op, None)))
             else:
                 assert preamble_op.is_getfield()
                 optheap = self.optimizer.optheap
                 if optheap is None:
                     continue
                 opinfo = self.optimizer.ensure_ptr_info_arg0(preamble_op)
+                pre_info = exported_state.exported_infos[op]
+                pop = PreambleOp(op, preamble_op, pre_info)
                 assert not opinfo.is_virtual()
-                opinfo._fields[preamble_op.getdescr().get_index()] = op
+                opinfo._fields[preamble_op.getdescr().get_index()] = pop
 
-        for op, info in exported_state.exported_infos.iteritems():
-            self.optimizer.setinfo_from_preamble(op, info)
         return
         self.inputargs = targetop.getarglist()
         target_token = targetop.getdescr()

From noreply at buildbot.pypy.org  Sun Jul 12 21:21:24 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 12 Jul 2015 21:21:24 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: Handle field access for record dtypes
 as a special case in getitem/setitem
Message-ID: <20150712192124.47EE41C03F4@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78533:723d0dacd7da
Date: 2015-07-12 21:20 +0200
http://bitbucket.org/pypy/pypy/changeset/723d0dacd7da/

Log:	Handle field access for record dtypes as a special case in
	getitem/setitem

diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py
--- a/pypy/module/micronumpy/arrayops.py
+++ b/pypy/module/micronumpy/arrayops.py
@@ -162,8 +162,10 @@
         shape = [arr.get_shape()[0] * repeats]
         w_res = W_NDimArray.from_shape(space, shape, arr.get_dtype(), w_instance=arr)
         for i in range(repeats):
-            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
-                 orig_size)]).apply(space, w_res).implementation.setslice(space, arr)
+            chunks = Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                 orig_size)])
+            view = chunks.apply(space, w_res)
+            view.implementation.setslice(space, arr)
     else:
         axis = space.int_w(w_axis)
         shape = arr.get_shape()[:]
@@ -174,7 +176,8 @@
         for i in range(repeats):
             chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
                                  orig_size)
-            Chunks(chunks).apply(space, w_res).implementation.setslice(space, arr)
+            view = Chunks(chunks).apply(space, w_res)
+            view.implementation.setslice(space, arr)
     return w_res
 
 
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -218,16 +218,10 @@
     @jit.unroll_safe
     def _prepare_slice_args(self, space, w_idx):
         if space.isinstance_w(w_idx, space.w_str):
-            idx = space.str_w(w_idx)
-            dtype = self.dtype
-            if not dtype.is_record():
-                raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
-                    "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
-                    "boolean arrays are valid indices")
-            elif idx not in dtype.fields:
-                raise oefmt(space.w_ValueError, "field named %s not found", idx)
-            return RecordChunk(idx)
-        elif (space.isinstance_w(w_idx, space.w_int) or
+            raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
+                "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
+                "boolean arrays are valid indices")
+        if (space.isinstance_w(w_idx, space.w_int) or
                 space.isinstance_w(w_idx, space.w_slice)):
             if len(self.get_shape()) == 0:
                 raise oefmt(space.w_ValueError, "cannot slice a 0-d array")
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -18,8 +18,9 @@
 from pypy.module.micronumpy.converters import multi_axis_converter, \
     order_converter, shape_converter, searchside_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
-from pypy.module.micronumpy.strides import get_shape_from_iterable, \
-    shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
+from pypy.module.micronumpy.strides import (
+    get_shape_from_iterable, shape_agreement, shape_agreement_multiple,
+    is_c_contiguous, is_f_contiguous, RecordChunk)
 from pypy.module.micronumpy.casting import can_cast_array
 
 
@@ -203,6 +204,10 @@
                                prefix)
 
     def descr_getitem(self, space, w_idx):
+        if self.get_dtype().is_record():
+            if space.isinstance_w(w_idx, space.w_str):
+                idx = space.str_w(w_idx)
+                return self.getfield(space, idx)
         if space.is_w(w_idx, space.w_Ellipsis):
             return self
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool():
@@ -229,6 +234,13 @@
         self.implementation.setitem_index(space, index_list, w_value)
 
     def descr_setitem(self, space, w_idx, w_value):
+        if self.get_dtype().is_record():
+            if space.isinstance_w(w_idx, space.w_str):
+                idx = space.str_w(w_idx)
+                view = self.getfield(space, idx)
+                w_value = convert_to_array(space, w_value)
+                view.implementation.setslice(space, w_value)
+                return
         if space.is_w(w_idx, space.w_Ellipsis):
             self.implementation.setslice(space, convert_to_array(space, w_value))
             return
@@ -241,6 +253,13 @@
         except ArrayArgumentException:
             self.setitem_array_int(space, w_idx, w_value)
 
+    def getfield(self, space, field):
+        dtype = self.get_dtype()
+        if field not in dtype.fields:
+            raise oefmt(space.w_ValueError, "field named %s not found", field)
+        chunks = RecordChunk(field)
+        return chunks.apply(space, self)
+
     def descr_delitem(self, space, w_idx):
         raise OperationError(space.w_ValueError, space.wrap(
             "cannot delete array elements"))

From noreply at buildbot.pypy.org  Sun Jul 12 22:12:21 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Jul 2015 22:12:21 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: use, fix title in
 monkeypatched _usefields; add test_bad_param
Message-ID: <20150712201221.6590D1C0135@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78534:6caae7e7edd6
Date: 2015-07-10 15:28 +0300
http://bitbucket.org/pypy/pypy/changeset/6caae7e7edd6/

Log:	use, fix title in monkeypatched _usefields; add test_bad_param

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -643,7 +643,10 @@
             if space.isinstance_w(w_fldname, space.w_tuple):
                 fldlist = space.listview(w_fldname)
                 fldnames[i] = space.str_w(fldlist[0])
-                titles[i] = space.str_w(fldlist[1])
+                if space.is_w(fldlist[1], space.w_None):
+                    titles[i] = None
+                else:
+                    titles[i] = space.str_w(fldlist[1])
                 if len(fldlist) != 2:
                     raise oefmt(space.w_TypeError, "data type not understood")
             elif space.isinstance_w(w_fldname, space.w_str): 
@@ -722,15 +725,14 @@
             alignment = -1
         format = dtype_from_spec(space, obj[0], alignment=alignment)
         if len(obj) > 2:
-            title = obj[2]
+            title = space.wrap(obj[2])
         else:
-            title = None
-        allfields.append((fname, format, num, title))
+            title = space.w_None
+        allfields.append((space.wrap(fname), format, num, title))
     allfields.sort(key=lambda x: x[2])
-    names   = [x[0] for x in allfields]
+    names   = [space.newtuple([x[0], x[3]]) for x in allfields]
     formats = [x[1] for x in allfields]
     offsets = [x[2] for x in allfields]
-    titles  = [x[3] for x in allfields]
     aslist = []
     if align:
         alignment = 0
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1307,7 +1307,7 @@
                        'formats':['i4', 'u1'],
                        'offsets':[0, 4]}, align=True)
         assert dt.itemsize == 8
-        dt = np.dtype([('f0', 'i4'), ('f1', 'u1')], align=True)
+        dt = np.dtype({'f0': ('i4', 0), 'f1':('u1', 4)}, align=True)
         assert dt.itemsize == 8
         assert dt.alignment == 4
         assert str(dt) == "{'names':['f0','f1'], 'formats':['<i4','u1'], 'offsets':[0,4], 'itemsize':8, 'aligned':True}"
@@ -1357,6 +1357,27 @@
         assert dt1 == dt2
         assert dt2 == dt3
 
+    def test_bad_param(self):
+        import numpy as np
+        # Can't give a size that's too small
+        raises(ValueError, np.dtype,
+                        {'names':['f0', 'f1'],
+                         'formats':['i4', 'i1'],
+                         'offsets':[0, 4],
+                         'itemsize':4})
+        # If alignment is enabled, the alignment (4) must divide the itemsize
+        raises(ValueError, np.dtype,
+                        {'names':['f0', 'f1'],
+                         'formats':['i4', 'i1'],
+                         'offsets':[0, 4],
+                         'itemsize':9}, align=True)
+        # If alignment is enabled, the individual fields must be aligned
+        raises(ValueError, np.dtype,
+                        {'names':['f0', 'f1'],
+                         'formats':['i1', 'f4'],
+                         'offsets':[0, 2]}, align=True)
+
+
 
 class AppTestNotDirect(BaseNumpyAppTest):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Sun Jul 12 22:12:22 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Jul 2015 22:12:22 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: add arg checking logic,
 allow specifying itemsize
Message-ID: <20150712201222.9AF861C0135@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78535:1f2bc798e9d3
Date: 2015-07-11 20:38 +0300
http://bitbucket.org/pypy/pypy/changeset/1f2bc798e9d3/

Log:	add arg checking logic, allow specifying itemsize

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -615,10 +615,12 @@
 
 
 @specialize.arg(2)
-def dtype_from_list(space, w_lst, simple, alignment, offsets=None):
+def dtype_from_list(space, w_lst, simple, alignment, offsets=None, itemsize=0):
     lst_w = space.listview(w_lst)
     fields = {}
+    use_supplied_offsets = True
     if offsets is None:
+        use_supplied_offsets = False
         offsets = [0] * len(lst_w)
     maxalign = alignment 
     fldnames = [''] * len(lst_w)
@@ -663,12 +665,13 @@
             delta = subdtype.alignment
             # Set offset to the next power-of-two above delta
             delta = (delta + maxalign -1) & (-maxalign)
-            if delta > offsets[i]:
-                for j in range(i):
-                    offsets[j+1] = delta + offsets[j]
-            if  i + 1 < len(offsets) and offsets[i + 1] == 0:
-                offsets[i + 1] = offsets[i] + max(delta, subdtype.elsize)
-        else:
+            if not use_supplied_offsets:
+                if delta > offsets[i]:
+                    for j in range(i):
+                        offsets[j+1] = delta + offsets[j]
+                if  i + 1 < len(offsets) and offsets[i + 1] == 0:
+                    offsets[i + 1] = offsets[i] + max(delta, subdtype.elsize)
+        elif not use_supplied_offsets:
             if  i + 1 < len(offsets) and offsets[i + 1] == 0:
                 offsets[i+1] = offsets[i] + subdtype.elsize
         subdtypes[i] = subdtype
@@ -680,6 +683,10 @@
             subdtype.alignment = maxalign
         if fldnames[i] in fields:
             raise oefmt(space.w_ValueError, "two fields with the same name")
+        if maxalign > 1 and offsets[i] % maxalign:
+            raise oefmt(space.w_ValueError, "offset %d for NumPy dtype with "
+                    "fields is not divisible by the field alignment %d "
+                    "with align=True", offsets[i], maxalign)
         fields[fldnames[i]] = offsets[i], subdtype
         if titles[i] is not None:
             if titles[i] in fields:
@@ -687,6 +694,17 @@
             fields[titles[i]] = offsets[i], subdtype
         names.append((fldnames[i], titles[i]))
     total = offsets[-1] + max(maxalign, fields[names[-1][0]][1].elsize)
+    if itemsize > 1:
+        if total > itemsize:
+            raise oefmt(space.w_ValueError,
+                     "NumPy dtype descriptor requires %d bytes, cannot"
+                     " override to smaller itemsize of %d", total, itemsize)
+        if alignment >= 0 and itemsize % maxalign:
+            raise oefmt(space.w_ValueError,
+                    "NumPy dtype descriptor requires alignment of %d bytes, "
+                    "which is not divisible into the specified itemsize %d",
+                    maxalign, itemsize) 
+        total = itemsize
     retval = W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
                    names=names, fields=fields, elsize=total)
     if alignment >=0:
@@ -751,6 +769,7 @@
     titles_w = _get_list_or_none(space, w_dict, 'titles')
     metadata_w = _get_val_or_none(space, w_dict, 'metadata')
     aligned_w = _get_val_or_none(space, w_dict, 'align')
+    itemsize_w = _get_val_or_none(space, w_dict, 'itemsize')
     if names_w is None or formats_w is None:
         if we_are_translated():
             return get_appbridge_cache(space).call_method(space,
@@ -782,9 +801,14 @@
             _names_w.append(space.newtuple([names_w[i], titles_w[i]]))
         names_w = _names_w
     aslist = []
+    if itemsize_w is None:
+        itemsize = 0
+    else:
+        itemsize = space.int_w(itemsize_w)
     for i in range(min(len(names_w), len(formats_w))):
         aslist.append(space.newtuple([names_w[i], formats_w[i]]))
-    retval = dtype_from_list(space, space.newlist(aslist), False, alignment, offsets=offsets)
+    retval = dtype_from_list(space, space.newlist(aslist), False, alignment,
+                             offsets=offsets, itemsize=itemsize)
     if metadata_w is not None:
         retval.descr_set_metadata(space, metadata_w)
     retval.flags |= NPY.NEEDS_PYAPI

From noreply at buildbot.pypy.org  Sun Jul 12 22:12:23 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Jul 2015 22:12:23 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: add more failing tests
Message-ID: <20150712201223.BE2A91C0135@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78536:7d6215d740d5
Date: 2015-07-11 22:34 +0300
http://bitbucket.org/pypy/pypy/changeset/7d6215d740d5/

Log:	add more failing tests

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -488,6 +488,11 @@
                 assert np.dtype(o).str == '|O8'
             else:
                 assert False,'self._ptr_size unknown'
+        # Issue gh-2798
+        a = np.array(['a'], dtype="O").astype(("O", [("name", "O")]))
+        assert a[0] == 'a'
+        assert a == 'a'
+        assert a['name'].dtype == a.dtype
 
 class AppTestTypes(BaseAppTestDtypes):
     def test_abstract_types(self):
@@ -1376,7 +1381,66 @@
                         {'names':['f0', 'f1'],
                          'formats':['i1', 'f4'],
                          'offsets':[0, 2]}, align=True)
+        dt = np.dtype(np.double)
+        attr = ["subdtype", "descr", "str", "name", "base", "shape",
+                "isbuiltin", "isnative", "isalignedstruct", "fields",
+                "metadata", "hasobject"]
+        for s in attr:
+            raises(AttributeError, delattr, dt, s)
 
+        raises(TypeError, np.dtype,
+            dict(names=set(['A', 'B']), formats=['f8', 'i4']))
+        raises(TypeError, np.dtype,
+            dict(names=['A', 'B'], formats=set(['f8', 'i4'])))
+
+    def test_complex_dtype_repr(self):
+        import numpy as np
+        dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
+                                ('rtile', '>f4', (64, 36))], (3,)),
+                       ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
+                                   ('bright', '>f4', (8, 36))])])
+        assert repr(dt) == (
+                     "dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)), "
+                     "('rtile', '>f4', (64, 36))], (3,)), "
+                     "('bottom', [('bleft', ('>f4', (8, 64)), (1,)), "
+                     "('bright', '>f4', (8, 36))])])")
+
+        dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'],
+                        'offsets': [0, 1, 2],
+                        'titles': ['Red pixel', 'Green pixel', 'Blue pixel']},
+                        align=True)
+        assert repr(dt) == (
+                    "dtype([(('Red pixel', 'r'), 'u1'), "
+                    "(('Green pixel', 'g'), 'u1'), "
+                    "(('Blue pixel', 'b'), 'u1')], align=True)")
+
+        dt = np.dtype({'names': ['rgba', 'r', 'g', 'b'],
+                       'formats': ['<u4', 'u1', 'u1', 'u1'],
+                       'offsets': [0, 0, 1, 2],
+                       'titles': ['Color', 'Red pixel',
+                                  'Green pixel', 'Blue pixel']}, align=True)
+        assert repr(dt) == (
+                    "dtype({'names':['rgba','r','g','b'],"
+                    " 'formats':['<u4','u1','u1','u1'],"
+                    " 'offsets':[0,0,1,2],"
+                    " 'titles':['Color','Red pixel',"
+                              "'Green pixel','Blue pixel'],"
+                    " 'itemsize':4}, align=True)")
+
+        dt = np.dtype({'names': ['r', 'b'], 'formats': ['u1', 'u1'],
+                        'offsets': [0, 2],
+                        'titles': ['Red pixel', 'Blue pixel'],
+                        'itemsize': 4})
+        assert repr(dt) == (
+                    "dtype({'names':['r','b'], "
+                    "'formats':['u1','u1'], "
+                    "'offsets':[0,2], "
+                    "'titles':['Red pixel','Blue pixel'], "
+                    "'itemsize':4})")
+
+        dt = np.dtype([('a', '<M8[D]'), ('b', '<m8[us]')])
+        assert repr(dt) == (
+                    "dtype([('a', '<M8[D]'), ('b', '<m8[us]')])")
 
 
 class AppTestNotDirect(BaseNumpyAppTest):

From noreply at buildbot.pypy.org  Sun Jul 12 22:12:24 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Jul 2015 22:12:24 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: split out _get_shape, wip
Message-ID: <20150712201224.DD21B1C0135@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78537:51bdd6b20e88
Date: 2015-07-11 22:34 +0300
http://bitbucket.org/pypy/pypy/changeset/51bdd6b20e88/

Log:	split out _get_shape, wip

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -865,6 +865,33 @@
         dtype.descr_set_metadata(space, w_metadata)
     return dtype
 
+def _get_shape(space, w_shape):
+    if w_shape is None:
+        return None
+    if space.isinstance_w(w_shape, space.w_int):
+        dim = space.int_w(w_shape)
+        return [dim]
+    shape_w = space.fixedview(w_shape)
+    if len(shape_w) == 1:
+        if not space.isinstance_w(shape_w[0], space.w_int):
+            return None
+    shape = []
+    for w_dim in shape_w:
+        try:
+            dim = space.int_w(w_dim)
+        except OperationError as e:
+            if e.match(space, space.w_OverflowError):
+                raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple.")
+            else:
+                raise
+        if dim > 2 ** 32 -1:
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                  "dimension does not fit into a C int.")
+        elif dim < 0:
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                  "dimension smaller than zero.")
+        shape.append(dim)
+    return shape
 
 @unwrap_spec(align=bool, copy=bool)
 def descr__new__(space, w_subtype, w_dtype, align=False, copy=False,
@@ -878,33 +905,15 @@
 
 def make_new_dtype(space, w_subtype, w_dtype, alignment, copy=False, w_shape=None, w_metadata=None):
     cache = get_dtype_cache(space)
-    if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or
-                                space.len_w(w_shape) > 0):
+    shape = _get_shape(space, w_shape)
+    if shape is not None:
         subdtype = make_new_dtype(space, w_subtype, w_dtype, alignment, copy, w_metadata=w_metadata)
+        if len(shape) == 1 and shape[0] == 1:
+            print '_get_shape returned', shape
+            return subdtype
+        print 'uhh, _get_shape returned', shape
         assert isinstance(subdtype, W_Dtype)
-        size = 1
-        if space.isinstance_w(w_shape, space.w_int):
-            dim = space.int_w(w_shape)
-            if dim == 1:
-                return subdtype
-            w_shape = space.newtuple([w_shape])
-        shape = []
-        for w_dim in space.fixedview(w_shape):
-            try:
-                dim = space.int_w(w_dim)
-            except OperationError as e:
-                if e.match(space, space.w_OverflowError):
-                    raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple.")
-                else:
-                    raise
-            if dim > 2 ** 32 -1:
-                raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
-                      "dimension does not fit into a C int.")
-            elif dim < 0:
-                raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
-                      "dimension smaller than zero.")
-            shape.append(dim)
-            size *= dim
+        size = support.product(shape)
         size *= subdtype.elsize
         if size >= 2 ** 31:
             raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
@@ -993,8 +1002,9 @@
                            W_Dtype.descr_set_names,
                            W_Dtype.descr_del_names),
     metadata = GetSetProperty(W_Dtype.descr_get_metadata,
-                           W_Dtype.descr_set_metadata,
-                           W_Dtype.descr_del_metadata),
+                           #W_Dtype.descr_set_metadata,
+                           #W_Dtype.descr_del_metadata,
+                            ),
     flags = GetSetProperty(W_Dtype.descr_get_flags),
 
     __eq__ = interp2app(W_Dtype.descr_eq),

From noreply at buildbot.pypy.org  Sun Jul 12 22:12:26 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Jul 2015 22:12:26 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: finish _get_shape
Message-ID: <20150712201226.0B9DD1C0135@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78538:639b44c09f75
Date: 2015-07-12 20:33 +0300
http://bitbucket.org/pypy/pypy/changeset/639b44c09f75/

Log:	finish _get_shape

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -870,10 +870,15 @@
         return None
     if space.isinstance_w(w_shape, space.w_int):
         dim = space.int_w(w_shape)
+        if dim == 1:
+            return None
         return [dim]
     shape_w = space.fixedview(w_shape)
+    if len(shape_w) < 1:
+        return None
     if len(shape_w) == 1:
-        if not space.isinstance_w(shape_w[0], space.w_int):
+        if (not space.isinstance_w(shape_w[0], space.w_int) and
+            not space.isinstance_w(shape_w[0], space.w_long)):
             return None
     shape = []
     for w_dim in shape_w:
@@ -908,10 +913,6 @@
     shape = _get_shape(space, w_shape)
     if shape is not None:
         subdtype = make_new_dtype(space, w_subtype, w_dtype, alignment, copy, w_metadata=w_metadata)
-        if len(shape) == 1 and shape[0] == 1:
-            print '_get_shape returned', shape
-            return subdtype
-        print 'uhh, _get_shape returned', shape
         assert isinstance(subdtype, W_Dtype)
         size = support.product(shape)
         size *= subdtype.elsize
@@ -921,7 +922,6 @@
         return _set_metadata_and_copy(space, w_metadata,
                W_Dtype(types.VoidType(space), space.gettypefor(boxes.W_VoidBox),
                        shape=shape, subdtype=subdtype, elsize=size))
-
     if space.is_none(w_dtype):
         return cache.w_float64dtype
     if space.isinstance_w(w_dtype, w_subtype):

From noreply at buildbot.pypy.org  Sun Jul 12 22:12:27 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Jul 2015 22:12:27 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: give up with (base_dtype,
 new_dtype) union spcecification for now, test for bad param
Message-ID: <20150712201227.2A49A1C0135@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78539:c7e922056cff
Date: 2015-07-12 23:10 +0300
http://bitbucket.org/pypy/pypy/changeset/c7e922056cff/

Log:	give up with (base_dtype, new_dtype) union spcecification for now,
	test for bad param

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -729,6 +729,8 @@
     w_val = _get_val_or_none(space, w_dict, key)
     if w_val is None:
         return None
+    if space.isinstance_w(w_val, space.w_set):
+        raise oefmt(space.w_TypeError, "'set' object does not support indexing")
     return space.listview(w_val)
 
 def _usefields(space, w_dict, align):
@@ -922,6 +924,17 @@
         return _set_metadata_and_copy(space, w_metadata,
                W_Dtype(types.VoidType(space), space.gettypefor(boxes.W_VoidBox),
                        shape=shape, subdtype=subdtype, elsize=size))
+    elif w_shape is not None and not space.isinstance_w(w_shape, space.w_int):
+        spec = space.listview(w_shape)
+        if len(spec) > 0:
+            # this is (base_dtype, new_dtype) so just make it a union by setting both
+            # parts' offset to 0
+            try:
+                dtype1 = make_new_dtype(space, w_subtype, w_shape, alignment)
+            except:
+                raise
+            raise oefmt(space.w_NotImplementedError, 
+                "(base_dtype, new_dtype) dtype spectification discouraged, not implemented")
     if space.is_none(w_dtype):
         return cache.w_float64dtype
     if space.isinstance_w(w_dtype, w_subtype):
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -489,6 +489,8 @@
             else:
                 assert False,'self._ptr_size unknown'
         # Issue gh-2798
+        if '__pypy__' in sys.builtin_module_names:
+            skip("(base_dtype, new_dtype) dtype specification discouraged")
         a = np.array(['a'], dtype="O").astype(("O", [("name", "O")]))
         assert a[0] == 'a'
         assert a == 'a'

From noreply at buildbot.pypy.org  Sun Jul 12 22:47:53 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Jul 2015 22:47:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: test,
	fix for pypy/numpy issue  number 34
Message-ID: <20150712204753.6FFCE1C026B@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78540:1f1d7e1c14c2
Date: 2015-07-12 23:48 +0300
http://bitbucket.org/pypy/pypy/changeset/1f1d7e1c14c2/

Log:	test, fix for pypy/numpy issue number 34

diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -77,7 +77,7 @@
     stop = 1
     step = 1
     lgt = 1
-    axis_step = 0
+    axis_step = 0 # both skip this axis in calculate_slice_strides and set stride => 0
 
     def __init__(self):
         pass
@@ -127,7 +127,7 @@
         except IndexError:
             continue
         if chunk.step != 0:
-            rstrides[j] = s_i * chunk.step
+            rstrides[j] = s_i * chunk.step * chunk.axis_step
             rbackstrides[j] = s_i * max(0, chunk.lgt - 1) * chunk.step
             rshape[j] = chunk.lgt
             j += 1
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -764,6 +764,8 @@
         assert (a[1:] == b).all()
         assert (a[1:,newaxis] == d).all()
         assert (a[newaxis,1:] == c).all()
+        assert a.strides == (8,)
+        assert a[:, newaxis].strides == (8, 0)
 
     def test_newaxis_assign(self):
         from numpy import array, newaxis

From noreply at buildbot.pypy.org  Sun Jul 12 23:35:23 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Jul 2015 23:35:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: test, fix ndarray.data for slices;
 pypy/numpy issue number 37.
Message-ID: <20150712213523.5C4EF1C1C72@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78541:0a3ca9a1ac09
Date: 2015-07-13 00:35 +0300
http://bitbucket.org/pypy/pypy/changeset/0a3ca9a1ac09/

Log:	test, fix ndarray.data for slices; pypy/numpy issue number 37.

	This unsafe interface will fail for non-contiguous views and can
	cause buffer under/overruns, do we really need it?

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2347,6 +2347,7 @@
         assert a[1] == 0xff
         assert len(a.data) == 16
         assert type(a.data) is buffer
+        assert a[1:].data._pypy_raw_address() - a.data._pypy_raw_address() == a.strides[0]
 
     def test_explicit_dtype_conversion(self):
         from numpy import array

From noreply at buildbot.pypy.org  Sun Jul 12 23:36:35 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Jul 2015 23:36:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: missed this file in 1f1d7e1c14c2
Message-ID: <20150712213635.1FD281C1C72@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78542:9eb2238aaf11
Date: 2015-07-13 00:37 +0300
http://bitbucket.org/pypy/pypy/changeset/9eb2238aaf11/

Log:	missed this file in 1f1d7e1c14c2

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -617,15 +617,17 @@
         self.impl = impl
         self.readonly = readonly
 
-    def getitem(self, item):
-        return raw_storage_getitem(lltype.Char, self.impl.storage, item)
+    def getitem(self, index):
+        return raw_storage_getitem(lltype.Char, self.impl.storage,
+                 index + self.impl.start)
 
-    def setitem(self, item, v):
-        raw_storage_setitem(self.impl.storage, item,
+    def setitem(self, index, v):
+        raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
     def getlength(self):
-        return self.impl.size
+        return self.impl.size - self.impl.start
 
     def get_raw_address(self):
-        return self.impl.storage
+        from rpython.rtyper.lltypesystem import rffi
+        return rffi.ptradd(self.impl.storage, self.impl.start)

From noreply at buildbot.pypy.org  Mon Jul 13 09:32:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 13 Jul 2015 09:32:19 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: small fixes
Message-ID: <20150713073219.6CD501C101F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78543:e6d31ebd3035
Date: 2015-07-13 09:32 +0200
http://bitbucket.org/pypy/pypy/changeset/e6d31ebd3035/

Log:	small fixes

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -306,11 +306,10 @@
             # XXX hack, use stuff on info somehow, a bit on the hard side
             #     but doable :-)
             try:
-                preamble_op = self.optunroll.ops_to_import[op]
+                self.optunroll.ops_to_import[op]
             except KeyError:
                 pass
             else:
-                self.optunroll.short.append(preamble_op)
                 self.optunroll.extra_label_args.append(op)
                 del self.optunroll.ops_to_import[op]
         if info is not None:
@@ -468,6 +467,11 @@
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
         return BasicLoopInfo(newargs), self._newoperations
 
+    def _clean_optimization_info(self, lst):
+        for op in lst:
+            if op.get_forwarded() is not None:
+                op.set_forwarded(None)
+
     def send_extra_operation(self, op):
         self.first_optimization.propagate_forward(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8103,9 +8103,9 @@
         jump(i1, i3)
         """
         expected = """
-        [i1, i2, i6, i3]
-        call_n(i6, descr=nonwritedescr)
-        jump(i1, i3, i6, i3)
+        [i1, i2, i3]
+        call_n(i3, descr=nonwritedescr)
+        jump(i1, i2, i3)
         """
         short = """
         [i1, i2]
@@ -8113,7 +8113,7 @@
         i4 = int_add(i3, i3)
         i5 = int_add(i4, i4)
         i6 = int_add(i5, i5)
-        jump(i1, i2, i6, i3)
+        #jump(i1, i2, i6, i3)
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -3,9 +3,9 @@
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
 from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes
 from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.jit.metainterp.optimizeopt import info
+from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
-     Optimization, LoopInfo
+     Optimization, LoopInfo, MININT, MAXINT
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
         BadVirtualState, VirtualStatesCantMatch)
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
@@ -50,6 +50,7 @@
         op = self.get_box_replacement(op)
         if isinstance(preamble_info, info.PtrInfo):
             if preamble_info.is_virtual():
+                # XXX do we want to sanitize this?
                 op.set_forwarded(preamble_info)
                 return
             if op.is_constant():
@@ -59,6 +60,15 @@
                 self.make_constant_class(op, known_class, False)
             if preamble_info.is_nonnull():
                 self.make_nonnull(op)
+        elif isinstance(preamble_info, intutils.IntBound):
+            if preamble_info.lower > MININT/2 or preamble_info.upper < MAXINT/2:
+                intbound = self.getintbound(op)
+                if preamble_info.lower > MININT/2:
+                    intbound.has_lower = True
+                    intbound.lower = preamble_info.lower
+                if preamble_info.upper < MAXINT/2:
+                    intbound.has_upper = True
+                    intbound.upper = preamble_info.upper
 
 
 class UnrollOptimizer(Optimization):
@@ -88,9 +98,12 @@
         self._check_no_forwarding([[start_label, end_label], ops])
         info, newops = self.optimizer.propagate_all_forward(
             start_label.getarglist()[:], ops)
-        self.flush()
+        self.optimizer.flush()
         exported_state = self.export_state(start_label, end_label,
                                            info.inputargs)
+        # we need to absolutely make sure that we've cleaned up all
+        # the optimization info
+        self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
 
     def optimize_peeled_loop(self, start_label, end_jump, ops, state):
@@ -107,7 +120,7 @@
         jump_args = state.virtual_state.make_inputargs(jump_args,
                     self.optimizer, force_boxes=True) + args_from_extras
         
-        self.flush()
+        self.optimizer.flush()
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer._newoperations.append(jump_op)
         return (UnrollInfo(self.make_short_preamble(start_label.getarglist()),
@@ -226,10 +239,8 @@
         for box in sb.short_boxes:
             infos[box] = self.optimizer.getinfo(box)
         label_args = virtual_state.make_inputargs(end_args, self.optimizer)
-        for arg in end_args:
-            if arg.get_forwarded() is not None:
-                arg.set_forwarded(None) # forget the optimization info
-                                        # (it's by infos exported)
+        self.optimizer._clean_optimization_info(end_args)
+        self.optimizer._clean_optimization_info(start_label.getarglist())
         return ExportedState(label_args, inparg_mapping, virtual_state, infos,
                              sb.short_boxes, renamed_inputargs)
 

From noreply at buildbot.pypy.org  Mon Jul 13 09:33:08 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 13 Jul 2015 09:33:08 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: call flush from
	propagate_all_forward
Message-ID: <20150713073308.BBE991C101F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78544:3319421c0bf3
Date: 2015-07-13 09:33 +0200
http://bitbucket.org/pypy/pypy/changeset/3319421c0bf3/

Log:	call flush from propagate_all_forward

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -464,6 +464,7 @@
         #self.loop.operations = self.get_newoperations()
         #self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
+        self.flush()
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
         return BasicLoopInfo(newargs), self._newoperations
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -98,7 +98,6 @@
         self._check_no_forwarding([[start_label, end_label], ops])
         info, newops = self.optimizer.propagate_all_forward(
             start_label.getarglist()[:], ops)
-        self.optimizer.flush()
         exported_state = self.export_state(start_label, end_label,
                                            info.inputargs)
         # we need to absolutely make sure that we've cleaned up all
@@ -120,7 +119,6 @@
         jump_args = state.virtual_state.make_inputargs(jump_args,
                     self.optimizer, force_boxes=True) + args_from_extras
         
-        self.optimizer.flush()
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer._newoperations.append(jump_op)
         return (UnrollInfo(self.make_short_preamble(start_label.getarglist()),

From noreply at buildbot.pypy.org  Mon Jul 13 14:56:16 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 13 Jul 2015 14:56:16 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: test and a fix
Message-ID: <20150713125616.2DE411C026B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78545:4b5a8cedb410
Date: 2015-07-13 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/4b5a8cedb410/

Log:	test and a fix

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -132,3 +132,15 @@
         """
         es, loop, preamble = self.optimize(loop)
         assert es.short_boxes[preamble.operations[0]]
+
+    def test_int_is_true(self):
+        loop = """
+        [i0]
+        i1 = int_is_true(i0)
+        guard_true(i1) []
+        jump(i0)
+        """
+        es, loop, preamble = self.optimize(loop)
+        op = preamble.operations[0]
+        assert es.short_boxes == {op:op}
+        assert es.exported_infos[op].is_constant()
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -43,6 +43,7 @@
         op = preamble_op.op
         self.optunroll.short.append(preamble_op.preamble_op)
         if preamble_op.info:
+            self.setinfo_from_preamble(op, preamble_op.info)
             preamble_op.info.make_guards(op, self.optunroll.short)
         return op
 

From noreply at buildbot.pypy.org  Mon Jul 13 17:20:00 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 13 Jul 2015 17:20:00 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: handle consts better
Message-ID: <20150713152000.DEB361C026B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78546:cde47437fc67
Date: 2015-07-13 17:20 +0200
http://bitbucket.org/pypy/pypy/changeset/cde47437fc67/

Log:	handle consts better

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -58,9 +58,7 @@
             op = optimizer.get_box_replacement(op)
             opnum = OpHelpers.getfield_for_descr(descr)
             getfield_op = ResOperation(opnum, [structbox], descr=descr)
-            if not op.is_constant():
-                # XXXX why?
-                shortboxes.add_potential(op, getfield_op)
+            shortboxes.add_potential(op, getfield_op)
         return
         for structvalue in self._cached_fields_getfield_op.keys():
             op = self._cached_fields_getfield_op[structvalue]
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -8,6 +8,7 @@
 class ShortBoxes(object):
     def __init__(self):
         self.potential_ops = []
+        self.produced_short_boxes = {}
         self.ops_used = {}
         self.extra_same_as = []
 
@@ -16,21 +17,25 @@
             self.ops_used[box] = None
         optimizer.produce_potential_short_preamble_ops(self)
 
-        self.short_boxes = {}
-        # short boxes has a map of "op from preamble" ->
-        # "op going to short preamble", where "op from preamble" can be
-        # anything, but the one going to short_preamble has to be either pure
+        self.short_boxes = []
+        # short boxes is a list of (op, preamble_op)
+        # where op can be
+        # anything, but the preamble_op has to be either pure
         # or a heap cache op
 
         for op, preamble_op in self.potential_ops:
             self.produce_short_preamble_op(op, preamble_op)
+        self.produced_short_boxes = None
         return self.short_boxes
 
     def add_to_short(self, op, short_op):
-        self.short_boxes[op] = short_op
+        self.short_boxes.append((op, short_op))
+        self.produced_short_boxes[op] = None
 
     def produce_short_preamble_op(self, op, preamble_op):
-        if isinstance(op, AbstractInputArg):
+        if isinstance(op, Const):
+            pass
+        elif isinstance(op, AbstractInputArg):
             if op not in self.ops_used:
                 return
         else:
@@ -41,7 +46,7 @@
                     pass
                 else:
                     return # can't produce
-        if op in self.short_boxes:
+        if op in self.produced_short_boxes:
             opnum = OpHelpers.same_as_for_type(op.type)
             same_as_op = ResOperation(opnum, [op])
             self.extra_same_as.append(same_as_op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_short.py b/rpython/jit/metainterp/optimizeopt/test/test_short.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_short.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_short.py
@@ -28,7 +28,7 @@
         op = ResOperation(rop.INT_ADD, [i0, i1])
         sb = ShortBoxes()
         sb.create_short_boxes(Opt([op]), [i0, i1])
-        assert sb.short_boxes == {op: op}
+        assert sb.short_boxes == [(op, op)]
 
     def test_pure_ops_does_not_work(self):
         i0 = InputArgInt()
@@ -36,7 +36,7 @@
         op = ResOperation(rop.INT_ADD, [i0, i1])
         sb = ShortBoxes()
         sb.create_short_boxes(Opt([op]), [i0])
-        assert sb.short_boxes == {}
+        assert sb.short_boxes == []
 
     def test_multiple_similar_ops(self):
         """ This can happen e.g. if heap cache and pure ops produce
@@ -56,10 +56,9 @@
         sb = ShortBoxes()
         sb.create_short_boxes(Opt([op, (op, op1)]), [i0, i1])
         assert len(sb.short_boxes) == 2
-        l = [x.getopnum() for x in sb.short_boxes.keys()]
+        l = [x.getopnum() for x, _ in sb.short_boxes]
         l.sort()
         assert l == [rop.INT_ADD, rop.SAME_AS_I]
-        assert op1 in sb.short_boxes.values()
-        assert op in sb.short_boxes.values()
-        assert op in sb.short_boxes.keys()
+        assert [x for x, y in sb.short_boxes][0] == op
+        assert [y for x, y in sb.short_boxes] == [op, op1]
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -69,7 +69,7 @@
         # we have exported values for i1, which happens to be an inputarg
         assert es.inputarg_mapping[0][1].getint() == 1
         assert isinstance(es.inputarg_mapping[0][1], ConstInt)
-        assert es.short_boxes == {}
+        assert es.short_boxes == []
 
     def test_not_constant(self):
         loop = """
@@ -82,7 +82,7 @@
         assert isinstance(vs.state[0], NotVirtualStateInfo)
         assert vs.state[0].level == LEVEL_UNKNOWN
         op = preamble.operations[0]
-        assert es.short_boxes == {op: op}
+        assert es.short_boxes == [(op, op)]
 
     def test_guard_class(self):
         loop = """
@@ -131,7 +131,7 @@
         jump(p0, i0)
         """
         es, loop, preamble = self.optimize(loop)
-        assert es.short_boxes[preamble.operations[0]]
+        assert es.short_boxes[0][0] == preamble.operations[0]
 
     def test_int_is_true(self):
         loop = """
@@ -142,5 +142,14 @@
         """
         es, loop, preamble = self.optimize(loop)
         op = preamble.operations[0]
-        assert es.short_boxes == {op:op}
+        assert es.short_boxes == [(op,op)]
         assert es.exported_infos[op].is_constant()
+
+    def test_only_setfield(self):
+        loop = """
+        [p0]
+        setfield_gc(p0, 5, descr=valuedescr)
+        jump(p0)
+        """
+        es, loop, preamble = self.optimize(loop)
+        assert es.short_boxes[0][0] == preamble.operations[0].getarg(1)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -235,8 +235,9 @@
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
-        for box in sb.short_boxes:
-            infos[box] = self.optimizer.getinfo(box)
+        for box, _ in sb.short_boxes:
+            if not isinstance(box, Const):
+                infos[box] = self.optimizer.getinfo(box)
         label_args = virtual_state.make_inputargs(end_args, self.optimizer)
         self.optimizer._clean_optimization_info(end_args)
         self.optimizer._clean_optimization_info(start_label.getarglist())
@@ -300,8 +301,9 @@
                 self.optimizer.setinfo_from_preamble(source, info)
         # import the optimizer state, starting from boxes that can be produced
         # by short preamble
-        for op, preamble_op in exported_state.short_boxes.items():
-            self.ops_to_import[op] = preamble_op
+        for op, preamble_op in exported_state.short_boxes:
+            if not isinstance(op, Const):
+                self.ops_to_import[op] = preamble_op
             if preamble_op.is_always_pure():
                 self.pure(op.getopnum(), PreambleOp(op, preamble_op,
                                 exported_state.exported_infos.get(op, None)))
@@ -311,7 +313,7 @@
                 if optheap is None:
                     continue
                 opinfo = self.optimizer.ensure_ptr_info_arg0(preamble_op)
-                pre_info = exported_state.exported_infos[op]
+                pre_info = exported_state.exported_infos.get(op, None)
                 pop = PreambleOp(op, preamble_op, pre_info)
                 assert not opinfo.is_virtual()
                 opinfo._fields[preamble_op.getdescr().get_index()] = pop

From noreply at buildbot.pypy.org  Mon Jul 13 17:41:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 13 Jul 2015 17:41:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2079: missed another obscure
 condition in type object creation
Message-ID: <20150713154131.621661C026B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78547:2f052c461fca
Date: 2015-07-13 17:41 +0200
http://bitbucket.org/pypy/pypy/changeset/2f052c461fca/

Log:	Issue #2079: missed another obscure condition in type object
	creation

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1031,6 +1031,48 @@
         A.__dict__['x'] = 5
         assert A.x == 5
 
+    def test_we_already_got_one_1(self):
+        # Issue #2079: highly obscure: CPython complains if we say
+        # ``__slots__="__dict__"`` and there is already a __dict__...
+        # but from the "best base" only.  If the __dict__ comes from
+        # another base, it doesn't complain.  Shrug and copy the logic.
+        class A(object):
+            __slots__ = ()
+        class B(object):
+            pass
+        class C(A, B):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(A, B):     # "best base" is A
+            __slots__ = ("__weakref__",)
+        try:
+            class E(B, A):   # "best base" is B
+                __slots__ = ("__dict__",)
+        except TypeError, e:
+            assert 'we already got one' in str(e)
+        else:
+            raise AssertionError("TypeError not raised")
+        try:
+            class F(B, A):   # "best base" is B
+                __slots__ = ("__weakref__",)
+        except TypeError, e:
+            assert 'we already got one' in str(e)
+        else:
+            raise AssertionError("TypeError not raised")
+
+    def test_we_already_got_one_2(self):
+        class A(object):
+            __slots__ = ()
+        class B:
+            pass
+        class C(A, B):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(A, B):     # "best base" is A
+            __slots__ = ("__weakref__",)
+        class C(B, A):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(B, A):     # "best base" is A
+            __slots__ = ("__weakref__",)
+
 
 class AppTestWithMethodCacheCounter:
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1022,7 +1022,7 @@
     w_self.nslots = w_bestbase.nslots
     return hasoldstylebase
 
-def create_all_slots(w_self, hasoldstylebase):
+def create_all_slots(w_self, hasoldstylebase, w_bestbase):
     space = w_self.space
     dict_w = w_self.dict_w
     if '__slots__' not in dict_w:
@@ -1040,12 +1040,12 @@
         for w_slot_name in slot_names_w:
             slot_name = space.str_w(w_slot_name)
             if slot_name == '__dict__':
-                if wantdict or w_self.hasdict:
+                if wantdict or w_bestbase.hasdict:
                     raise oefmt(space.w_TypeError,
                                 "__dict__ slot disallowed: we already got one")
                 wantdict = True
             elif slot_name == '__weakref__':
-                if wantweakref or w_self.weakrefable:
+                if wantweakref or w_bestbase.weakrefable:
                     raise oefmt(space.w_TypeError,
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
@@ -1106,7 +1106,7 @@
         w_self.flag_abstract |= w_base.flag_abstract
 
     hasoldstylebase = copy_flags_from_bases(w_self, w_bestbase)
-    create_all_slots(w_self, hasoldstylebase)
+    create_all_slots(w_self, hasoldstylebase, w_bestbase)
 
     ensure_common_attributes(w_self)
 

From noreply at buildbot.pypy.org  Mon Jul 13 20:00:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 13 Jul 2015 20:00:45 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: improve the situation somewhat,
 despite failing tests
Message-ID: <20150713180045.74AAF1C101F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78548:3a0e074cc535
Date: 2015-07-13 20:00 +0200
http://bitbucket.org/pypy/pypy/changeset/3a0e074cc535/

Log:	improve the situation somewhat, despite failing tests

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -9,12 +9,12 @@
     def __init__(self):
         self.potential_ops = []
         self.produced_short_boxes = {}
-        self.ops_used = {}
         self.extra_same_as = []
 
     def create_short_boxes(self, optimizer, inputargs):
         for box in inputargs:
-            self.ops_used[box] = None
+            self.produced_short_boxes[box] = None
+
         optimizer.produce_potential_short_preamble_ops(self)
 
         self.short_boxes = []
@@ -33,19 +33,13 @@
         self.produced_short_boxes[op] = None
 
     def produce_short_preamble_op(self, op, preamble_op):
-        if isinstance(op, Const):
-            pass
-        elif isinstance(op, AbstractInputArg):
-            if op not in self.ops_used:
-                return
-        else:
-            for arg in op.getarglist():
-                if isinstance(arg, Const):
-                    pass
-                elif arg in self.ops_used:
-                    pass
-                else:
-                    return # can't produce
+        for arg in preamble_op.getarglist():
+            if isinstance(arg, Const):
+                pass
+            elif arg in self.produced_short_boxes:
+                pass
+            else:
+                return # can't produce
         if op in self.produced_short_boxes:
             opnum = OpHelpers.same_as_for_type(op.type)
             same_as_op = ResOperation(opnum, [op])
@@ -59,4 +53,3 @@
             self.potential_ops.append((op, op))
         else:
             self.potential_ops.append((op, short_preamble_op))
-        self.ops_used[op] = None
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -867,21 +867,21 @@
     def test_dont_delay_setfields(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc_i(p1, descr=nextdescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i2b = int_is_true(i2)
         guard_true(i2b) []
-        setfield_gc(p2, i2, descr=nextdescr)
+        setfield_gc(p2, i2, descr=valuedescr)
         p3 = new_with_vtable(descr=nodesize)
         jump(p2, p3)
         """
         preamble = """
         [p1, p2]
-        i1 = getfield_gc_i(p1, descr=nextdescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i2b = int_is_true(i2)
         guard_true(i2b) []
-        setfield_gc(p2, i2, descr=nextdescr)
+        setfield_gc(p2, i2, descr=valuedescr)
         jump(p2, i2)
         """
         expected = """
@@ -890,7 +890,7 @@
         i2b = int_is_true(i2)
         guard_true(i2b) []
         p3 = new_with_vtable(descr=nodesize)
-        setfield_gc(p3, i2, descr=nextdescr)
+        setfield_gc(p3, i2, descr=valuedescr)
         jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -228,7 +228,8 @@
         end_args = [self.get_box_replacement(a) for a in original_label_args]
         virtual_state = self.get_virtual_state(end_args)
         sb = ShortBoxes()
-        sb.create_short_boxes(self.optimizer, end_args)
+        sb.create_short_boxes(self.optimizer, [self.get_box_replacement(a)
+                                            for a in start_label.getarglist()])
         inparg_mapping = [(start_label.getarg(i), end_args[i])
                           for i in range(len(end_args)) if
                           start_label.getarg(i) is not end_args[i]]

From noreply at buildbot.pypy.org  Tue Jul 14 00:01:56 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 14 Jul 2015 00:01:56 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: kill RecordChunk
Message-ID: <20150713220156.CB2721C12B9@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78549:e9f26b0c2ae0
Date: 2015-07-13 23:59 +0200
http://bitbucket.org/pypy/pypy/changeset/e9f26b0c2ae0/

Log:	kill RecordChunk

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -11,7 +11,7 @@
 from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (
     Chunk, Chunks, NewAxisChunk, EllipsisChunk,
-    RecordChunk, calc_strides, calc_new_strides, shape_agreement,
+    calc_strides, calc_new_strides, shape_agreement,
     calculate_broadcast_strides, calc_backstrides, calc_start, is_c_contiguous,
     is_f_contiguous)
 from rpython.rlib.objectmodel import keepalive_until_here
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -20,7 +20,7 @@
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import (
     get_shape_from_iterable, shape_agreement, shape_agreement_multiple,
-    is_c_contiguous, is_f_contiguous, RecordChunk)
+    is_c_contiguous, is_f_contiguous, calc_strides)
 from pypy.module.micronumpy.casting import can_cast_array
 
 
@@ -257,8 +257,23 @@
         dtype = self.get_dtype()
         if field not in dtype.fields:
             raise oefmt(space.w_ValueError, "field named %s not found", field)
-        chunks = RecordChunk(field)
-        return chunks.apply(space, self)
+        arr = self.implementation
+        ofs, subdtype = arr.dtype.fields[field]
+        # ofs only changes start
+        # create a view of the original array by extending
+        # the shape, strides, backstrides of the array
+        strides, backstrides = calc_strides(subdtype.shape,
+                                            subdtype.subdtype, arr.order)
+        final_shape = arr.shape + subdtype.shape
+        final_strides = arr.get_strides() + strides
+        final_backstrides = arr.get_backstrides() + backstrides
+        final_dtype = subdtype
+        if subdtype.subdtype:
+            final_dtype = subdtype.subdtype
+        return W_NDimArray.new_slice(space, arr.start + ofs, final_strides,
+                                     final_backstrides,
+                                     final_shape, arr, self, final_dtype)
+
 
     def descr_delitem(self, space, w_idx):
         raise OperationError(space.w_ValueError, space.wrap(
@@ -1317,7 +1332,6 @@
 def descr_new_array(space, w_subtype, w_shape, w_dtype=None, w_buffer=None,
                     offset=0, w_strides=None, w_order=None):
     from pypy.module.micronumpy.concrete import ConcreteArray
-    from pypy.module.micronumpy.strides import calc_strides
     dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
         space.gettypefor(descriptor.W_Dtype), w_dtype))
     shape = shape_converter(space, w_shape, dtype)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,29 +10,6 @@
     pass
 
 
-class RecordChunk(BaseChunk):
-    def __init__(self, name):
-        self.name = name
-
-    def apply(self, space, orig_arr):
-        arr = orig_arr.implementation
-        ofs, subdtype = arr.dtype.fields[self.name]
-        # ofs only changes start
-        # create a view of the original array by extending
-        # the shape, strides, backstrides of the array
-        strides, backstrides = calc_strides(subdtype.shape,
-                                            subdtype.subdtype, arr.order)
-        final_shape = arr.shape + subdtype.shape
-        final_strides = arr.get_strides() + strides
-        final_backstrides = arr.get_backstrides() + backstrides
-        final_dtype = subdtype
-        if subdtype.subdtype:
-            final_dtype = subdtype.subdtype
-        return W_NDimArray.new_slice(space, arr.start + ofs, final_strides,
-                                     final_backstrides,
-                                     final_shape, arr, orig_arr, final_dtype)
-
-
 class Chunks(BaseChunk):
     def __init__(self, l):
         self.l = l

From noreply at buildbot.pypy.org  Tue Jul 14 11:21:52 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 14 Jul 2015 11:21:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: inline is a stupid keyword
Message-ID: <20150714092152.3A4761C0980@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78550:293ebfd6fa0b
Date: 2015-07-14 11:14 +0200
http://bitbucket.org/pypy/pypy/changeset/293ebfd6fa0b/

Log:	inline is a stupid keyword

diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
--- a/pypy/module/_vmprof/src/getpc.h
+++ b/pypy/module/_vmprof/src/getpc.h
@@ -132,7 +132,7 @@
   }
 };
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   // See comment above struct CallUnrollInfo.  Only try instruction
   // flow matching if both eip and esp looks reasonable.
   const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
@@ -168,7 +168,7 @@
 typedef int ucontext_t;
 #endif
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
   return NULL;
 }
@@ -178,7 +178,7 @@
 // the right value for your system, and add it to the list in
 // configure.ac (or set it manually in your config.h).
 #else
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
 }
 

From noreply at buildbot.pypy.org  Tue Jul 14 11:21:53 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 14 Jul 2015 11:21:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: hack for freebsd
Message-ID: <20150714092153.6A0531C0EC0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78551:697799eeb074
Date: 2015-07-14 11:20 +0200
http://bitbucket.org/pypy/pypy/changeset/697799eeb074/

Log:	hack for freebsd

diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -262,13 +262,31 @@
 	int marker = MARKER_TRAILER;
 	write(profile_file, &marker, 1);
 
+#ifdef __linux__
     // copy /proc/PID/maps to the end of the profile file
     sprintf(buf, "/proc/%d/maps", getpid());
-    src = fopen(buf, "r");    
+    src = fopen(buf, "r");
+    if (!src) {
+        vmprof_error = "error opening proc maps";
+        return -1;
+    }
     while ((size = fread(buf, 1, BUFSIZ, src))) {
         write(profile_file, buf, size);
     }
     fclose(src);
+#else
+    // freebsd and mac
+    sprintf(buf, "procstat -v %d", getpid());
+    src = popen(buf, "r");
+    if (!src) {
+        vmprof_error = "error calling procstat";
+        return -1;
+    }
+    while ((size = fread(buf, 1, BUFSIZE, src))) {
+        write(profile_file, buf, size);
+    }
+    pclose(src);
+#endif
     close(profile_file);
 	return 0;
 }

From noreply at buildbot.pypy.org  Tue Jul 14 11:21:54 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 14 Jul 2015 11:21:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150714092154.CFF2D1C0980@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78552:d4482678ba26
Date: 2015-07-14 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/d4482678ba26/

Log:	merge

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -617,15 +617,17 @@
         self.impl = impl
         self.readonly = readonly
 
-    def getitem(self, item):
-        return raw_storage_getitem(lltype.Char, self.impl.storage, item)
+    def getitem(self, index):
+        return raw_storage_getitem(lltype.Char, self.impl.storage,
+                 index + self.impl.start)
 
-    def setitem(self, item, v):
-        raw_storage_setitem(self.impl.storage, item,
+    def setitem(self, index, v):
+        raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
     def getlength(self):
-        return self.impl.size
+        return self.impl.size - self.impl.start
 
     def get_raw_address(self):
-        return self.impl.storage
+        from rpython.rtyper.lltypesystem import rffi
+        return rffi.ptradd(self.impl.storage, self.impl.start)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -77,7 +77,7 @@
     stop = 1
     step = 1
     lgt = 1
-    axis_step = 0
+    axis_step = 0 # both skip this axis in calculate_slice_strides and set stride => 0
 
     def __init__(self):
         pass
@@ -127,7 +127,7 @@
         except IndexError:
             continue
         if chunk.step != 0:
-            rstrides[j] = s_i * chunk.step
+            rstrides[j] = s_i * chunk.step * chunk.axis_step
             rbackstrides[j] = s_i * max(0, chunk.lgt - 1) * chunk.step
             rshape[j] = chunk.lgt
             j += 1
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -764,6 +764,8 @@
         assert (a[1:] == b).all()
         assert (a[1:,newaxis] == d).all()
         assert (a[newaxis,1:] == c).all()
+        assert a.strides == (8,)
+        assert a[:, newaxis].strides == (8, 0)
 
     def test_newaxis_assign(self):
         from numpy import array, newaxis
@@ -2345,6 +2347,7 @@
         assert a[1] == 0xff
         assert len(a.data) == 16
         assert type(a.data) is buffer
+        assert a[1:].data._pypy_raw_address() - a.data._pypy_raw_address() == a.strides[0]
 
     def test_explicit_dtype_conversion(self):
         from numpy import array
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1031,6 +1031,48 @@
         A.__dict__['x'] = 5
         assert A.x == 5
 
+    def test_we_already_got_one_1(self):
+        # Issue #2079: highly obscure: CPython complains if we say
+        # ``__slots__="__dict__"`` and there is already a __dict__...
+        # but from the "best base" only.  If the __dict__ comes from
+        # another base, it doesn't complain.  Shrug and copy the logic.
+        class A(object):
+            __slots__ = ()
+        class B(object):
+            pass
+        class C(A, B):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(A, B):     # "best base" is A
+            __slots__ = ("__weakref__",)
+        try:
+            class E(B, A):   # "best base" is B
+                __slots__ = ("__dict__",)
+        except TypeError, e:
+            assert 'we already got one' in str(e)
+        else:
+            raise AssertionError("TypeError not raised")
+        try:
+            class F(B, A):   # "best base" is B
+                __slots__ = ("__weakref__",)
+        except TypeError, e:
+            assert 'we already got one' in str(e)
+        else:
+            raise AssertionError("TypeError not raised")
+
+    def test_we_already_got_one_2(self):
+        class A(object):
+            __slots__ = ()
+        class B:
+            pass
+        class C(A, B):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(A, B):     # "best base" is A
+            __slots__ = ("__weakref__",)
+        class C(B, A):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(B, A):     # "best base" is A
+            __slots__ = ("__weakref__",)
+
 
 class AppTestWithMethodCacheCounter:
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1022,7 +1022,7 @@
     w_self.nslots = w_bestbase.nslots
     return hasoldstylebase
 
-def create_all_slots(w_self, hasoldstylebase):
+def create_all_slots(w_self, hasoldstylebase, w_bestbase):
     space = w_self.space
     dict_w = w_self.dict_w
     if '__slots__' not in dict_w:
@@ -1040,12 +1040,12 @@
         for w_slot_name in slot_names_w:
             slot_name = space.str_w(w_slot_name)
             if slot_name == '__dict__':
-                if wantdict or w_self.hasdict:
+                if wantdict or w_bestbase.hasdict:
                     raise oefmt(space.w_TypeError,
                                 "__dict__ slot disallowed: we already got one")
                 wantdict = True
             elif slot_name == '__weakref__':
-                if wantweakref or w_self.weakrefable:
+                if wantweakref or w_bestbase.weakrefable:
                     raise oefmt(space.w_TypeError,
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
@@ -1106,7 +1106,7 @@
         w_self.flag_abstract |= w_base.flag_abstract
 
     hasoldstylebase = copy_flags_from_bases(w_self, w_bestbase)
-    create_all_slots(w_self, hasoldstylebase)
+    create_all_slots(w_self, hasoldstylebase, w_bestbase)
 
     ensure_common_attributes(w_self)
 

From noreply at buildbot.pypy.org  Tue Jul 14 11:24:07 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 14 Jul 2015 11:24:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix
Message-ID: <20150714092407.AD4EF1C0980@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78553:a7dedcc55d55
Date: 2015-07-14 11:24 +0200
http://bitbucket.org/pypy/pypy/changeset/a7dedcc55d55/

Log:	fix

diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -282,7 +282,7 @@
         vmprof_error = "error calling procstat";
         return -1;
     }
-    while ((size = fread(buf, 1, BUFSIZE, src))) {
+    while ((size = fread(buf, 1, BUFSIZ, src))) {
         write(profile_file, buf, size);
     }
     pclose(src);

From noreply at buildbot.pypy.org  Tue Jul 14 16:13:26 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 14 Jul 2015 16:13:26 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: inserting guard early exit in user
 traces if the config turns on these setting
Message-ID: <20150714141326.2158C1C06CD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78554:5190e9b953e7
Date: 2015-07-14 16:07 +0200
http://bitbucket.org/pypy/pypy/changeset/5190e9b953e7/

Log:	inserting guard early exit in user traces if the config turns on
	these setting

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1122,7 +1122,7 @@
 
         if self.metainterp.seen_loop_header_for_jdindex < 0:
             if not any_operation:
-                if jitdriver_sd.vectorize:
+                if jitdriver_sd.vectorize or jitdriver_sd.warmstate.vectorize_user:
                     self.metainterp.generate_guard(rop.GUARD_EARLY_EXIT)
                 return
             if self.metainterp.portal_call_depth or not self.metainterp.get_procedure_token(greenboxes, True):

From noreply at buildbot.pypy.org  Wed Jul 15 12:50:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 15 Jul 2015 12:50:57 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: add a proposal
Message-ID: <20150715105057.2C6A31C028A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5539:5b5d9a9ac7bb
Date: 2015-07-15 12:44 +0200
http://bitbucket.org/pypy/extradoc/changeset/5b5d9a9ac7bb/

Log:	add a proposal

diff --git a/talk/pyconpl-2015/proposal.rst b/talk/pyconpl-2015/proposal.rst
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2015/proposal.rst
@@ -0,0 +1,29 @@
+Python & PyPy performance (not) for dummies
+-------------------------------------------
+
+In this talk we would like to talk a bit about the performance
+characteristics of python programs, how to (or when not to) write benchmarks
+and how to measure performance of the existing python programs both
+under CPython (the standard python interpreter) and PyPy.
+
+One of the key points will be presentation of vmprof, a low-overhead
+Python statistical profiler written by us.
+
+The talk will cover:
+
+* the basics of statistics and benchmarking
+
+* tools available for measuring the performance of python programs
+
+* common strategies to improve the bottlenecks
+
+About author:
+-------------
+
+Maciej is a freelancer working mostly on PyPy for the past several
+years. He's a core developer since 2006, working on all kinds of parts
+in the entire codebase including JIT, GC and assembler
+backends. Maciej has been going to many conferences, advertising PyPy
+to a broader audience for the past several years, including a keynote
+at Pycon 2010. He's also the main maintainer of jitviewer and vmprof, tools
+analyzing performance of your python programs under PyPy.

From noreply at buildbot.pypy.org  Wed Jul 15 12:50:58 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 15 Jul 2015 12:50:58 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge
Message-ID: <20150715105058.879431C07DE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5540:ea1511561764
Date: 2015-07-15 12:51 +0200
http://bitbucket.org/pypy/extradoc/changeset/ea1511561764/

Log:	merge

diff --git a/talk/ep2015/pypy-abstract.rst b/talk/ep2015/pypy-abstract.rst
new file mode 100644
--- /dev/null
+++ b/talk/ep2015/pypy-abstract.rst
@@ -0,0 +1,22 @@
+=============================
+PyPy for mediocre programmers
+=============================
+
+This is a talk for mediocre Python programmers by a mediocre programmer.  PyPy
+is an alternative implementation of Python. It is notorious for being fast, but
+also for using clever algorithms pertaining to advanced concepts such as type
+inference, garbage collection, just-in-time compilation, etc. So, can we,
+mediocre programmers, realistically use PyPy?
+
+Yes, absolutely. In fact, PyPy developers did all that hard work so that we
+wouldn't have to. As we'll see, it runs most Python code exactly like CPython
+does, save that it magically makes it faster.
+
+Porting existing applications is always more involved than running a simple
+script, so we'll also examine likely difficulties such as code relying on
+CPython implementation details, and dependencies on C extensions, and explore
+simple principles to let PyPy run your code even faster.
+
+Finally, we'll have a glimpse of the future by looking at what's brewing in 
+the PyPy lair, such as software transactional memory, new speed optimisations,
+better support for Python 3 and NumPy, ...
diff --git a/talk/pycon-italy-2015/Makefile b/talk/pycon-italy-2015/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/Makefile
@@ -0,0 +1,18 @@
+# you can find rst2beamer.py here:
+# http://codespeak.net/svn/user/antocuni/bin/rst2beamer.py
+
+# WARNING: to work, it needs this patch for docutils
+# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414
+
+talk.pdf: talk.rst author.latex title.latex stylesheet.latex
+	python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
+	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
+	pdflatex talk.latex  || exit
+
+view: talk.pdf
+	evince talk.pdf &
+
+xpdf: talk.pdf
+	xpdf talk.pdf &
diff --git a/talk/pycon-italy-2015/author.latex b/talk/pycon-italy-2015/author.latex
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/author.latex
@@ -0,0 +1,8 @@
+\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0}
+
+\title[PyPy JIT]{PyPy JIT (not) for dummies}
+\author[antocuni]
+{Antonio Cuni}
+
+\institute{PyCon Sei}
+\date{April 17, 2015}
diff --git a/talk/pycon-italy-2015/beamerdefs.txt b/talk/pycon-italy-2015/beamerdefs.txt
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/beamerdefs.txt
@@ -0,0 +1,108 @@
+.. colors
+.. ===========================
+
+.. role:: green
+.. role:: red
+
+
+.. general useful commands
+.. ===========================
+
+.. |pause| raw:: latex
+
+   \pause
+
+.. |small| raw:: latex
+
+   {\small
+
+.. |end_small| raw:: latex
+
+   }
+
+.. |scriptsize| raw:: latex
+
+   {\scriptsize
+
+.. |end_scriptsize| raw:: latex
+
+   }
+
+.. |strike<| raw:: latex
+
+   \sout{
+
+.. closed bracket
+.. ===========================
+
+.. |>| raw:: latex
+
+   }
+
+
+.. example block
+.. ===========================
+
+.. |example<| raw:: latex
+
+   \begin{exampleblock}{
+
+
+.. |end_example| raw:: latex
+
+   \end{exampleblock}
+
+
+
+.. alert block
+.. ===========================
+
+.. |alert<| raw:: latex
+
+   \begin{alertblock}{
+
+
+.. |end_alert| raw:: latex
+
+   \end{alertblock}
+
+
+
+.. columns
+.. ===========================
+
+.. |column1| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.45\textwidth}
+
+.. |column2| raw:: latex
+
+      \end{column}
+      \begin{column}{0.45\textwidth}
+
+
+.. |end_columns| raw:: latex
+
+      \end{column}
+   \end{columns}
+
+
+
+.. |snake| image:: ../../img/py-web-new.png
+           :scale: 15%
+           
+
+
+.. nested blocks
+.. ===========================
+
+.. |nested| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.85\textwidth}
+
+.. |end_nested| raw:: latex
+
+      \end{column}
+   \end{columns}
diff --git a/talk/pycon-italy-2015/jit-overview.odg b/talk/pycon-italy-2015/jit-overview.odg
new file mode 100644
index 0000000000000000000000000000000000000000..43aec7c16da3c72bec87a6be52ce41969f4db8c7
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/jit-overview1.pdf b/talk/pycon-italy-2015/jit-overview1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..6b511590edd06e9deb157eb02cecb7cf64c61926
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/jit-overview2.pdf b/talk/pycon-italy-2015/jit-overview2.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..c07d0e3948403a7c42951a65ba023311d4739ac8
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/jit-overview3.pdf b/talk/pycon-italy-2015/jit-overview3.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..277b2067bc2daf79f7696edaabd22614182e6bfa
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/speed.png b/talk/pycon-italy-2015/speed.png
new file mode 100644
index 0000000000000000000000000000000000000000..e2372b9e0d5fc3fe7711b27d641d2d9706185fd3
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/src/decode0.py b/talk/pycon-italy-2015/src/decode0.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/src/decode0.py
@@ -0,0 +1,25 @@
+import sys
+import struct
+
+P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00'
+P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00'
+
+PLIST = [P1, P2] * 2000
+
+def read_x(p):
+    return struct.unpack_from('l', p, 0)[0]
+
+def read_y(p):
+    return struct.unpack_from('l', p, 4)[0]
+
+def read_color(p):
+    return struct.unpack_from('i', p, 8)[0]
+
+def main():
+    res = 0
+    for p in PLIST:
+        x = read_x(p)
+        res += x
+    print res
+
+main()
diff --git a/talk/pycon-italy-2015/src/decode1.py b/talk/pycon-italy-2015/src/decode1.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/src/decode1.py
@@ -0,0 +1,41 @@
+import sys
+import struct
+
+P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00'
+P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00'
+
+PLIST = [P1, P2] * 2000
+
+class Field(object):
+
+    def __init__(self, fmt, offset):
+        self.fmt = fmt
+        self.offset = offset
+
+
+class Message(object):
+
+    def __init__(self, name, fields):
+        self._name = name
+        self._fields = fields
+
+    def read(self, buf, name):
+        f = self._fields[name]
+        return struct.unpack_from(f.fmt, buf, f.offset)[0]
+
+
+Point = Message('Point', {
+    'x': Field('l', 0),
+    'y': Field('l', 4),
+    'color': Field('i', 8)
+    })
+
+
+def main():
+    res = 0
+    for p in PLIST:
+        x = Point.read(p, 'x')
+        res += x
+    print res
+
+main()
diff --git a/talk/pycon-italy-2015/src/decode2.py b/talk/pycon-italy-2015/src/decode2.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/src/decode2.py
@@ -0,0 +1,42 @@
+import sys
+import struct
+
+P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00'
+P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00'
+
+PLIST = [P1, P2] * 2000
+
+class Field(object):
+
+    def __init__(self, fmt, offset):
+        self.fmt = fmt
+        self.offset = offset
+
+def Message(name, fields):
+    class M(object):
+        def read(self, buf, name):
+            f = getattr(self, name)
+            return struct.unpack_from(f.fmt, buf, f.offset)[0]            
+
+    for fname, f in fields.iteritems():
+        setattr(M, fname, f)
+
+    M.__name__ = name
+    return M()
+
+
+Point = Message('Point', {
+    'x': Field('l', 0),
+    'y': Field('l', 4),
+    'color': Field('i', 8)
+    })
+
+
+def main():
+    res = 0
+    for p in PLIST:
+        x = Point.read(p, 'x')
+        res += x
+    print res
+
+main()
diff --git a/talk/pycon-italy-2015/src/decode3.py b/talk/pycon-italy-2015/src/decode3.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/src/decode3.py
@@ -0,0 +1,32 @@
+import sys
+import struct
+
+P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00'
+P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00'
+
+PLIST = [P1, P2] * 2000
+
+class Field(object):
+    def __init__(self, fmt, offset):
+        self.fmt = fmt
+        self.offset = offset
+
+    def __get__(self, obj, cls):
+        return struct.unpack_from(self.fmt, obj._buf, self.offset)[0]
+
+class Point(object):
+    def __init__(self, buf):
+        self._buf = buf
+
+    x = Field('l', 0)
+    y = Field('l', 4)
+    color = Field('h', 8)
+
+def main():
+    res = 0
+    for p in PLIST:
+        p = Point(p)
+        res += p.x
+    print res
+
+main()
diff --git a/talk/pycon-italy-2015/stylesheet.latex b/talk/pycon-italy-2015/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/stylesheet.latex
@@ -0,0 +1,10 @@
+\usetheme{Boadilla}
+\setbeamercovered{transparent}
+\setbeamertemplate{navigation symbols}{}
+
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/pycon-italy-2015/talk.pdf b/talk/pycon-italy-2015/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..1492bcaa865783996c4f06a4afc7eabcea54249b
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/talk.pdf.info b/talk/pycon-italy-2015/talk.pdf.info
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/talk.pdf.info
@@ -0,0 +1,11 @@
+AvailableTransitions=[Crossfade]
+TransitionDuration = 100
+EstimatedDuration = 45*60       # in seconds
+MinutesOnly = True
+
+PageProps = {
+    1: {
+    'reset': FirstTimeOnly,
+    'progress': False,
+    },
+}
diff --git a/talk/pycon-italy-2015/talk.rst b/talk/pycon-italy-2015/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/talk.rst
@@ -0,0 +1,580 @@
+.. include:: beamerdefs.txt
+
+================================
+PyPy JIT (not) for dummies
+================================
+
+About me
+---------
+
+- PyPy core dev
+
+- ``pdb++``, ``fancycompleter``, ...
+
+- Consultant, trainer
+
+- http://antocuni.eu
+
+
+What is PyPy
+---------------
+
+- Alternative, fast Python implementation
+
+- Performance: JIT compiler, advanced GC
+
+- STM: goodbye GIL
+
+- PyPy 2.5.1 (2.7.8)
+
+- Py3k as usual in progress (3.2.5 out, 3.3 in development)
+
+- http://pypy.org
+
+
+STM
+---
+
+- pypy-stm-2.5.1 is out
+
+  * 64 bit Linux only
+
+- no GIL!
+
+- 25-40% slowdown for single core programs
+
+  * still ``7*0.75 = 5.25x`` faster than CPython :)
+
+- parallelism up to 4 threads
+
+- concurrency slow-but-correct by default
+
+  * compared to fast-but-buggy by using threads
+
+- conflict detection
+
+- TransactionQueue: parallelize your program without using threads!
+
+
+Extension modules
+------------------
+
+- CFFI: stable, mature and widely used
+
+  * psycopg2cffi, lxml-cffi, pysdl2-cffi, etc.
+
+  * should be used even for CPython-only projects!
+
+- numpy:
+
+  * support for linalg
+
+  * support for pure Python, JIT friendly ufuncs
+
+  * object dtype in-progress
+
+- scipy: see next slide :)
+
+
+Pymetabiosis
+--------------
+
+- embed CPython in PyPy
+
+- import and use CPython modules in PyPy
+
+- ALPHA status
+
+- slow when passing arbitrary objects
+
+- but fast for numpy arrays
+
+- matplotlib and scipy works
+
+- https://github.com/rguillebert/pymetabiosis
+
+
+
+Speed: 7x faster than CPython
+-------------------------------
+
+.. image:: speed.png
+   :scale: 47%
+
+
+The JIT
+--------
+
+.. image:: jit-overview1.pdf
+   :scale: 50%
+
+
+The JIT
+--------
+
+.. image:: jit-overview2.pdf
+   :scale: 50%
+
+
+The JIT
+--------
+
+.. image:: jit-overview3.pdf
+   :scale: 50%
+
+
+JIT overview
+-------------
+
+- Tracing JIT
+
+  * detect and compile "hot" loops
+
+  * (although not only loops)
+
+- **Specialization**
+
+- Precompute as much as possible
+
+- Constant propagation
+
+- Aggressive inlining
+
+
+Specialization (1)
+-------------------
+
+- ``obj.foo()``
+
+- which code is executed? (SIMPLIFIED)
+
+  * lookup ``foo`` in obj.__dict__
+
+  * lookup ``foo`` in obj.__class__
+
+  * lookup ``foo`` in obj.__bases__[0], etc.
+
+  * finally, execute ``foo``
+
+- without JIT, you need to do these steps again and again
+
+- Precompute the lookup?
+
+
+Specialization (2)
+--------------------
+
+- pretend and assume that ``obj.__class__`` IS constant
+
+  * "promotion"
+
+- guard
+
+  * check our assumption: if it's false, bail out
+
+- now we can directly jump to ``foo`` code
+
+  * ...unless ``foo`` is in ``obj.__dict__``: GUARD!
+
+  * ...unless ``foo.__class__.__dict__`` changed: GUARD!
+
+- Too many guard failures?
+
+  * Compile some more assembler!
+
+- guards are cheap
+
+  * out-of-line guards even more
+
+
+Specialization (3)
+---------------------
+
+- who decides what to promote/specialize for?
+
+  * we, the PyPy devs :)
+
+  * heuristics
+
+- instance attributes are never promoted
+
+- class attributes are promoted by default (with some exceptions)
+
+- module attributes (i.e., globals) as well
+
+- bytecode constants
+
+
+Specialization trade-offs
+--------------------------
+
+- Too much specialization
+
+  * guards fails often
+
+  * explosion of assembler
+
+- Not enough specialization
+
+  * inefficient code
+
+
+Virtuals
+--------
+
+- Remove unnecessary allocations
+
+- Remove unnecessary load/store
+
+|small|
+|example<| |small| virtuals.py |end_small| |>|
+
+.. sourcecode:: python
+
+    res = 0
+    while res < 10000:
+        obj = Foo(x, y, z)
+        res += obj.x
+
+|end_example|
+|end_small|
+
+
+Example
+--------
+
+- Real world example
+
+- Decoding binary messages
+
+- Messages: strings of bytes
+
+|small|
+|example<| |small| Point |end_small| |>|
+
+.. sourcecode:: C
+
+    struct Point {
+        int x;
+        int y;
+        short color;
+    }
+
+|end_example|
+|end_small|
+
+
+
+Example: low-level solution
+----------------------------
+
+|scriptsize|
+|example<| |small| decode0.py |end_small| |>|
+
+.. sourcecode:: python
+
+    P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00\x00\x00'
+    P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00\x00\x00'
+    
+    PLIST = [P1, P2] * 2000
+    
+    def read_x(p):
+        return struct.unpack_from('l', p, 0)[0]
+    
+    def main():
+        res = 0
+        for p in PLIST:
+            x = read_x(p)
+            res += x
+        print res
+
+|end_example|
+|end_scriptsize|
+
+Example: low-level solution
+----------------------------
+
+|scriptsize|
+|example<| |small| decode0.py trace |end_small| |>|
+
+.. sourcecode:: python
+
+    debug_merge_point(1, 1, '<code object read_x> #0 LOAD_GLOBAL')
+    debug_merge_point(1, 1, '<code object read_x> #3 LOOKUP_METHOD')
+    debug_merge_point(1, 1, '<code object read_x> #6 LOAD_CONST')
+    debug_merge_point(1, 1, '<code object read_x> #9 LOAD_FAST')
+    debug_merge_point(1, 1, '<code object read_x> #12 LOAD_CONST')
+    debug_merge_point(1, 1, '<code object read_x> #15 CALL_METHOD')
+    +606: i91 = strlen(p88)
+    +609: i92 = int_lt(i91, 4)
+    guard_false(i92, descr=<Guard0xb3a14b20>)
+    +618: i93 = strgetitem(p88, 0)
+    +622: i94 = strgetitem(p88, 1)
+    +632: i95 = int_lshift(i94, 8)
+    +635: i96 = int_or(i93, i95)
+    +637: i97 = strgetitem(p88, 2)
+    +653: i98 = int_lshift(i97, 16)
+    +656: i99 = int_or(i96, i98)
+    +658: i100 = strgetitem(p88, 3)
+    +662: i101 = int_ge(i100, 128)
+    guard_false(i101, descr=<Guard0xb3a14ac0>)
+    +674: i102 = int_lshift(i100, 24)
+    +677: i103 = int_or(i99, i102)
+    
+|end_example|
+|end_scriptsize|
+
+Example: better API
+---------------------
+
+|scriptsize|
+|example<| |small| decode1.py |end_small| |>|
+
+.. sourcecode:: python
+
+    class Field(object):
+        def __init__(self, fmt, offset):
+            self.fmt = fmt; self.offset = offset
+
+    class Message(object):
+        def __init__(self, name, fields):
+            self._name = name; self._fields = fields
+    
+        def read(self, buf, name):
+            f = self._fields[name]
+            return struct.unpack_from(f.fmt, buf, f.offset)[0]
+
+    Point = Message('Point', {'x': Field('l', 0), 
+                              'y': Field('l', 8),
+                              'color': Field('i', 16)})
+
+    def main():
+        res = 0
+        for p in PLIST:
+            x = Point.read(p, 'x')
+            res += x
+        print res
+
+|end_example|
+|end_scriptsize|
+
+Example: better API
+----------------------------
+
+|scriptsize|
+|example<| |small| decode1.py trace (1) |end_small| |>|
+
+.. sourcecode:: python
+
+    debug_merge_point(1, 1, '<code object read> #34 CALL_METHOD')
+    p156 = getfield_gc_pure(p154, descr=<W_BytesObject.inst__value 8>)
+    i157 = getfield_gc_pure(p155, descr=<W_IntObject.inst_intval 8>)
+    p158 = new_with_vtable(-1228074336)
+    setfield_gc(p158, 0, descr=<CalcSizeFormatIterator.inst_totalsize 8>)
+    call(interpret_trampoline__v238___simple_call__function_i, p158, p156, ...)
+    guard_no_exception(descr=<Guard0xb3a8cd00>)
+    i159 = getfield_gc(p158, descr=<CalcSizeFormatIterator.inst_totalsize 8>)
+    i160 = int_lt(i157, 0)
+    guard_false(i160, descr=<Guard0xb3a8ccd0>)
+    i161 = strlen(p141)
+    i162 = int_sub(i161, i157)
+    i163 = int_lt(i162, i159)
+    guard_false(i163, descr=<Guard0xb3a8cca0>)
+    i164 = int_ge(i159, 0)
+    guard_true(i164, descr=<Guard0xb3a8cc70>)
+    p165 = force_token()
+    p166 = new_with_vtable(-1228077368)
+    p167 = new_with_vtable(-1228077280)
+    p168 = new_with_vtable(-1228267680)
+    setfield_gc(p167, 1, descr=<FieldU rpython.rlib.buffer.Buffer.inst_readonly 8>)
+|end_example|
+|end_scriptsize|
+
+Example: better API
+----------------------------
+
+|scriptsize|
+|example<| |small| decode1.py trace (2) |end_small| |>|
+
+.. sourcecode:: python
+
+    p169 = new(descr=<SizeDescr 12>)
+    p170 = new_array_clear(0, descr=<ArrayP 4>)
+    p171 = new_with_vtable(-1229823908)
+    setfield_gc(p171, p145, descr=<JitVirtualRef.virtual_token 8>)
+    setfield_gc(p171, p145, descr=<JitVirtualRef.virtual_token 8>)
+    setfield_gc(p171, ConstPtr(null), descr=<FieldP JitVirtualRef.forced 12>)
+    setfield_gc(p51, p171, descr=<ExecutionContext.inst_topframeref 40>)
+    setfield_gc(p0, p165, descr=<PyFrame.vable_token 8>)
+    setfield_gc(p168, 1, descr=<Buffer.inst_readonly 8>)
+    setfield_gc(p168, p141, descr=<StringBuffer.inst_value 12>)
+    setfield_gc(p167, p168, descr=<SubBuffer.inst_buffer 12>)
+    setfield_gc(p167, i157, descr=<SubBuffer.inst_offset 16>)
+    setfield_gc(p167, i159, descr=<SubBuffer.inst_size 20>)
+    setfield_gc(p166, p167, descr=<UnpackFormatIterator.inst_buf 8>)
+    setfield_gc(p166, i159, descr=<UnpackFormatIterator.inst_length 12>)
+    setfield_gc(p166, 0, descr=<UnpackFormatIterator.inst_pos 16>)
+    setfield_gc(p169, 0, descr=<list.length 4>)
+    setfield_gc(p169, p170, descr=<list.items 8>)
+    setfield_gc(p166, p169, descr=<UnpackFormatIterator.inst_result_w 20>)
+
+|end_example|
+|end_scriptsize|
+
+
+Example: better API
+----------------------------
+
+|scriptsize|
+|example<| |small| decode1.py trace (3) |end_small| |>|
+
+.. sourcecode:: python
+
+    call_may_force(interpret_trampoline__v628___simple_call__function_i), p166, p156, descr=<Callv 0 rr EF=6>)
+    guard_not_forced(descr=<Guard0xb3a8b9d0>)
+    guard_no_exception(descr=<Guard0xb3a8cc40>)
+    p172 = getfield_gc(p166, descr=<UnpackFormatIterator.inst_result_w 20>)
+    i173 = getfield_gc(p172, descr=<list.length 4>)
+    p174 = new_array_clear(i173, descr=<ArrayP 4>)
+    p175 = getfield_gc(p172, descr=<list.items 8>)
+    call(ll_arraycopy__arrayPtr_arrayPtr_Signed_Signed_Signed), p175, p174, 0, 0, i173, descr=<Callv 0 rriii EF=2 OS=1>)
+    i176 = int_eq(i173, 2)
+    guard_false(i176, descr=<Guard0xb3a8cc10>)
+
+|end_example|
+|end_scriptsize|
+
+Example: faster API
+---------------------
+
+|scriptsize|
+|example<| |small| decode2.py |end_small| |>|
+
+.. sourcecode:: python
+
+    def Message(name, fields):
+        class M(object):
+            def read(self, buf, name):
+                f = getattr(self, name)
+                return struct.unpack_from(f.fmt, buf, f.offset)[0]            
+    
+        for fname, f in fields.iteritems():
+            setattr(M, fname, f)
+    
+        M.__name__ = name
+        return M()
+    
+    Point = Message('Point', {
+        'x': Field('l', 0),
+        'y': Field('l', 4),
+        'color': Field('i', 8)
+        })
+    
+     ...
+     x = Point.read(p, 'x')
+     ...
+
+|end_example|
+|end_scriptsize|
+
+Example: faster API
+----------------------------
+
+|scriptsize|
+|example<| |small| decode2.py trace (3) |end_small| |>|
+
+.. sourcecode:: python
+
+    debug_merge_point(1, 1, '<code object read> #36 CALL_METHOD')
+    +670: i104 = strlen(p101)
+    +673: i105 = int_lt(i104, 4)
+    guard_false(i105, descr=<Guard0xb3afac10>)
+    +682: i106 = strgetitem(p101, 0)
+    +686: i107 = strgetitem(p101, 1)
+    +696: i108 = int_lshift(i107, 8)
+    +699: i109 = int_or(i106, i108)
+    +701: i110 = strgetitem(p101, 2)
+    +717: i111 = int_lshift(i110, 16)
+    +720: i112 = int_or(i109, i111)
+    +722: i113 = strgetitem(p101, 3)
+    +726: i114 = int_ge(i113, 128)
+    guard_false(i114, descr=<Guard0xb3afabe0>)
+    +738: i115 = int_lshift(i113, 24)
+    +741: i116 = int_or(i112, i115)
+
+|end_example|
+|end_scriptsize|
+
+What happened?
+---------------
+
+- dict lookups inside classes are specialized
+
+- decode1.py
+
+  * ``fields`` is "normal data" and expected to change
+
+  * one JIT code for **all** possible messages
+
+- decode2.py
+
+  * ``fields`` is expected to be constant
+
+  * one JIT code for **each** message
+
+- Behaviour is the same, different performance
+
+
+Example: even better API :)
+-----------------------------
+
+|scriptsize|
+|example<| |small| decode3.py |end_small| |>|
+
+.. sourcecode:: python
+
+    class Field(object):
+        def __init__(self, fmt, offset):
+            self.fmt = fmt
+            self.offset = offset
+    
+        def __get__(self, obj, cls):
+            return struct.unpack_from(self.fmt, obj._buf, self.offset)[0]
+    
+    class Point(object):
+        def __init__(self, buf):
+            self._buf = buf
+    
+        x = Field('l', 0)
+        y = Field('l', 4)
+        color = Field('h', 8)
+    
+    def main():
+        res = 0
+        for p in PLIST:
+            p = Point(p)
+            res += p.x
+        print res
+
+|end_example|
+|end_scriptsize|
+
+
+
+Contacts, Q&A
+--------------
+
+- http://pypy.org
+
+- http://morepypy.blogspot.com/
+
+- twitter: @antocuni
+
+- Available for consultancy & training:
+
+  * http://antocuni.eu
+
+  * info at antocuni.eu
+
+- Any question?
+
diff --git a/talk/pycon-italy-2015/title.latex b/talk/pycon-italy-2015/title.latex
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/title.latex
@@ -0,0 +1,5 @@
+\begin{titlepage}
+\begin{figure}[h]
+\includegraphics[width=80px]{../img/py-web.png}
+\end{figure}
+\end{titlepage}

From noreply at buildbot.pypy.org  Wed Jul 15 16:34:32 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 15 Jul 2015 16:34:32 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: start working on a talk
Message-ID: <20150715143432.E28811C13BE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5541:d0835161a1e9
Date: 2015-07-15 16:35 +0200
http://bitbucket.org/pypy/extradoc/changeset/d0835161a1e9/

Log:	start working on a talk

diff --git a/talk/ep2015/performance/talk.rst b/talk/ep2015/performance/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/ep2015/performance/talk.rst
@@ -0,0 +1,84 @@
+
+Python & PyPy performance
+-------------------------
+
+What is performance?
+--------------------
+
+* it's a metric
+
+* usually, time spent doing task X
+
+* sometimes number of requests, latency, etc.
+
+* some statistical properties about that metric (average, minimum, maximum)
+
+Do you have a performance problem?
+----------------------------------
+
+* define the metric
+
+* measure it (production, benchmarks, etc.)
+
+* see if Python is the cause here (if it's not, we can't help you,
+  but I'm sure someone help)
+
+* make sure you can change and test stuff quickly (e.g. benchmarks are better
+  than changing stuff in production)
+
+We have a python problem
+------------------------
+
+* tools, timers etc.
+
+* systems are too complicated to **guess** which will be faster
+
+* find your bottlenecks
+
+* 20/80 (but 20% of million lines is 200 000 lines, remember that)
+
+Profilers landscape
+-------------------
+
+* cProfile, runSnakeRun (high overhead) - exact profiler
+
+* plop, vmprof - statistical profiler
+
+* cProfile & vmprof work on pypy
+
+vmprof
+------
+
+XXXxxx
+
+using vmprof
+------------
+
+yyyyyyy
+
+interpreting the results
+------------------------
+
+xxxx
+
+using vmprof in production
+--------------------------
+
+demo
+----
+
+let's optimize some code
+------------------------
+
+let's optimize some more complex code
+-------------------------------------
+
+Extras: what's cool what's not cool on cpython and pypy
+
+CPython vs PyPy
+---------------
+
+* very different performance characteristics
+
+* XXX list them
+

From noreply at buildbot.pypy.org  Wed Jul 15 17:32:04 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 15 Jul 2015 17:32:04 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rewriting parts of the guard strength
 reduction to eliminate array bound checks
Message-ID: <20150715153204.E2E501C101F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78555:961725ecb559
Date: 2015-07-15 17:32 +0200
http://bitbucket.org/pypy/pypy/changeset/961725ecb559/

Log:	rewriting parts of the guard strength reduction to eliminate array
	bound checks

diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -9,34 +9,34 @@
         MemoryRef, Node, IndexVar)
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.jit.metainterp.history import (ConstInt, BoxVector, 
-        BoxFloat, BoxInt, ConstFloat, Box)
+        BoxFloat, BoxInt, ConstFloat, Box, Const)
+from rpython.rlib.objectmodel import we_are_translated
 
 class Guard(object):
     """ An object wrapper around a guard. Helps to determine
         if one guard implies another
     """
-    def __init__(self, index, op, cmp_op, lhs, lhs_arg, rhs, rhs_arg):
+    def __init__(self, index, op, cmp_op, lhs_arg, rhs_arg):
         self.index = index
         self.op = op
         self.cmp_op = cmp_op
-        self.lhs = lhs
-        self.rhs = rhs
         self.lhs_arg = lhs_arg
         self.rhs_arg = rhs_arg
-        self.implied = False
-        self.stronger = False
+        self.lhs_key = None
+        self.rhs_key = None
 
     def implies(self, guard, opt):
         if self.op.getopnum() != guard.op.getopnum():
             return False
 
-        my_key = opt._get_key(self.cmp_op)
-        ot_key = opt._get_key(guard.cmp_op)
-
-        if my_key[1] == ot_key[1]:
+        if self.lhs_key == guard.lhs_key:
             # same operation
-            lc = self.compare(self.lhs, guard.lhs)
-            rc = self.compare(self.rhs, guard.rhs)
+            valid, lc = self.compare(self.lhs, guard.lhs)
+            if not valid:
+                return False
+            valid, rc = self.compare(self.rhs, guard.rhs)
+            if not valid:
+                return False
             opnum = self.get_compare_opnum()
             if opnum == -1:
                 return False
@@ -65,40 +65,43 @@
         otherop = other.op
         assert isinstance(otherop, GuardResOp)
         assert isinstance(myop, GuardResOp)
-        self.stronger = True
         self.index = other.index
 
         descr = myop.getdescr()
-        descr.copy_all_attributes_from(other.op.getdescr())
-        myop.rd_frame_info_list = otherop.rd_frame_info_list
-        myop.rd_snapshot = otherop.rd_snapshot
-        myop.setfailargs(otherop.getfailargs())
+        if we_are_translated():
+            descr.copy_all_attributes_from(other.op.getdescr())
+            myop.rd_frame_info_list = otherop.rd_frame_info_list
+            myop.rd_snapshot = otherop.rd_snapshot
+            myop.setfailargs(otherop.getfailargs())
 
     def compare(self, key1, key2):
         if isinstance(key1, Box):
-            assert isinstance(key2, Box)
-            assert key1 is key2 # key of hash enforces this
-            return 0
+            if isinstance(key2, Box) and key1 is key2:
+                return True, 0
+            return False, 0
         #
         if isinstance(key1, ConstInt):
-            assert isinstance(key2, ConstInt)
+            if not isinstance(key2, ConstInt):
+                return False, 0
             v1 = key1.value
             v2 = key2.value
             if v1 == v2:
-                return 0
+                return True, 0
             elif v1 < v2:
-                return -1
+                return True, -1
             else:
-                return 1
+                return True, 1
         #
         if isinstance(key1, IndexVar):
             assert isinstance(key2, IndexVar)
-            return key1.compare(key2)
+            return True, key1.compare(key2)
         #
         raise AssertionError("cannot compare: " + str(key1) + " <=> " + str(key2))
 
     def emit_varops(self, opt, var, old_arg):
         if isinstance(var, IndexVar):
+            if var.is_identity():
+                return var.var
             box = var.emit_operations(opt)
             opt.renamer.start_renaming(old_arg, box)
             return box
@@ -117,105 +120,122 @@
         guard.setarg(0, box_result)
         opt.emit_operation(guard)
 
-class GuardStrengthenOpt(object):
-    def __init__(self, index_vars):
-        self.index_vars = index_vars
-        self._newoperations = []
-        self._same_as = {}
-        self.strength_reduced = 0 # how many guards could be removed?
+    def update_keys(self, index_vars):
+        self.lhs = index_vars.get(self.lhs_arg, self.lhs_arg)
+        if isinstance(self.lhs, IndexVar):
+            self.lhs = self.lhs.var
+        self.lhs_key = self.lhs
+        #
+        self.rhs = index_vars.get(self.rhs_arg, self.rhs_arg)
+        if isinstance(self.rhs, IndexVar):
+            self.rhs = self.rhs.var
+        self.rhs_key = self.rhs
 
-    def find_compare_guard_bool(self, boolarg, operations, index):
+    @staticmethod
+    def of(boolarg, operations, index):
+        guard_op = operations[index]
         i = index - 1
         # most likely hit in the first iteration
         while i > 0:
             op = operations[i]
             if op.result and op.result == boolarg:
-                return op
+                if rop.INT_LT <= op.getopnum() <= rop.INT_GE:
+                    cmp_op = op
+                    break
+                return None
             i -= 1
+        else:
+            raise AssertionError("guard_true/false first arg not defined")
+        #
+        lhs_arg = cmp_op.getarg(0)
+        rhs_arg = cmp_op.getarg(1)
+        return Guard(i, guard_op, cmp_op, lhs_arg, rhs_arg)
 
-        raise AssertionError("guard_true/false first arg not defined")
+class GuardStrengthenOpt(object):
+    def __init__(self, index_vars):
+        self.index_vars = index_vars
+        self._newoperations = []
+        self.strength_reduced = 0 # how many guards could be removed?
+        self.strongest_guards = {}
+        self.guards = {}
 
-    def _get_key(self, cmp_op):
-        if cmp_op and rop.INT_LT <= cmp_op.getopnum() <= rop.INT_GE:
-            lhs_arg = cmp_op.getarg(0)
-            rhs_arg = cmp_op.getarg(1)
-            lhs_index_var = self.index_vars.get(lhs_arg, None)
-            rhs_index_var = self.index_vars.get(rhs_arg, None)
+    #def _get_key(self, cmp_op):
+    #    assert cmp_op
+    #    lhs_arg = cmp_op.getarg(0)
+    #    rhs_arg = cmp_op.getarg(1)
+    #    lhs_index_var = self.index_vars.get(lhs_arg, None)
+    #    rhs_index_var = self.index_vars.get(rhs_arg, None)
 
-            cmp_opnum = cmp_op.getopnum()
-            # get the key, this identifies the guarded operation
-            if lhs_index_var and rhs_index_var:
-                key = (lhs_index_var.getvariable(), cmp_opnum, rhs_index_var.getvariable())
-            elif lhs_index_var:
-                key = (lhs_index_var.getvariable(), cmp_opnum, rhs_arg)
-            elif rhs_index_var:
-                key = (lhs_arg, cmp_opnum, rhs_index_var)
-            else:
-                key = (lhs_arg, cmp_opnum, rhs_arg)
-            return key
-        return (None, 0, None)
+    #    cmp_opnum = cmp_op.getopnum()
+    #    # get the key, this identifies the guarded operation
+    #    if lhs_index_var and rhs_index_var:
+    #        return (lhs_index_var.getvariable(), cmp_opnum, rhs_index_var.getvariable())
+    #    elif lhs_index_var:
+    #        return (lhs_index_var.getvariable(), cmp_opnum, None)
+    #    elif rhs_index_var:
+    #        return (None, cmp_opnum, rhs_index_var)
+    #    else:
+    #        return (None, cmp_opnum, None)
+    #    return key
 
-    def get_key(self, guard_bool, operations, i):
-        cmp_op = self.find_compare_guard_bool(guard_bool.getarg(0), operations, i)
-        return self._get_key(cmp_op)
-
-    def propagate_all_forward(self, loop):
-        """ strengthens the guards that protect an integral value """
-        strongest_guards = {}
-        guards = {}
-        # the guards are ordered. guards[i] is before guards[j] iff i < j
+    def collect_guard_information(self, loop):
         operations = loop.operations
         last_guard = None
         for i,op in enumerate(operations):
             op = operations[i]
-            if op.is_guard() and op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
-                cmp_op = self.find_compare_guard_bool(op.getarg(0), operations, i)
-                key = self._get_key(cmp_op)
-                if key[0] is not None:
-                    lhs_arg = cmp_op.getarg(0)
-                    lhs = self.index_vars.get(lhs_arg, lhs_arg)
-                    rhs_arg = cmp_op.getarg(1)
-                    rhs = self.index_vars.get(rhs_arg, rhs_arg)
-                    other = strongest_guards.get(key, None)
-                    if not other:
-                        guard = Guard(i, op, cmp_op,
-                                      lhs, lhs_arg,
-                                      rhs, rhs_arg)
-                        strongest_guards[key] = guard
-                        # nothing known, at this position emit the guard
-                        guards[i] = guard
-                    else: # implicit index(strongest) < index(current)
-                        guard = Guard(i, op, cmp_op,
-                                      lhs, lhs_arg, rhs, rhs_arg)
-                        if guard.implies(other, self):
-                            guard.inhert_attributes(other)
+            if not op.is_guard():
+                continue
+            if op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
+                guard = Guard.of(op.getarg(0), operations, i)
+                if guard is None:
+                    continue
+                guard.update_keys(self.index_vars)
+                self.record_guard(guard.lhs_key, guard)
+                self.record_guard(guard.rhs_key, guard)
 
-                            strongest_guards[key] = guard
-                            guards[other.index] = guard
-                            # do not mark as emit
-                            continue
-                        elif other.implies(guard, self):
-                            guard.implied = True
-                        # mark as emit
-                        guards[i] = guard
-                else:
-                    # emit non guard_true/false guards
-                    guards[i] = Guard(i, op, None, None, None, None, None)
+    def record_guard(self, key, guard):
+        if key is None:
+            return
+        # the operations are processed from 1..n (forward),
+        # thus if the key is not present (1), the guard is saved
+        # (2) guard(s) with this key is/are already present,
+        # thus each of is seen as possible candidate to strengthen
+        # or imply the current. in both cases the current guard is
+        # not emitted and the original is replaced with the current
+        others = self.strongest_guards.setdefault(key, [])
+        if len(others) > 0: # (2)
+            for i,other in enumerate(others):
+                if guard.implies(other, self):
+                    # strengthend
+                    guard.inhert_attributes(other)
+                    others[i] = guard
+                    self.guards[other.index] = guard
+                    self.guards[guard.index] = None # mark as 'do not emit'
+                    continue
+                elif other.implies(guard, self):
+                    # implied
+                    self.guards[guard.index] = None # mark as 'do not emit'
+                    continue
+        else: # (2)
+            others.append(guard)
 
-        strongest_guards = None
-        #
+    def eliminate_guards(self, loop):
         self.renamer = Renamer()
-        last_op_idx = len(operations)-1
-        for i,op in enumerate(operations):
-            op = operations[i]
-            if op.is_guard() and op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
-                guard = guards.get(i, None)
-                if not guard or guard.implied:
+        for i,op in enumerate(loop.operations):
+            op = loop.operations[i]
+            if op.is_guard():
+                if i in self.guards:
+                    # either a stronger guard has been saved
+                    # or it should not be emitted
+                    guard = self.guards[i]
                     # this guard is implied or marked as not emitted (= None)
                     self.strength_reduced += 1
+                    if guard is None:
+                        continue
+                    guard.emit_operations(self)
                     continue
-                if guard.stronger:
-                    guard.emit_operations(self)
+                else:
+                    self.emit_operation(op)
                     continue
             if op.result:
                 index_var = self.index_vars.get(op.result, None)
@@ -224,8 +244,15 @@
                         index_var.emit_operations(self, op.result)
                         continue
             self.emit_operation(op)
+        #
+        loop.operations = self._newoperations[:]
 
-        loop.operations = self._newoperations[:]
+    def propagate_all_forward(self, loop):
+        """ strengthens the guards that protect an integral value """
+        # the guards are ordered. guards[i] is before guards[j] iff i < j
+        self.collect_guard_information(loop)
+        #
+        self.eliminate_guards(loop)
 
     def emit_operation(self, op):
         self.renamer.rename(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_guard.py b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
@@ -0,0 +1,159 @@
+import py
+
+from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
+from rpython.jit.metainterp.optimizeopt.util import equaloplists
+from rpython.jit.metainterp.optimizeopt.vectorize import (VecScheduleData,
+        Pack, NotAProfitableLoop, VectorizingOptimizer)
+from rpython.jit.metainterp.optimizeopt.dependency import Node, DependencyGraph
+from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
+from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
+from rpython.jit.metainterp.optimizeopt.test.test_schedule import SchedulerBaseTest
+from rpython.jit.metainterp.optimizeopt.test.test_vectorize import (FakeMetaInterpStaticData,
+        FakeJitDriverStaticData)
+from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.tool.oparser_model import get_model
+
+class FakeMemoryRef(object):
+    def __init__(self, array, iv):
+        self.index_var = iv
+        self.array = array
+
+    def is_adjacent_to(self, other):
+        if self.array is not other.array:
+            return False
+        iv = self.index_var
+        ov = other.index_var
+        val = (int(str(ov.var)[1:]) - int(str(iv.var)[1:]))
+        # i0 and i1 are adjacent
+        # i1 and i0 ...
+        # but not i0, i2
+        # ...
+        return abs(val) == 1
+
+class GuardBaseTest(SchedulerBaseTest):
+    def optguards(self, loop):
+        dep = DependencyGraph(loop)
+        opt = GuardStrengthenOpt(dep.index_vars)
+        opt.propagate_all_forward(loop)
+        return opt
+
+    def assert_guard_count(self, loop, count):
+        guard = 0
+        for op in loop.operations:
+            if op.is_guard():
+                guard += 1
+        if guard != count:
+            self.debug_print_operations(loop)
+        assert guard == count
+
+    def assert_contains_sequence(self, loop, instr):
+        class Glob(object):
+            def __repr__(self):
+                return '*'
+        from rpython.jit.tool.oparser import OpParser, default_fail_descr
+        parser = OpParser(instr, self.cpu, self.namespace(), 'lltype', None, default_fail_descr, True, None)
+        operations = []
+        last_glob = None
+        prev_op = None
+        for line in instr.splitlines():
+            line = line.strip()
+            if line.startswith("#") or \
+               line == "":
+                continue
+            if line.startswith("..."):
+                last_glob = Glob()
+                last_glob.prev = prev_op
+                operations.append(last_glob)
+                continue
+            op = parser.parse_next_op(line)
+            if last_glob is not None:
+                last_glob.next = op
+                last_glob = None
+            operations.append(op)
+            prev_op = op
+
+        def check(op, candidate, rename):
+            if isinstance(candidate, Glob):
+                if candidate.next is None:
+                    return 0 # consumes the rest
+                if op.getopnum() != candidate.next.getopnum():
+                    return 0
+                candidate = candidate.next
+            if op.getopnum() == candidate.getopnum():
+                for i,arg in enumerate(op.getarglist()):
+                    oarg = candidate.getarg(i)
+                    if arg in rename:
+                        assert rename[arg] is oarg
+                    else:
+                        rename[arg] = oarg
+
+                if op.result:
+                    rename[op.result] = candidate.result
+                return 1
+            return 0
+        j = 0
+        rename = {}
+        for i, op in enumerate(loop.operations):
+            candidate = operations[j]
+            j += check(op, candidate, rename)
+        if isinstance(operations[0], Glob):
+            assert j == len(operations)-2
+        else:
+            assert j == len(operations)-1
+
+    def test_basic(self):
+        loop1 = self.parse("""
+        i10 = int_lt(i1, 42)
+        guard_true(i10) []
+        i11 = int_add(i1, 1)
+        i12 = int_lt(i11, 42)
+        guard_true(i12) []
+        """)
+        opt = self.optguards(loop1)
+        self.assert_guard_count(loop1, 1)
+        self.assert_contains_sequence(loop1, """
+        ...
+        i11 = int_add(i1, 1)
+        i12 = int_lt(i11, 42)
+        guard_true(i12) []
+        ...
+        """)
+
+    def test_basic_sub(self):
+        loop1 = self.parse("""
+        i10 = int_gt(i1, 42)
+        guard_true(i10) []
+        i11 = int_sub(i1, 1)
+        i12 = int_gt(i11, 42)
+        guard_true(i12) []
+        """)
+        opt = self.optguards(loop1)
+        self.assert_guard_count(loop1, 1)
+        self.assert_contains_sequence(loop1, """
+        ...
+        i11 = int_sub(i1, 1)
+        i12 = int_gt(i11, 42)
+        guard_true(i12) []
+        ...
+        """)
+
+    def test_collapse(self):
+        loop1 = self.parse("""
+        i10 = int_gt(i1, 42)
+        guard_true(i10) []
+        i11 = int_sub(i1, 1)
+        i12 = int_gt(i11, 42)
+        guard_true(i12) []
+        """)
+        opt = self.optguards(loop1)
+        self.assert_guard_count(loop1, 1)
+        self.assert_contains_sequence(loop1, """
+        ...
+        i11 = int_sub(i1, 1)
+        i12 = int_gt(i11, 42)
+        guard_true(i12) []
+        ...
+        """)
+
+class Test(GuardBaseTest, LLtypeMixin):
+    pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -25,13 +25,8 @@
 
 class SchedulerBaseTest(DependencyBaseTest):
 
-    def parse(self, source, inc_label_jump=True,
-              pargs=2,
-              iargs=10,
-              fargs=6,
-              additional_args=None,
-              replace_args=None):
-        ns = {
+    def namespace(self):
+        return {
             'double': self.floatarraydescr,
             'float': self.singlefloatarraydescr,
             'long': self.intarraydescr,
@@ -39,6 +34,13 @@
             'short': self.int16arraydescr,
             'char': self.chararraydescr,
         }
+
+    def parse(self, source, inc_label_jump=True,
+              pargs=2,
+              iargs=10,
+              fargs=6,
+              additional_args=None,
+              replace_args=None):
         args = []
         for prefix, rang in [('p',range(pargs)), ('i',range(iargs)), ('f',range(fargs))]:
             for i in rang:
@@ -56,7 +58,7 @@
         joinedargs = ','.join(args)
         fmt = (indent, joinedargs, source, indent, joinedargs)
         src = "%s[%s]\n%s\n%sjump(%s)" % fmt
-        loop = opparse(src, cpu=self.cpu, namespace=ns)
+        loop = opparse(src, cpu=self.cpu, namespace=self.namespace())
         if inc_label_jump:
             token = JitCellToken()
             loop.operations = \
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1360,19 +1360,47 @@
 
     def test_abc(self):
         trace="""
-        [i16, i17, i18, i5, p6, p7, f19, p9, p10, p11, p12, p13, p14, p15, i20, i21]
+        [p0, p1, p5, p6, p7, p12, p13, i14, i15, i16, i17, i18, i19, i20]
         guard_early_exit() []
-        f22 = raw_load(i20, i18, descr=<ArrayF 8>)
-        guard_not_invalidated(descr=<rpython.jit.metainterp.compile.ResumeGuardNotInvalidated object at 0x7fc428762410>) [i5, i18, i17, i16, p15, p14, p13, p12, p11, p10, p9, p7, p6, f22, f19]
-        f23 = raw_load(i21, i17, descr=<ArrayF 8>)
-        f24 = float_mul(f22, f23)
-        f25 = float_add(f19, f24)
-        i27 = int_add(i18, 8)
-        i29 = int_add(i17, 8)
-        i30 = int_lt(i16, i5)
-        guard_true(i30, descr=<rpython.jit.metainterp.compile.ResumeGuardTrueDescr object at 0x7fc4287689d0>) [i5, i27, i29, i16, p15, p14, p13, p12, p11, p10, p9, p7, p6, f25, None]
-        i33 = int_add(i16, 1)
-        jump(i33, i29, i27, i5, p6, p7, f25, p9, p10, p11, p12, p13, p14, p15, i20, i21)
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #117 LOAD_NAME')
+        guard_not_invalidated(descr=<ResumeGuardNotInvalidated object at 0x7fc657d7be20>) [p1, p0, p5, p6, p7, p12, p13]
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #120 LOAD_CONST')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #123 COMPARE_OP')
+        i22 = int_lt(i14, 2024)
+        guard_true(i22, descr=<ResumeGuardTrueDescr object at 0x7fc657d7bdc0>) [p1, p0, p5, p6, p7, p12, p13, i14]
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #126 POP_JUMP_IF_FALSE')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #129 LOAD_NAME')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #132 LOAD_NAME')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #135 BINARY_SUBSCR')
+        i23 = int_lt(i14, i15)
+        guard_true(i23, descr=<ResumeGuardTrueDescr object at 0x7fc657d7bd60>) [p1, p0, i14, p5, p6, p7, p12, p13, None]
+        f25 = getarrayitem_raw(i16, i14, descr=floatarraydescr)
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #136 LOAD_NAME')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #139 LOAD_NAME')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #142 BINARY_SUBSCR')
+        i26 = int_lt(i14, i17)
+        guard_true(i26, descr=<ResumeGuardTrueDescr object at 0x7fc657d7bca0>) [p1, p0, i14, p5, p6, p7, p12, p13, f25, None]
+        f27 = getarrayitem_raw(i18, i14, descr=floatarraydescr)
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #143 BINARY_ADD')
+        f28 = float_add(f25, f27)
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #144 LOAD_NAME')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #147 LOAD_NAME')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #150 STORE_SUBSCR')
+        i29 = int_lt(i14, i19)
+        guard_true(i29, descr=<ResumeGuardTrueDescr object at 0x7fc657d7bc40>) [p1, p0, i14, p5, p6, p7, p12, p13, f28, None, None]
+        setarrayitem_raw(i20, i14, f28, descr=floatarraydescr)
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #151 LOAD_NAME')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #154 LOAD_CONST')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #157 INPLACE_ADD')
+        i31 = int_add(i14, 1)
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #158 STORE_NAME')
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #161 JUMP_ABSOLUTE')
+        i33 = getfield_raw(140489852409120, descr=<FieldS pypysig_long_struct.c_value 0>)
+        setfield_gc(1234, i31, descr=<FieldS pypy.objspace.std.typeobject.IntMutableCell.inst_intvalue 8>)
+        i36 = int_lt(i33, 0)
+        guard_false(i36, descr=<ResumeGuardFalseDescr object at 0x7fc657d7be80>) [p1, p0, p5, p6, p7, p12, p13, None, None, None]
+        debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #117 LOAD_NAME')
+        jump(p0, p1, p5, p6, p7, p12, p13, i31, i15, i16, i17, i18, i19, i20)
         """
         # schedule 885 -> ptype is non for raw_load?
         opt = self.vectorize(self.parse_loop(trace))

From noreply at buildbot.pypy.org  Wed Jul 15 21:19:12 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 15 Jul 2015 21:19:12 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: merge default into branch
Message-ID: <20150715191912.2C0CD1C028A@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78556:b07b520cb8f4
Date: 2015-07-13 00:39 +0300
http://bitbucket.org/pypy/pypy/changeset/b07b520cb8f4/

Log:	merge default into branch

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -617,15 +617,17 @@
         self.impl = impl
         self.readonly = readonly
 
-    def getitem(self, item):
-        return raw_storage_getitem(lltype.Char, self.impl.storage, item)
+    def getitem(self, index):
+        return raw_storage_getitem(lltype.Char, self.impl.storage,
+                 index + self.impl.start)
 
-    def setitem(self, item, v):
-        raw_storage_setitem(self.impl.storage, item,
+    def setitem(self, index, v):
+        raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
     def getlength(self):
-        return self.impl.size
+        return self.impl.size - self.impl.start
 
     def get_raw_address(self):
-        return self.impl.storage
+        from rpython.rtyper.lltypesystem import rffi
+        return rffi.ptradd(self.impl.storage, self.impl.start)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -77,7 +77,7 @@
     stop = 1
     step = 1
     lgt = 1
-    axis_step = 0
+    axis_step = 0 # both skip this axis in calculate_slice_strides and set stride => 0
 
     def __init__(self):
         pass
@@ -127,7 +127,7 @@
         except IndexError:
             continue
         if chunk.step != 0:
-            rstrides[j] = s_i * chunk.step
+            rstrides[j] = s_i * chunk.step * chunk.axis_step
             rbackstrides[j] = s_i * max(0, chunk.lgt - 1) * chunk.step
             rshape[j] = chunk.lgt
             j += 1
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -764,6 +764,8 @@
         assert (a[1:] == b).all()
         assert (a[1:,newaxis] == d).all()
         assert (a[newaxis,1:] == c).all()
+        assert a.strides == (8,)
+        assert a[:, newaxis].strides == (8, 0)
 
     def test_newaxis_assign(self):
         from numpy import array, newaxis
@@ -2345,6 +2347,7 @@
         assert a[1] == 0xff
         assert len(a.data) == 16
         assert type(a.data) is buffer
+        assert a[1:].data._pypy_raw_address() - a.data._pypy_raw_address() == a.strides[0]
 
     def test_explicit_dtype_conversion(self):
         from numpy import array
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8637,5 +8637,27 @@
         """
         self.optimize_loop(ops, expected, preamble)
 
+    def test_getfield_proven_constant(self):
+        py.test.skip("not working")
+        ops = """
+        [p0]
+        i1 = getfield_gc(p0, descr=valuedescr)
+        guard_value(i1, 13) []
+        escape(i1)
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        escape(13)
+        jump(p0)
+        """
+        expected_short = """
+        [p0]
+        i1 = getfield_gc(p0, descr=valuedescr)
+        guard_value(i1, 13) []
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected, expected_short=expected_short)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Wed Jul 15 21:19:13 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 15 Jul 2015 21:19:13 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: hack at dict-based vs.
 list-based repr(dtype) for compatibility, fix union dtypes, fix translation
Message-ID: <20150715191913.6E1A41C028A@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78557:8937dc4116db
Date: 2015-07-15 22:19 +0300
http://bitbucket.org/pypy/pypy/changeset/8937dc4116db/

Log:	hack at dict-based vs. list-based repr(dtype) for compatibility, fix
	union dtypes, fix translation

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -182,14 +182,27 @@
             name = name[:-1]
         return name
 
-    def descr_get_name(self, space):
-        name = self.get_name()
+    def descr_get_name(self, space, quote=False):
+        if quote:
+            name = "'" + self.get_name() + "'"
+        else:
+            name = self.get_name()
         if self.is_flexible() and self.elsize != 0:
             return space.wrap(name + str(self.elsize * 8))
         return space.wrap(name)
 
-    def descr_get_str(self, space):
-        return space.wrap(self.get_str())
+    def descr_get_str(self, space, ignore='|', simple=True):
+        if not simple and self.fields and len(self.fields) > 0:
+            return self.descr_get_descr(space)
+        total = 0
+        for s in self.shape:
+            total += s
+        if not simple and total > 0:
+            return space.newtuple(
+                [space.wrap(self.subdtype.get_str(ignore='')), 
+                 space.newtuple([space.wrap(s) for s in self.shape]),
+                ])
+        return space.wrap(self.get_str(ignore=ignore))
 
     def get_str(self, ignore='|'):
         basic = self.kind
@@ -203,11 +216,14 @@
             size >>= 2
         return "%s%s%s" % (endian, basic, size)
 
-    def descr_get_descr(self, space, style='descr'):
+    def descr_get_descr(self, space, style='descr', force_dict=False):
+        simple = False
+        if style == 'descr':
+            simple = True
         if not self.is_record():
             return space.newlist([space.newtuple([space.wrap(""),
-                                                  self.descr_get_str(space)])])
-        elif self.alignment >= 0 and style != 'descr':
+                                                  self.descr_get_str(space, simple=simple)])])
+        elif (self.alignment > 1 and style != 'descr') or force_dict:
             # we need to force a sorting order for the keys,
             # so return a string instead of a dict. Also, numpy formats
             # the lists without spaces between elements, so we cannot simply
@@ -217,6 +233,9 @@
             offsets = "'offsets':["
             titles = "'titles':["
             use_titles = False
+            show_offsets = False
+            offsets_n = []
+            total = 0
             for name, title in self.names:
                 offset, subdtype = self.fields[name]
                 if subdtype.is_record():
@@ -232,20 +251,37 @@
                 titles += "'" + str(title) + "',"
                 if title is not None:
                     use_titles = True
+                if total != offset:
+                    show_offsets = True
+                total += subdtype.elsize
+                # make sure offsets_n is sorted
+                i = 0
+                for i in range(len(offsets_n)):
+                    if offset < offsets_n[i]:
+                        break
+                offsets_n.insert(i, offset)
+            total = 0
+            for i in range(len(offsets_n)):
+                if offsets_n[i] != self.alignment * i:
+                    show_offsets = True
             formats = formats[:-1] + ']'
             offsets = offsets[:-1] + ']'
             names = names[:-1] + ']'
             titles = titles[:-1] + ']'
-            if style == 'str':
+            if self.alignment < 2:
+                suffix = "}"
+            elif style == 'str':
                 suffix = ", 'aligned':True}"
             elif style == 'substr':
                 suffix = '}'
             else:
                 suffix = "}, align=True"
-            if use_titles: 
+            if use_titles and not show_offsets: 
+                return self.descr_get_descr(space, style='descr')
+            elif use_titles:
                 return space.wrap('{' + names + ', ' + formats + ', ' +
-                            offsets + ', ' + "'itemsize':" + str(self.elsize) +
-                            titles + ', ' + suffix)
+                            offsets + ', ' + titles + ", 'itemsize':" + 
+                            str(self.elsize) + suffix)
             else:
                 return space.wrap('{' + names + ', ' + formats + ', ' +
                             offsets + ', ' + "'itemsize':" + str(self.elsize) +
@@ -253,18 +289,31 @@
             
         else:
             descr = []
+            total = 0
             for name, title in self.names:
-                subdtype = self.fields[name][1]
-                subdescr = [space.wrap(name)]
+                offset, subdtype = self.fields[name]
+                show_offsets = False
+                if total != offset:
+                    # whoops, need to use other format
+                    return self.descr_get_descr(space, style=style, force_dict=True)
+                total += subdtype.elsize
+                ignore = '|'
+                if title:
+                    subdescr = [space.newtuple([space.wrap(title), space.wrap(name)])]
+                    ignore = ''
+                else:
+                    subdescr = [space.wrap(name)]
                 if subdtype.is_record():
                     subdescr.append(subdtype.descr_get_descr(space, style))
                 elif subdtype.subdtype is not None:
-                    subdescr.append(subdtype.subdtype.descr_get_str(space))
+                    subdescr.append(subdtype.subdtype.descr_get_str(space, simple=False))
                 else:
-                    subdescr.append(subdtype.descr_get_str(space))
+                    subdescr.append(subdtype.descr_get_str(space, ignore=ignore, simple=False))
                 if subdtype.shape != []:
                     subdescr.append(subdtype.descr_get_shape(space))
                 descr.append(space.newtuple(subdescr[:]))
+            if self.alignment >= 0:
+                return space.wrap(space.str_w(space.repr(space.newlist(descr))) + ', align=True')                 
             return space.newlist(descr)
 
     def descr_get_hasobject(self, space):
@@ -457,9 +506,11 @@
                 size = self.elsize
                 if self.num == NPY.UNICODE:
                     size >>= 2
-                r = space.wrap(byteorder + self.char + str(size))
+                r = space.wrap("'" + byteorder + self.char + str(size) + "'")
             else:
-                r = self.descr_get_name(space)
+                r = self.descr_get_name(space, quote=True)
+        if space.isinstance_w(r, space.w_str):
+            return space.wrap("dtype(%s)" % space.str_w(r))
         return space.wrap("dtype(%s)" % space.str_w(space.repr(r)))
 
     def descr_getitem(self, space, w_item):
@@ -626,6 +677,7 @@
     fldnames = [''] * len(lst_w)
     subdtypes = [None] * len(lst_w)
     titles = [None] * len(lst_w)
+    total = 0
     for i in range(len(lst_w)):
         w_elem = lst_w[i]
         if simple:
@@ -675,6 +727,15 @@
             if  i + 1 < len(offsets) and offsets[i + 1] == 0:
                 offsets[i+1] = offsets[i] + subdtype.elsize
         subdtypes[i] = subdtype
+        if use_supplied_offsets:
+            sz = subdtype.elsize
+        else:
+            sz = max(maxalign, subdtype.elsize)
+        if offsets[i] + sz > total:
+            total = offsets[i] + sz
+    # padding?
+    if alignment >= 0 and total % maxalign:
+        total = total // maxalign * maxalign + maxalign
     names = []
     for i in range(len(subdtypes)):
         subdtype = subdtypes[i]
@@ -683,7 +744,7 @@
             subdtype.alignment = maxalign
         if fldnames[i] in fields:
             raise oefmt(space.w_ValueError, "two fields with the same name")
-        if maxalign > 1 and offsets[i] % maxalign:
+        if maxalign > 1 and offsets[i] % subdtype.alignment:
             raise oefmt(space.w_ValueError, "offset %d for NumPy dtype with "
                     "fields is not divisible by the field alignment %d "
                     "with align=True", offsets[i], maxalign)
@@ -693,7 +754,6 @@
                 raise oefmt(space.w_ValueError, "two fields with the same name")
             fields[titles[i]] = offsets[i], subdtype
         names.append((fldnames[i], titles[i]))
-    total = offsets[-1] + max(maxalign, fields[names[-1][0]][1].elsize)
     if itemsize > 1:
         if total > itemsize:
             raise oefmt(space.w_ValueError,
@@ -878,10 +938,6 @@
     shape_w = space.fixedview(w_shape)
     if len(shape_w) < 1:
         return None
-    if len(shape_w) == 1:
-        if (not space.isinstance_w(shape_w[0], space.w_int) and
-            not space.isinstance_w(shape_w[0], space.w_long)):
-            return None
     shape = []
     for w_dim in shape_w:
         try:
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1439,7 +1439,8 @@
                     "'offsets':[0,2], "
                     "'titles':['Red pixel','Blue pixel'], "
                     "'itemsize':4})")
-
+        if 'datetime64' not in dir(np):
+            skip('datetime dtype not available')
         dt = np.dtype([('a', '<M8[D]'), ('b', '<m8[us]')])
         assert repr(dt) == (
                     "dtype([('a', '<M8[D]'), ('b', '<m8[us]')])")

From noreply at buildbot.pypy.org  Thu Jul 16 12:16:52 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 16 Jul 2015 12:16:52 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added ABC optimization that is turned on
 when executed when user code is vectorized
Message-ID: <20150716101652.3126A1C12F1@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78558:04f489dd60dd
Date: 2015-07-16 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/04f489dd60dd/

Log:	added ABC optimization that is turned on when executed when user
	code is vectorized note that this optimization versions the loop
	immediatly (to be tested) it introduces a guard before the loop is
	entered to remove guards that are contained within the loop body

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -746,7 +746,7 @@
 
 class LoopVersion(object):
 
-    def __init__(self, operations, opt_ops, invariant_arg_count=0, aligned=False):
+    def __init__(self, operations, aligned=False):
         self.operations = operations
         self.aligned = aligned
         self.faildescrs = []
@@ -756,6 +756,8 @@
         label = operations[idx]
         self.label_pos = idx
         self.inputargs = label.getarglist()
+
+    def register_all_guards(self, opt_ops, invariant_arg_count=0):
         idx = index_of_first(rop.LABEL, opt_ops)
         assert idx >= 0
         version_failargs = opt_ops[idx].getarglist()
@@ -769,6 +771,7 @@
 
         for op in opt_ops:
             if op.is_guard():
+                import pdb; pdb.set_trace()
                 assert isinstance(op, GuardResOp)
                 descr = op.getdescr()
                 if descr.loop_version():
@@ -780,6 +783,13 @@
                     op.setfailargs(version_failargs)
                     op.rd_snapshot = None
 
+    def register_guard(self, op):
+        assert isinstance(op, GuardResOp)
+        descr = op.getdescr()
+        self.faildescrs.append(descr)
+        op.setfailargs(self.inputargs)
+        op.rd_snapshot = None
+
     def copy_operations(self):
         return [op.clone() for op in self.operations]
 
@@ -803,7 +813,6 @@
     call_pure_results = None
     logops = None
     quasi_immutable_deps = None
-    versions = None
 
     def _token(*args):
         raise Exception("TreeLoop.token is killed")
@@ -816,6 +825,7 @@
 
     def __init__(self, name):
         self.name = name
+        self.versions = []
         # self.operations = list of ResOperations
         #   ops of the kind 'guard_xxx' contain a further list of operations,
         #   which may itself contain 'guard_xxx' and so on, making a tree.
@@ -841,6 +851,14 @@
         """ return the first operation having the same opnum or -1 """
         return index_of_first(opnum, self.operations)
 
+    def snapshot(self):
+        version = LoopVersion(self.copy_operations(), [])
+        self.versions.append(version)
+        return version
+
+    def copy_operations(self):
+        return [ op.clone() for op in self.operations ]
+
     def get_display_text(self):    # for graphpage.py
         return self.name + '\n' + repr(self.inputargs)
 
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -73,7 +73,7 @@
                  or warmstate.vectorize_user):
                 optimize_vector(metainterp_sd, jitdriver_sd, loop,
                                 optimizations, inline_short_preamble,
-                                start_state, warmstate.vec_cost)
+                                start_state, warmstate)
             else:
                 return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
                                        optimizations, inline_short_preamble,
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -891,8 +891,9 @@
         return mycoeff + self.constant - (othercoeff + other.constant)
 
     def emit_operations(self, opt, result_box=None):
-        assert not self.is_identity()
         box = self.var
+        if self.is_identity():
+            return box
         last_op = None
         if self.coefficient_mul != 1:
             box_result = box.clonebox()
@@ -904,25 +905,31 @@
             last_op = ResOperation(rop.INT_FLOORDIV, [box, ConstInt(self.coefficient_div)], box_result)
             opt.emit_operation(last_op)
             box = box_result
-        if self.constant != 0:
+        if self.constant > 0:
             box_result = box.clonebox()
             last_op = ResOperation(rop.INT_ADD, [box, ConstInt(self.constant)], box_result)
             opt.emit_operation(last_op)
             box = box_result
+        if self.constant < 0:
+            box_result = box.clonebox()
+            last_op = ResOperation(rop.INT_SUB, [box, ConstInt(self.constant)], box_result)
+            opt.emit_operation(last_op)
+            box = box_result
         if result_box is not None:
             last_op.result = box = result_box
         return box
 
     def compare(self, other):
-        assert isinstance(other, IndexVar)
+        """ returns if the two are compareable as a first result
+            and a number (-1,0,1) of the ordering
+        """
         v1 = (self.coefficient_mul // self.coefficient_div) + self.constant
         v2 = (other.coefficient_mul // other.coefficient_div) + other.constant
-        if v1 == v2:
-            return 0
-        elif v1 < v2:
-            return -1
-        else:
-            return 1
+        c = (v1 - v2)
+        if self.var.same_box(other.var):
+            #print "cmp(",self,",",other,") =>", (v1 - v2)
+            return True, (v1 - v2)
+        return False, 0
 
     def __repr__(self):
         if self.is_identity():
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -16,27 +16,38 @@
     """ An object wrapper around a guard. Helps to determine
         if one guard implies another
     """
-    def __init__(self, index, op, cmp_op, lhs_arg, rhs_arg):
+    def __init__(self, index, op, cmp_op, index_vars):
         self.index = index
         self.op = op
         self.cmp_op = cmp_op
-        self.lhs_arg = lhs_arg
-        self.rhs_arg = rhs_arg
         self.lhs_key = None
         self.rhs_key = None
+        lhs = cmp_op.getarg(0)
+        self.lhs = index_vars.get(lhs, None)
+        if self.lhs is None:
+            self.lhs = IndexVar(lhs)
+        #
+        rhs = cmp_op.getarg(1)
+        self.rhs = index_vars.get(rhs, None)
+        if self.rhs is None:
+            self.rhs = IndexVar(rhs)
+
+    def getleftkey(self):
+        return self.lhs.getvariable()
+
+    def getrightkey(self):
+        return self.rhs.getvariable()
 
     def implies(self, guard, opt):
         if self.op.getopnum() != guard.op.getopnum():
             return False
 
-        if self.lhs_key == guard.lhs_key:
+        if self.getleftkey() is guard.getleftkey():
             # same operation
-            valid, lc = self.compare(self.lhs, guard.lhs)
-            if not valid:
-                return False
-            valid, rc = self.compare(self.rhs, guard.rhs)
-            if not valid:
-                return False
+            valid, lc = self.lhs.compare(guard.lhs)
+            if not valid: return False
+            valid, rc = self.rhs.compare(guard.rhs)
+            if not valid: return False
             opnum = self.get_compare_opnum()
             if opnum == -1:
                 return False
@@ -53,6 +64,35 @@
                 return (lc <= 0 and rc >= 0) or (lc == 0 and rc >= 0)
         return False
 
+    def transitive_imply(self, other, opt):
+        if self.op.getopnum() != other.op.getopnum():
+            # stronger restriction, intermixing e.g. <= and < would be possible
+            return None
+        if self.getleftkey() is not other.getleftkey():
+            return None
+        if not self.rhs.is_identity():
+            # stronger restriction
+            return None
+        # this is a valid transitive guard that eliminates the loop guard
+        opnum = self.transitive_cmpop(self.cmp_op.getopnum())
+        box_rhs = self.emit_varops(opt, self.rhs, self.cmp_op.getarg(1))
+        other_rhs = self.emit_varops(opt, other.rhs, other.cmp_op.getarg(1))
+        box_result = self.cmp_op.result.clonebox()
+        opt.emit_operation(ResOperation(opnum, [box_rhs, other_rhs], box_result))
+        # guard
+        guard = self.op.clone()
+        guard.setarg(0, box_result)
+        opt.emit_operation(guard)
+
+        return guard
+
+    def transitive_cmpop(self, opnum):
+        if opnum == rop.INT_LT:
+            return rop.INT_LE
+        if opnum == rop.INT_GT:
+            return rop.INT_GE
+        return opnum
+
     def get_compare_opnum(self):
         opnum = self.op.getopnum()
         if opnum == rop.GUARD_TRUE:
@@ -74,65 +114,38 @@
             myop.rd_snapshot = otherop.rd_snapshot
             myop.setfailargs(otherop.getfailargs())
 
-    def compare(self, key1, key2):
-        if isinstance(key1, Box):
-            if isinstance(key2, Box) and key1 is key2:
-                return True, 0
-            return False, 0
-        #
-        if isinstance(key1, ConstInt):
-            if not isinstance(key2, ConstInt):
-                return False, 0
-            v1 = key1.value
-            v2 = key2.value
-            if v1 == v2:
-                return True, 0
-            elif v1 < v2:
-                return True, -1
-            else:
-                return True, 1
-        #
-        if isinstance(key1, IndexVar):
-            assert isinstance(key2, IndexVar)
-            return True, key1.compare(key2)
-        #
-        raise AssertionError("cannot compare: " + str(key1) + " <=> " + str(key2))
-
     def emit_varops(self, opt, var, old_arg):
-        if isinstance(var, IndexVar):
-            if var.is_identity():
-                return var.var
-            box = var.emit_operations(opt)
-            opt.renamer.start_renaming(old_arg, box)
-            return box
-        else:
-            return var
+        assert isinstance(var, IndexVar)
+        if var.is_identity():
+            return var.var
+        box = var.emit_operations(opt)
+        opt.renamer.start_renaming(old_arg, box)
+        return box
 
     def emit_operations(self, opt):
-        lhs, opnum, rhs = opt._get_key(self.cmp_op)
         # create trace instructions for the index
-        box_lhs = self.emit_varops(opt, self.lhs, self.lhs_arg)
-        box_rhs = self.emit_varops(opt, self.rhs, self.rhs_arg)
+        box_lhs = self.emit_varops(opt, self.lhs, self.cmp_op.getarg(0))
+        box_rhs = self.emit_varops(opt, self.rhs, self.cmp_op.getarg(1))
         box_result = self.cmp_op.result.clonebox()
-        opt.emit_operation(ResOperation(opnum, [box_lhs, box_rhs], box_result))
-        # guard
+        opnum = self.cmp_op.getopnum()
+        cmp_op = ResOperation(opnum, [box_lhs, box_rhs], box_result)
+        opt.emit_operation(cmp_op)
+        # emit that actual guard
         guard = self.op.clone()
         guard.setarg(0, box_result)
         opt.emit_operation(guard)
+        guard.index = opt.operation_position()-1
+        guard.op = guard
+        guard.cmp_op = cmp_op
 
-    def update_keys(self, index_vars):
-        self.lhs = index_vars.get(self.lhs_arg, self.lhs_arg)
-        if isinstance(self.lhs, IndexVar):
-            self.lhs = self.lhs.var
-        self.lhs_key = self.lhs
-        #
-        self.rhs = index_vars.get(self.rhs_arg, self.rhs_arg)
-        if isinstance(self.rhs, IndexVar):
-            self.rhs = self.rhs.var
-        self.rhs_key = self.rhs
+    def set_to_none(self, operations):
+        assert operations[self.index] is self.op
+        operations[self.index] = None
+        if operations[self.index-1] is self.cmp_op:
+            operations[self.index-1] = None
 
     @staticmethod
-    def of(boolarg, operations, index):
+    def of(boolarg, operations, index, index_vars):
         guard_op = operations[index]
         i = index - 1
         # most likely hit in the first iteration
@@ -147,9 +160,7 @@
         else:
             raise AssertionError("guard_true/false first arg not defined")
         #
-        lhs_arg = cmp_op.getarg(0)
-        rhs_arg = cmp_op.getarg(1)
-        return Guard(i, guard_op, cmp_op, lhs_arg, rhs_arg)
+        return Guard(index, guard_op, cmp_op, index_vars)
 
 class GuardStrengthenOpt(object):
     def __init__(self, index_vars):
@@ -159,25 +170,6 @@
         self.strongest_guards = {}
         self.guards = {}
 
-    #def _get_key(self, cmp_op):
-    #    assert cmp_op
-    #    lhs_arg = cmp_op.getarg(0)
-    #    rhs_arg = cmp_op.getarg(1)
-    #    lhs_index_var = self.index_vars.get(lhs_arg, None)
-    #    rhs_index_var = self.index_vars.get(rhs_arg, None)
-
-    #    cmp_opnum = cmp_op.getopnum()
-    #    # get the key, this identifies the guarded operation
-    #    if lhs_index_var and rhs_index_var:
-    #        return (lhs_index_var.getvariable(), cmp_opnum, rhs_index_var.getvariable())
-    #    elif lhs_index_var:
-    #        return (lhs_index_var.getvariable(), cmp_opnum, None)
-    #    elif rhs_index_var:
-    #        return (None, cmp_opnum, rhs_index_var)
-    #    else:
-    #        return (None, cmp_opnum, None)
-    #    return key
-
     def collect_guard_information(self, loop):
         operations = loop.operations
         last_guard = None
@@ -186,12 +178,11 @@
             if not op.is_guard():
                 continue
             if op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
-                guard = Guard.of(op.getarg(0), operations, i)
+                guard = Guard.of(op.getarg(0), operations, i, self.index_vars)
                 if guard is None:
                     continue
-                guard.update_keys(self.index_vars)
-                self.record_guard(guard.lhs_key, guard)
-                self.record_guard(guard.rhs_key, guard)
+                self.record_guard(guard.getleftkey(), guard)
+                self.record_guard(guard.getrightkey(), guard)
 
     def record_guard(self, key, guard):
         if key is None:
@@ -204,18 +195,23 @@
         # not emitted and the original is replaced with the current
         others = self.strongest_guards.setdefault(key, [])
         if len(others) > 0: # (2)
+            replaced = False
             for i,other in enumerate(others):
                 if guard.implies(other, self):
                     # strengthend
+                    others[i] = guard
+                    self.guards[guard.index] = None # mark as 'do not emit'
                     guard.inhert_attributes(other)
-                    others[i] = guard
                     self.guards[other.index] = guard
-                    self.guards[guard.index] = None # mark as 'do not emit'
+                    replaced = True
                     continue
                 elif other.implies(guard, self):
                     # implied
                     self.guards[guard.index] = None # mark as 'do not emit'
+                    replaced = True
                     continue
+            if not replaced:
+                others.append(guard)
         else: # (2)
             others.append(guard)
 
@@ -247,14 +243,58 @@
         #
         loop.operations = self._newoperations[:]
 
-    def propagate_all_forward(self, loop):
+    def propagate_all_forward(self, loop, user_code=False):
         """ strengthens the guards that protect an integral value """
         # the guards are ordered. guards[i] is before guards[j] iff i < j
         self.collect_guard_information(loop)
-        #
         self.eliminate_guards(loop)
 
+        if user_code:
+            version = loop.snapshot()
+            self.eliminate_array_bound_checks(loop, version)
+
     def emit_operation(self, op):
         self.renamer.rename(op)
         self._newoperations.append(op)
 
+    def operation_position(self):
+        return len(self._newoperations)
+
+    def eliminate_array_bound_checks(self, loop, version):
+        self._newoperations = []
+        for key, guards in self.strongest_guards.items():
+            if len(guards) <= 1:
+                continue
+            # there is more than one guard for that key,
+            # that is why we could imply the guards 2..n
+            # iff we add invariant guards
+            one = guards[0]
+            for other in guards[1:]:
+                transitive_guard = one.transitive_imply(other, self)
+                if transitive_guard:
+                    other.set_to_none(loop.operations)
+                    version.register_guard(transitive_guard)
+
+        loop.operations = self._newoperations + \
+                [op for op in loop.operations if op]
+
+    # OLD
+    #def _get_key(self, cmp_op):
+    #    assert cmp_op
+    #    lhs_arg = cmp_op.getarg(0)
+    #    rhs_arg = cmp_op.getarg(1)
+    #    lhs_index_var = self.index_vars.get(lhs_arg, None)
+    #    rhs_index_var = self.index_vars.get(rhs_arg, None)
+
+    #    cmp_opnum = cmp_op.getopnum()
+    #    # get the key, this identifies the guarded operation
+    #    if lhs_index_var and rhs_index_var:
+    #        return (lhs_index_var.getvariable(), cmp_opnum, rhs_index_var.getvariable())
+    #    elif lhs_index_var:
+    #        return (lhs_index_var.getvariable(), cmp_opnum, None)
+    #    elif rhs_index_var:
+    #        return (None, cmp_opnum, rhs_index_var)
+    #    else:
+    #        return (None, cmp_opnum, None)
+    #    return key
+
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_guard.py b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_guard.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
@@ -31,10 +31,10 @@
         return abs(val) == 1
 
 class GuardBaseTest(SchedulerBaseTest):
-    def optguards(self, loop):
+    def optguards(self, loop, user_code=False):
         dep = DependencyGraph(loop)
         opt = GuardStrengthenOpt(dep.index_vars)
-        opt.propagate_all_forward(loop)
+        opt.propagate_all_forward(loop, user_code)
         return opt
 
     def assert_guard_count(self, loop, count):
@@ -48,6 +48,8 @@
 
     def assert_contains_sequence(self, loop, instr):
         class Glob(object):
+            next = None
+            prev = None
             def __repr__(self):
                 return '*'
         from rpython.jit.tool.oparser import OpParser, default_fail_descr
@@ -73,33 +75,36 @@
             prev_op = op
 
         def check(op, candidate, rename):
+            m = 0
             if isinstance(candidate, Glob):
                 if candidate.next is None:
                     return 0 # consumes the rest
                 if op.getopnum() != candidate.next.getopnum():
                     return 0
+                m = 1
                 candidate = candidate.next
             if op.getopnum() == candidate.getopnum():
                 for i,arg in enumerate(op.getarglist()):
                     oarg = candidate.getarg(i)
                     if arg in rename:
-                        assert rename[arg] is oarg
+                        assert rename[arg].same_box(oarg)
                     else:
                         rename[arg] = oarg
 
                 if op.result:
                     rename[op.result] = candidate.result
-                return 1
+                m += 1
+                return m
             return 0
         j = 0
         rename = {}
         for i, op in enumerate(loop.operations):
             candidate = operations[j]
             j += check(op, candidate, rename)
-        if isinstance(operations[0], Glob):
-            assert j == len(operations)-2
+        if isinstance(operations[-1], Glob):
+            assert j == len(operations)-1, self.debug_print_operations(loop)
         else:
-            assert j == len(operations)-1
+            assert j == len(operations), self.debug_print_operations(loop)
 
     def test_basic(self):
         loop1 = self.parse("""
@@ -141,17 +146,18 @@
         loop1 = self.parse("""
         i10 = int_gt(i1, 42)
         guard_true(i10) []
-        i11 = int_sub(i1, 1)
-        i12 = int_gt(i11, 42)
+        i11 = int_add(i1, 1)
+        i12 = int_gt(i11, i2)
         guard_true(i12) []
         """)
-        opt = self.optguards(loop1)
-        self.assert_guard_count(loop1, 1)
+        opt = self.optguards(loop1, True)
+        self.assert_guard_count(loop1, 2)
         self.assert_contains_sequence(loop1, """
+        i40 = int_ge(42, i2)
+        guard_true(i40) []
         ...
-        i11 = int_sub(i1, 1)
-        i12 = int_gt(i11, 42)
-        guard_true(i12) []
+        i10 = int_gt(i1, 42)
+        guard_true(i10) []
         ...
         """)
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -32,11 +32,11 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 
 def optimize_vector(metainterp_sd, jitdriver_sd, loop, optimizations,
-                    inline_short_preamble, start_state, cost_threshold):
+                    inline_short_preamble, start_state, warmstate):
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, False)
-    orig_ops = loop.operations
-    if len(orig_ops) >= 75:
+    version = loop.snapshot()
+    if len(loop.operations) >= 75:
         # if more than 75 operations are present in this loop,
         # it won't be possible to vectorize. There are too many
         # guards that prevent parallel execution of instructions
@@ -52,9 +52,11 @@
         opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, cost_threshold)
         opt.propagate_all_forward()
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
-        gso.propagate_all_forward(opt.loop)
+        user_code = not jitdriver_sd.vectorize and warmstate.vectorize_user
+        gso.propagate_all_forward(opt.loop, user_code)
         # loop versioning
-        loop.versions = [LoopVersion(orig_ops, loop.operations, opt.appended_arg_count)]
+        version.register_all_guards(loop.operations, opt.appended_arg_count)
+        loop.versions.append(version)
         #
         #
         end = time.clock()

From noreply at buildbot.pypy.org  Thu Jul 16 16:06:37 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 16 Jul 2015 16:06:37 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rpython hints,
 saving the loop version the compileloopversion descr
Message-ID: <20150716140637.D6DC61C13F5@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78559:841f77615665
Date: 2015-07-16 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/841f77615665/

Log:	rpython hints, saving the loop version the compileloopversion descr

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -612,7 +612,7 @@
             addr = rawstart + tok.pos_jump_offset
             tok.faildescr.adr_jump_offset = addr
             descr = tok.faildescr
-            if isinstance(descr, CompileLoopVersionDescr):
+            if descr.loop_version():
                 continue # patch them later
             relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4)
             assert rx86.fits_in_32bits(relative_target)
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -725,6 +725,7 @@
     operations = None
     inputargs = None
     faillocs = None
+    version = None
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         assert 0, "this guard must never fail"
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -553,7 +553,7 @@
     def save_to_descr(self, descr, position):
         from rpython.jit.metainterp.compile import ResumeGuardDescr
         from rpython.jit.metainterp.resume import AccumInfo
-        assert isinstance(descr,ResumeGuardDescr)
+        assert isinstance(descr, ResumeGuardDescr)
         ai = AccumInfo(descr.rd_accum_list, position, self.operator, self.var)
         descr.rd_accum_list = ai
 
@@ -758,6 +758,7 @@
         self.inputargs = label.getarglist()
 
     def register_all_guards(self, opt_ops, invariant_arg_count=0):
+        from rpython.jit.metainterp.compile import CompileLoopVersionDescr
         idx = index_of_first(rop.LABEL, opt_ops)
         assert idx >= 0
         version_failargs = opt_ops[idx].getarglist()
@@ -771,21 +772,26 @@
 
         for op in opt_ops:
             if op.is_guard():
-                import pdb; pdb.set_trace()
                 assert isinstance(op, GuardResOp)
                 descr = op.getdescr()
                 if descr.loop_version():
-                    # currently there is only ONE versioning,
-                    # that is the original loop after unrolling.
-                    # if there are more possibilites, let the descr
-                    # know which loop version he preferes
-                    self.faildescrs.append(descr)
-                    op.setfailargs(version_failargs)
-                    op.rd_snapshot = None
+                    assert isinstance(descr, CompileLoopVersionDescr)
+                    if descr.version is None:
+                        # currently there is only ONE versioning,
+                        # that is the original loop after unrolling.
+                        # if there are more possibilites, let the descr
+                        # know which loop version he preferes
+                        descr.version = self
+                        self.faildescrs.append(descr)
+                        op.setfailargs(version_failargs)
+                        op.rd_snapshot = None
 
     def register_guard(self, op):
+        from rpython.jit.metainterp.compile import CompileLoopVersionDescr
         assert isinstance(op, GuardResOp)
         descr = op.getdescr()
+        assert isinstance(descr, CompileLoopVersionDescr)
+        descr.version = self
         self.faildescrs.append(descr)
         op.setfailargs(self.inputargs)
         op.rd_snapshot = None
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -16,12 +16,12 @@
     """ An object wrapper around a guard. Helps to determine
         if one guard implies another
     """
+    _attrs_ = ('index', 'op', 'cmp_op', 'rhs', 'lhs')
+
     def __init__(self, index, op, cmp_op, index_vars):
         self.index = index
         self.op = op
         self.cmp_op = cmp_op
-        self.lhs_key = None
-        self.rhs_key = None
         lhs = cmp_op.getarg(0)
         self.lhs = index_vars.get(lhs, None)
         if self.lhs is None:
@@ -32,6 +32,15 @@
         if self.rhs is None:
             self.rhs = IndexVar(rhs)
 
+    def setindex(self, index):
+        self.index = index
+
+    def setoperation(self, op):
+        self.op = op
+
+    def setcmp(self, c):
+        self.cmp_op = c
+
     def getleftkey(self):
         return self.lhs.getvariable()
 
@@ -108,11 +117,10 @@
         self.index = other.index
 
         descr = myop.getdescr()
-        if we_are_translated():
-            descr.copy_all_attributes_from(other.op.getdescr())
-            myop.rd_frame_info_list = otherop.rd_frame_info_list
-            myop.rd_snapshot = otherop.rd_snapshot
-            myop.setfailargs(otherop.getfailargs())
+        descr.copy_all_attributes_from(other.op.getdescr())
+        myop.rd_frame_info_list = otherop.rd_frame_info_list
+        myop.rd_snapshot = otherop.rd_snapshot
+        myop.setfailargs(otherop.getfailargs())
 
     def emit_varops(self, opt, var, old_arg):
         assert isinstance(var, IndexVar)
@@ -134,9 +142,9 @@
         guard = self.op.clone()
         guard.setarg(0, box_result)
         opt.emit_operation(guard)
-        guard.index = opt.operation_position()-1
-        guard.op = guard
-        guard.cmp_op = cmp_op
+        self.setindex(opt.operation_position()-1)
+        self.setoperation(guard)
+        self.setcmp(cmp_op)
 
     def set_to_none(self, operations):
         assert operations[self.index] is self.op
@@ -249,7 +257,7 @@
         self.collect_guard_information(loop)
         self.eliminate_guards(loop)
 
-        if user_code:
+        if user_code or True:
             version = loop.snapshot()
             self.eliminate_array_bound_checks(loop, version)
 
@@ -278,23 +286,3 @@
         loop.operations = self._newoperations + \
                 [op for op in loop.operations if op]
 
-    # OLD
-    #def _get_key(self, cmp_op):
-    #    assert cmp_op
-    #    lhs_arg = cmp_op.getarg(0)
-    #    rhs_arg = cmp_op.getarg(1)
-    #    lhs_index_var = self.index_vars.get(lhs_arg, None)
-    #    rhs_index_var = self.index_vars.get(rhs_arg, None)
-
-    #    cmp_opnum = cmp_op.getopnum()
-    #    # get the key, this identifies the guarded operation
-    #    if lhs_index_var and rhs_index_var:
-    #        return (lhs_index_var.getvariable(), cmp_opnum, rhs_index_var.getvariable())
-    #    elif lhs_index_var:
-    #        return (lhs_index_var.getvariable(), cmp_opnum, None)
-    #    elif rhs_index_var:
-    #        return (None, cmp_opnum, rhs_index_var)
-    #    else:
-    #        return (None, cmp_opnum, None)
-    #    return key
-
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1366,8 +1366,6 @@
         guard_not_invalidated(descr=<ResumeGuardNotInvalidated object at 0x7fc657d7be20>) [p1, p0, p5, p6, p7, p12, p13]
         debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #120 LOAD_CONST')
         debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #123 COMPARE_OP')
-        i22 = int_lt(i14, 2024)
-        guard_true(i22, descr=<ResumeGuardTrueDescr object at 0x7fc657d7bdc0>) [p1, p0, p5, p6, p7, p12, p13, i14]
         debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #126 POP_JUMP_IF_FALSE')
         debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #129 LOAD_NAME')
         debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #132 LOAD_NAME')
@@ -1400,6 +1398,8 @@
         i36 = int_lt(i33, 0)
         guard_false(i36, descr=<ResumeGuardFalseDescr object at 0x7fc657d7be80>) [p1, p0, p5, p6, p7, p12, p13, None, None, None]
         debug_merge_point(0, 0, '<code object <module>. file '/home/rich/proj/da/thesis/bench/user1.py'. line 2> #117 LOAD_NAME')
+        i22 = int_lt(i14, 2024)
+        guard_true(i22, descr=<ResumeGuardTrueDescr object at 0x7fc657d7bdc0>) [p1, p0, p5, p6, p7, p12, p13, i14]
         jump(p0, p1, p5, p6, p7, p12, p13, i31, i15, i16, i17, i18, i19, i20)
         """
         # schedule 885 -> ptype is non for raw_load?
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -49,14 +49,13 @@
         start = time.clock()
         #
         #
-        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, cost_threshold)
+        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, 0)
         opt.propagate_all_forward()
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
         user_code = not jitdriver_sd.vectorize and warmstate.vectorize_user
         gso.propagate_all_forward(opt.loop, user_code)
-        # loop versioning
+        # connect all compile loop version fail descriptors to this version
         version.register_all_guards(loop.operations, opt.appended_arg_count)
-        loop.versions.append(version)
         #
         #
         end = time.clock()
@@ -66,20 +65,23 @@
         #
         nano = int((end-start)*10.0**9)
         debug_print("# vecopt factor: %d opcount: (%d -> %d) took %dns" % \
-                      (opt.unroll_count+1, len(orig_ops), len(loop.operations), nano))
+                      (opt.unroll_count+1, len(version.operations), len(loop.operations), nano))
         debug_stop("vec-opt-loop")
         #
     except NotAVectorizeableLoop:
         debug_stop("vec-opt-loop")
         # vectorization is not possible
-        loop.operations = orig_ops
+        loop.operations = version.operations
+        loop.versions = None
     except NotAProfitableLoop:
         debug_stop("vec-opt-loop")
         # cost model says to skip this loop
-        loop.operations = orig_ops
+        loop.operations = version.operations
+        loop.versions = None
     except Exception as e:
         debug_stop("vec-opt-loop")
-        loop.operations = orig_ops
+        loop.operations = version.operations
+        loop.versions = None
         debug_print("failed to vectorize loop. THIS IS A FATAL ERROR!")
         if we_are_translated():
             from rpython.rtyper.lltypesystem import lltype

From noreply at buildbot.pypy.org  Thu Jul 16 19:47:36 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 16 Jul 2015 19:47:36 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: Kill Chunks and simply use a
 new_view() function instead
Message-ID: <20150716174736.BAEF91C13CD@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78560:360ba1502914
Date: 2015-07-16 18:47 +0100
http://bitbucket.org/pypy/pypy/changeset/360ba1502914/

Log:	Kill Chunks and simply use a new_view() function instead

diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py
--- a/pypy/module/micronumpy/arrayops.py
+++ b/pypy/module/micronumpy/arrayops.py
@@ -5,7 +5,7 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
 from pypy.module.micronumpy.converters import clipmode_converter
 from pypy.module.micronumpy.strides import (
-    Chunk, Chunks, shape_agreement, shape_agreement_multiple)
+    Chunk, new_view, shape_agreement, shape_agreement_multiple)
 from .casting import find_binop_result_dtype, find_result_type
 
 
@@ -148,7 +148,8 @@
             continue
         chunks[axis] = Chunk(axis_start, axis_start + arr.get_shape()[axis], 1,
                              arr.get_shape()[axis])
-        Chunks(chunks).apply(space, res).implementation.setslice(space, arr)
+        view = new_view(space, res, chunks)
+        view.implementation.setslice(space, arr)
         axis_start += arr.get_shape()[axis]
     return res
 
@@ -162,9 +163,8 @@
         shape = [arr.get_shape()[0] * repeats]
         w_res = W_NDimArray.from_shape(space, shape, arr.get_dtype(), w_instance=arr)
         for i in range(repeats):
-            chunks = Chunks([Chunk(i, shape[0] - repeats + i, repeats,
-                 orig_size)])
-            view = chunks.apply(space, w_res)
+            chunks = [Chunk(i, shape[0] - repeats + i, repeats, orig_size)]
+            view = new_view(space, w_res, chunks)
             view.implementation.setslice(space, arr)
     else:
         axis = space.int_w(w_axis)
@@ -176,7 +176,7 @@
         for i in range(repeats):
             chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
                                  orig_size)
-            view = Chunks(chunks).apply(space, w_res)
+            view = new_view(space, w_res, chunks)
             view.implementation.setslice(space, arr)
     return w_res
 
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib import jit, rgc
 from rpython.rlib.buffer import Buffer
-from rpython.rlib.debug import make_sure_not_resized, debug_print
+from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
     raw_storage_getitem, raw_storage_setitem, RAW_STORAGE
 from rpython.rtyper.lltypesystem import rffi, lltype, llmemory
@@ -10,13 +10,11 @@
     ArrayArgumentException, W_NumpyObject
 from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (
-    Chunk, Chunks, NewAxisChunk, EllipsisChunk,
+    Chunk, new_view, NewAxisChunk, EllipsisChunk,
     calc_strides, calc_new_strides, shape_agreement,
     calculate_broadcast_strides, calc_backstrides, calc_start, is_c_contiguous,
     is_f_contiguous)
 from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.annlowlevel import cast_gcref_to_instance
-from pypy.interpreter.baseobjspace import W_Root
 
 class BaseConcreteArray(object):
     _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]',
@@ -225,16 +223,16 @@
                 space.isinstance_w(w_idx, space.w_slice)):
             if len(self.get_shape()) == 0:
                 raise oefmt(space.w_ValueError, "cannot slice a 0-d array")
-            return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))])
+            return [Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]
         elif isinstance(w_idx, W_NDimArray) and w_idx.is_scalar():
             w_idx = w_idx.get_scalar_value().item(space)
             if not space.isinstance_w(w_idx, space.w_int) and \
                     not space.isinstance_w(w_idx, space.w_bool):
                 raise OperationError(space.w_IndexError, space.wrap(
                     "arrays used as indices must be of integer (or boolean) type"))
-            return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))])
+            return [Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]
         elif space.is_w(w_idx, space.w_None):
-            return Chunks([NewAxisChunk()])
+            return [NewAxisChunk()]
         result = []
         i = 0
         has_ellipsis = False
@@ -253,7 +251,7 @@
                 result.append(Chunk(*space.decode_index4(w_item,
                                                          self.get_shape()[i])))
                 i += 1
-        return Chunks(result)
+        return result
 
     def descr_getitem(self, space, orig_arr, w_index):
         try:
@@ -262,7 +260,7 @@
         except IndexError:
             # not a single result
             chunks = self._prepare_slice_args(space, w_index)
-            return chunks.apply(space, orig_arr)
+            return new_view(space, orig_arr, chunks)
 
     def descr_setitem(self, space, orig_arr, w_index, w_value):
         try:
@@ -271,7 +269,7 @@
         except IndexError:
             w_value = convert_to_array(space, w_value)
             chunks = self._prepare_slice_args(space, w_index)
-            view = chunks.apply(space, orig_arr)
+            view = new_view(space, orig_arr, chunks)
             view.implementation.setslice(space, w_value)
 
     def transpose(self, orig_array, axes=None):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -20,7 +20,7 @@
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import (
     get_shape_from_iterable, shape_agreement, shape_agreement_multiple,
-    is_c_contiguous, is_f_contiguous, calc_strides)
+    is_c_contiguous, is_f_contiguous, calc_strides, new_view)
 from pypy.module.micronumpy.casting import can_cast_array
 
 
@@ -179,7 +179,7 @@
         if iter_shape is None:
             # w_index is a list of slices, return a view
             chunks = self.implementation._prepare_slice_args(space, w_index)
-            return chunks.apply(space, self)
+            return new_view(space, self, chunks)
         shape = res_shape + self.get_shape()[len(indexes):]
         w_res = W_NDimArray.from_shape(space, shape, self.get_dtype(),
                                        self.get_order(), w_instance=self)
@@ -195,7 +195,7 @@
         if iter_shape is None:
             # w_index is a list of slices
             chunks = self.implementation._prepare_slice_args(space, w_index)
-            view = chunks.apply(space, self)
+            view = new_view(space, self, chunks)
             view.implementation.setslice(space, val_arr)
             return
         if support.product(iter_shape) == 0:
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,31 +10,6 @@
     pass
 
 
-class Chunks(BaseChunk):
-    def __init__(self, l):
-        self.l = l
-
-    @jit.unroll_safe
-    def extend_shape(self, old_shape):
-        shape = []
-        i = -1
-        for i, c in enumerate_chunks(self.l):
-            if c.step != 0:
-                shape.append(c.lgt)
-        s = i + 1
-        assert s >= 0
-        return shape[:] + old_shape[s:]
-
-    def apply(self, space, orig_arr):
-        arr = orig_arr.implementation
-        shape = self.extend_shape(arr.shape)
-        r = calculate_slice_strides(arr.shape, arr.start, arr.get_strides(),
-                                    arr.get_backstrides(), self.l)
-        _, start, strides, backstrides = r
-        return W_NDimArray.new_slice(space, start, strides[:], backstrides[:],
-                                     shape[:], arr, orig_arr)
-
-
 class Chunk(BaseChunk):
     axis_step = 1
 
@@ -64,6 +39,27 @@
         pass
 
 
+def new_view(space, w_arr, chunks):
+    arr = w_arr.implementation
+    shape = _extend_shape(arr.shape, chunks)
+    r = calculate_slice_strides(arr.shape, arr.start, arr.get_strides(),
+                                arr.get_backstrides(), chunks)
+    _, start, strides, backstrides = r
+    return W_NDimArray.new_slice(space, start, strides[:], backstrides[:],
+                                 shape[:], arr, w_arr)
+
+ at jit.unroll_safe
+def _extend_shape(old_shape, chunks):
+    shape = []
+    i = -1
+    for i, c in enumerate_chunks(chunks):
+        if c.step != 0:
+            shape.append(c.lgt)
+    s = i + 1
+    assert s >= 0
+    return shape[:] + old_shape[s:]
+
+
 class BaseTransform(object):
     pass
 
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -4,7 +4,7 @@
 
 from pypy.conftest import option
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
-from pypy.module.micronumpy.strides import Chunk, Chunks
+from pypy.module.micronumpy.strides import Chunk, new_view
 from pypy.module.micronumpy.ndarray import W_NDimArray
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
@@ -22,7 +22,7 @@
 
 
 def create_slice(space, a, chunks):
-    return Chunks(chunks).apply(space, W_NDimArray(a)).implementation
+    return new_view(space, W_NDimArray(a), chunks).implementation
 
 
 def create_array(*args, **kwargs):

From noreply at buildbot.pypy.org  Fri Jul 17 01:58:58 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 01:58:58 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20150716235858.336591C1192@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78561:05c251509e69
Date: 2015-07-17 01:58 +0200
http://bitbucket.org/pypy/pypy/changeset/05c251509e69/

Log:	hg merge default

diff too long, truncating to 2000 out of 4707 lines

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.1.2
+Version: 1.2.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.2"
-__version_info__ = (1, 1, 2)
+__version__ = "1.2.0"
+__version_info__ = (1, 2, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -236,6 +236,30 @@
             cdecl = self._typeof(cdecl)
         return self._backend.newp(cdecl, init)
 
+    def new_allocator(self, alloc=None, free=None,
+                      should_clear_after_alloc=True):
+        """Return a new allocator, i.e. a function that behaves like ffi.new()
+        but uses the provided low-level 'alloc' and 'free' functions.
+
+        'alloc' is called with the size as argument.  If it returns NULL, a
+        MemoryError is raised.  'free' is called with the result of 'alloc'
+        as argument.  Both can be either Python function or directly C
+        functions.  If 'free' is None, then no free function is called.
+        If both 'alloc' and 'free' are None, the default is used.
+
+        If 'should_clear_after_alloc' is set to False, then the memory
+        returned by 'alloc' is assumed to be already cleared (or you are
+        fine with garbage); otherwise CFFI will clear it.
+        """
+        compiled_ffi = self._backend.FFI()
+        allocator = compiled_ffi.new_allocator(alloc, free,
+                                               should_clear_after_alloc)
+        def allocate(cdecl, init=None):
+            if isinstance(cdecl, basestring):
+                cdecl = self._typeof(cdecl)
+            return allocator(cdecl, init)
+        return allocate
+
     def cast(self, cdecl, source):
         """Similar to a C cast: returns an instance of the named C
         type initialized with the given 'source'.  The source is
@@ -286,7 +310,7 @@
         """
         return self._backend.from_buffer(self.BCharA, python_buffer)
 
-    def callback(self, cdecl, python_callable=None, error=None):
+    def callback(self, cdecl, python_callable=None, error=None, onerror=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
         The callback invokes the specified 'python_callable' (which may
@@ -298,7 +322,8 @@
             if not callable(python_callable):
                 raise TypeError("the 'python_callable' argument "
                                 "is not callable")
-            return self._backend.callback(cdecl, python_callable, error)
+            return self._backend.callback(cdecl, python_callable,
+                                          error, onerror)
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl, consider_function_as_funcptr=True)
         if python_callable is None:
@@ -327,6 +352,13 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
+        try:
+            gcp = self._backend.gcp
+        except AttributeError:
+            pass
+        else:
+            return gcp(cdata, destructor)
+        #
         with self._lock:
             try:
                 gc_weakrefs = self.gc_weakrefs
@@ -428,6 +460,8 @@
             raise TypeError("ffi.include() expects an argument that is also of"
                             " type cffi.FFI, not %r" % (
                                 type(ffi_to_include).__name__,))
+        if ffi_to_include is self:
+            raise ValueError("self.include(self)")
         with ffi_to_include._lock:
             with self._lock:
                 self._parser.include(ffi_to_include._parser)
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -989,7 +989,8 @@
     def cast(self, BType, source):
         return BType._cast_from(source)
 
-    def callback(self, BType, source, error):
+    def callback(self, BType, source, error, onerror):
+        assert onerror is None   # XXX not implemented
         return BType(source, error)
 
     typeof = type
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -633,6 +633,8 @@
 
     def include(self, other):
         for name, tp in other._declarations.items():
+            if name.startswith('anonymous $enum_$'):
+                continue   # fix for test_anonymous_enum_include
             kind = name.split(' ', 1)[0]
             if kind in ('struct', 'union', 'enum', 'anonymous'):
                 self._declare(name, tp, included=True)
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -35,9 +35,6 @@
     def is_integer_type(self):
         return False
 
-    def sizeof_enabled(self):
-        return False
-
     def get_cached_btype(self, ffi, finishlist, can_delay=False):
         try:
             BType = ffi._cached_btypes[self]
@@ -80,8 +77,7 @@
 
 
 class BasePrimitiveType(BaseType):
-    def sizeof_enabled(self):
-        return True
+    pass
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -205,9 +201,6 @@
 class FunctionPtrType(BaseFunctionType):
     _base_pattern = '(*&)(%s)'
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         result = self.result.get_cached_btype(ffi, finishlist)
         args = []
@@ -233,9 +226,6 @@
             extra = self._base_pattern
         self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra)
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True)
         return global_cache(self, ffi, 'new_pointer_type', BItem)
@@ -276,9 +266,6 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def sizeof_enabled(self):
-        return self.item.sizeof_enabled() and self.length is not None
-
     def resolve_length(self, newlength):
         return ArrayType(self.item, newlength)
 
@@ -433,9 +420,6 @@
             from . import ffiplatform
             raise ffiplatform.VerificationMissing(self._get_c_name())
 
-    def sizeof_enabled(self):
-        return self.fldtypes is not None
-
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
@@ -464,9 +448,6 @@
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
-    def sizeof_enabled(self):
-        return True     # not strictly true, but external enums are obscure
-
     def force_the_name(self, forcename):
         StructOrUnionOrEnum.force_the_name(self, forcename)
         if self.forcename is None:
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -26,6 +26,7 @@
 #define _CFFI_OP_GLOBAL_VAR     33
 #define _CFFI_OP_DLOPEN_FUNC    35
 #define _CFFI_OP_DLOPEN_CONST   37
+#define _CFFI_OP_GLOBAL_VAR_F   39
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -981,10 +981,6 @@
         if not self.target_is_python and tp.is_integer_type():
             type_op = CffiOp(OP_CONSTANT_INT, -1)
         else:
-            if not tp.sizeof_enabled():
-                raise ffiplatform.VerificationError(
-                    "constant '%s' is of type '%s', whose size is not known"
-                    % (name, tp._get_c_name()))
             if self.target_is_python:
                 const_kind = OP_DLOPEN_CONST
             else:
@@ -1069,18 +1065,36 @@
         self._do_collect_type(self._global_type(tp, name))
 
     def _generate_cpy_variable_decl(self, tp, name):
-        pass
+        prnt = self._prnt
+        tp = self._global_type(tp, name)
+        if isinstance(tp, model.ArrayType) and tp.length is None:
+            tp = tp.item
+            ampersand = ''
+        else:
+            ampersand = '&'
+        # This code assumes that casts from "tp *" to "void *" is a
+        # no-op, i.e. a function that returns a "tp *" can be called
+        # as if it returned a "void *".  This should be generally true
+        # on any modern machine.  The only exception to that rule (on
+        # uncommon architectures, and as far as I can tell) might be
+        # if 'tp' were a function type, but that is not possible here.
+        # (If 'tp' is a function _pointer_ type, then casts from "fn_t
+        # **" to "void *" are again no-ops, as far as I can tell.)
+        prnt('static ' + tp.get_c_name('*_cffi_var_%s(void)' % (name,)))
+        prnt('{')
+        prnt('  return %s(%s);' % (ampersand, name))
+        prnt('}')
+        prnt()
 
     def _generate_cpy_variable_ctx(self, tp, name):
         tp = self._global_type(tp, name)
         type_index = self._typesdict[tp]
-        type_op = CffiOp(OP_GLOBAL_VAR, type_index)
-        if tp.sizeof_enabled():
-            size = "sizeof(%s)" % (name,)
+        if self.target_is_python:
+            op = OP_GLOBAL_VAR
         else:
-            size = 0
+            op = OP_GLOBAL_VAR_F
         self._lsts["global"].append(
-            GlobalExpr(name, '&%s' % name, type_op, size))
+            GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index)))
 
     # ----------
     # emitting the opcodes for individual types
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -24,14 +24,14 @@
     "_codecs", "atexit", "gc", "_weakref", "marshal", "errno", "imp",
     "itertools", "math", "cmath", "_sre", "_pickle_support", "operator",
     "parser", "symbol", "token", "_ast", "_random", "__pypy__",
-    "_string", "_testing"
+    "_string", "_testing", "time"
 ])
 
 
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_minimal_curses",
     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -70,6 +70,20 @@
 .. _`use virtualenv (as documented here)`: getting-started.html#installing-using-virtualenv
 
 
+Module xyz does not work in the sandboxed PyPy?
+-----------------------------------------------
+
+You cannot import *any* extension module in a `sandboxed PyPy`_,
+sorry.  Even the built-in modules available are very limited.
+Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
+it is only a proof of concept.  It seriously requires someone working
+on it.  Before this occurs, it can only be used it for "pure Python"
+examples: programs that import mostly nothing (or only pure Python
+modules, recursively).
+
+.. _`sandboxed PyPy`: sandbox.html
+
+
 .. _`See below.`:
 
 Do CPython Extension modules work with PyPy?
diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -103,12 +103,15 @@
 Howto
 -----
 
-In pypy/goal::
+Grab a copy of the pypy repository_.  In the directory pypy/goal, run::
 
    ../../rpython/bin/rpython -O2 --sandbox targetpypystandalone.py
 
 If you don't have a regular PyPy installed, you should, because it's
-faster to translate, but you can also run ``python translate.py`` instead.
+faster to translate; but you can also run the same line with ``python``
+in front.
+
+.. _repository: https://bitbucket.org/pypy/pypy
 
 
 To run it, use the tools in the pypy/sandbox directory::
@@ -136,8 +139,6 @@
 Not all operations are supported; e.g. if you type os.readlink('...'),
 the controller crashes with an exception and the subprocess is killed.
 Other operations make the subprocess die directly with a "Fatal RPython
-error".  None of this is a security hole; it just means that if you try
-to run some random program, it risks getting killed depending on the
-Python built-in functions it tries to call.  This is a matter of the
-sandboxing layer being incomplete so far, but it should not really be
-a problem in practice.
+error".  None of this is a security hole.  More importantly, *most other
+built-in modules are not enabled.  Please read all the warnings in this
+page before complaining about this.  Contributions welcome.*
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -20,5 +20,20 @@
 .. branch: run-create_cffi_imports
 
 Build cffi import libraries as part of translation by monkey-patching an 
-aditional task into translation
+additional task into translation
 
+.. branch: int-float-list-strategy
+
+Use a compact strategy for Python lists that mix integers and floats,
+at least if the integers fit inside 32 bits.  These lists are now
+stored as an array of floats, like lists that contain only floats; the
+difference is that integers are stored as tagged NaNs.  (This should
+have no visible effect!  After ``lst = [42, 42.5]``, the value of
+``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
+
+.. branch: cffi-callback-onerror
+.. branch: cffi-new-allocator
+
+.. branch: unicode-dtype
+
+Partial implementation of unicode dtype and unicode scalars.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -303,7 +303,12 @@
         options = make_dict(config)
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
+        if config.objspace.usemodules._cffi_backend:
+            self.hack_for_cffi_modules(driver)
 
+        return self.get_entry_point(config)
+    
+    def hack_for_cffi_modules(self, driver):
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_c to build_cffi_imports
         # this should probably get cleaned up and merged with driver.create_exe
@@ -342,8 +347,6 @@
         driver.default_goal = 'build_cffi_imports'
         # HACKHACKHACK end
 
-        return self.get_entry_point(config)
-
     def jitpolicy(self, driver):
         from pypy.module.pypyjit.policy import PyPyJitPolicy
         from pypy.module.pypyjit.hooks import pypy_hooks
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -274,7 +274,8 @@
                                w_t, w_v, w_tb],
                 """(where, objrepr, extra_line, t, v, tb):
                     import sys, traceback
-                    sys.stderr.write('From %s%s:\\n' % (where, objrepr))
+                    if where or objrepr:
+                        sys.stderr.write('From %s%s:\\n' % (where, objrepr))
                     if extra_line:
                         sys.stderr.write(extra_line)
                     traceback.print_exception(t, v, tb)
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -15,7 +15,10 @@
         self.running = False
 
     def descr__repr__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = '<finished>'
+        else:
+            code_name = self.pycode.co_name
         addrstring = self.getaddrstring(space)
         return space.wrap("<generator object %s at 0x%s>" %
                           (code_name, addrstring))
@@ -45,6 +48,8 @@
         w_framestate, w_running = args_w
         if space.is_w(w_framestate, space.w_None):
             self.frame = None
+            self.space = space
+            self.pycode = None
         else:
             frame = instantiate(space.FrameClass)   # XXX fish
             frame.descr__setstate__(space, w_framestate)
@@ -62,9 +67,10 @@
 
     def send_ex(self, w_arg, operr=None):
         pycode = self.pycode
-        if jit.we_are_jitted() and should_not_inline(pycode):
-            generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
-                                                  operr=operr, pycode=pycode)
+        if pycode is not None:
+            if jit.we_are_jitted() and should_not_inline(pycode):
+                generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
+                                                    operr=operr, pycode=pycode)
         return self._send_ex(w_arg, operr)
 
     def _send_ex(self, w_arg, operr):
@@ -163,7 +169,10 @@
         return self.pycode
 
     def descr__name__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = '<finished>'
+        else:
+            code_name = self.pycode.co_name
         return space.wrap(code_name)
 
     # Results can be either an RPython list of W_Root, or it can be an
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -530,6 +530,22 @@
             del sys.modules['mod']
 
 
+    def test_pickle_generator_crash(self):
+        import pickle
+
+        def f():
+            yield 0
+
+        x = f()
+        x.next()
+        try:
+            x.next()
+        except StopIteration:
+            y = pickle.loads(pickle.dumps(x))
+        assert 'finished' in y.__name__
+        assert 'finished' in repr(y)
+        assert y.gi_code is None
+
 class AppTestGeneratorCloning:
 
     def setup_class(cls):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.2"
+VERSION = "1.2.0"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -0,0 +1,86 @@
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+class W_Allocator(W_Root):
+    _immutable_ = True
+
+    def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc):
+        self.ffi = ffi    # may be None
+        self.w_alloc = w_alloc
+        self.w_free = w_free
+        self.should_clear_after_alloc = should_clear_after_alloc
+
+    def allocate(self, space, datasize, ctype, length=-1):
+        from pypy.module._cffi_backend import cdataobj, ctypeptr
+        if self.w_alloc is None:
+            if self.should_clear_after_alloc:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=True)
+            else:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=False)
+            return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+        else:
+            w_raw_cdata = space.call_function(self.w_alloc,
+                                              space.wrap(datasize))
+            if not isinstance(w_raw_cdata, cdataobj.W_CData):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata object (got %T)",
+                            w_raw_cdata)
+            if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata pointer, not '%s'",
+                            w_raw_cdata.ctype.name)
+            #
+            ptr = w_raw_cdata.unsafe_escaping_ptr()
+            if not ptr:
+                raise oefmt(space.w_MemoryError, "alloc() returned NULL")
+            #
+            if self.should_clear_after_alloc:
+                rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0,
+                              rffi.cast(rffi.SIZE_T, datasize))
+            #
+            if self.w_free is None:
+                # use this class which does not have a __del__, but still
+                # keeps alive w_raw_cdata
+                res = cdataobj.W_CDataNewNonStdNoFree(space, ptr, ctype, length)
+            else:
+                res = cdataobj.W_CDataNewNonStdFree(space, ptr, ctype, length)
+                res.w_free = self.w_free
+            res.w_raw_cdata = w_raw_cdata
+            return res
+
+    @unwrap_spec(w_init=WrappedDefault(None))
+    def descr_call(self, space, w_arg, w_init):
+        ffi = self.ffi
+        assert ffi is not None
+        w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING | ffi.ACCEPT_CTYPE)
+        return w_ctype.newp(w_init, self)
+
+
+W_Allocator.typedef = TypeDef(
+        'FFIAllocator',
+        __call__ = interp2app(W_Allocator.descr_call),
+        )
+W_Allocator.typedef.acceptable_as_base_class = False
+
+
+def new_allocator(ffi, w_alloc, w_free, should_clear_after_alloc):
+    space = ffi.space
+    if space.is_none(w_alloc):
+        w_alloc = None
+    if space.is_none(w_free):
+        w_free = None
+    if w_alloc is None and w_free is not None:
+        raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'")
+    alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc))
+    return space.wrap(alloc)
+
+
+default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True)
+nonzero_allocator = W_Allocator(None, None, None,should_clear_after_alloc=False)
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -22,8 +22,9 @@
 class W_CDataCallback(W_CData):
     #_immutable_fields_ = ...
     ll_error = lltype.nullptr(rffi.CCHARP.TO)
+    w_onerror = None
 
-    def __init__(self, space, ctype, w_callable, w_error):
+    def __init__(self, space, ctype, w_callable, w_error, w_onerror):
         raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc())
         W_CData.__init__(self, space, raw_closure, ctype)
         #
@@ -31,6 +32,12 @@
             raise oefmt(space.w_TypeError,
                         "expected a callable object, not %T", w_callable)
         self.w_callable = w_callable
+        if not space.is_none(w_onerror):
+            if not space.is_true(space.callable(w_onerror)):
+                raise oefmt(space.w_TypeError,
+                            "expected a callable object for 'onerror', not %T",
+                            w_onerror)
+            self.w_onerror = w_onerror
         #
         fresult = self.getfunctype().ctitem
         size = fresult.size
@@ -161,6 +168,29 @@
 STDERR = 2
 
 
+ at jit.dont_look_inside
+def _handle_applevel_exception(space, callback, e, ll_res, extra_line):
+    callback.write_error_return_value(ll_res)
+    if callback.w_onerror is None:
+        callback.print_error(e, extra_line)
+    else:
+        try:
+            e.normalize_exception(space)
+            w_t = e.w_type
+            w_v = e.get_w_value(space)
+            w_tb = space.wrap(e.get_traceback())
+            w_res = space.call_function(callback.w_onerror,
+                                        w_t, w_v, w_tb)
+            if not space.is_none(w_res):
+                callback.convert_result(ll_res, w_res)
+        except OperationError, e2:
+            # double exception! print a double-traceback...
+            callback.print_error(e, extra_line)    # original traceback
+            e2.write_unraisable(space, '', with_traceback=True,
+                                extra_line="\nDuring the call to 'onerror', "
+                                           "another exception occurred:\n\n")
+
+
 @jit.jit_callback("CFFI")
 def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
     """ Callback specification.
@@ -178,7 +208,7 @@
         try:
             os.write(STDERR, "SystemError: invoking a callback "
                              "that was already freed\n")
-        except OSError:
+        except:
             pass
         # In this case, we don't even know how big ll_res is.  Let's assume
         # it is just a 'ffi_arg', and store 0 there.
@@ -195,9 +225,7 @@
             extra_line = "Trying to convert the result back to C:\n"
             callback.convert_result(ll_res, w_res)
         except OperationError, e:
-            # got an app-level exception
-            callback.print_error(e, extra_line)
-            callback.write_error_return_value(ll_res)
+            _handle_applevel_exception(space, callback, e, ll_res, extra_line)
         #
     except Exception, e:
         # oups! last-level attempt to recover.
@@ -205,7 +233,7 @@
             os.write(STDERR, "SystemError: callback raised ")
             os.write(STDERR, str(e))
             os.write(STDERR, "\n")
-        except OSError:
+        except:
             pass
         callback.write_error_return_value(ll_res)
     if must_leave:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -363,16 +363,19 @@
     def _sizeof(self):
         return self.ctype.size
 
+    def with_gc(self, w_destructor):
+        with self as ptr:
+            return W_CDataGCP(self.space, ptr, self.ctype, self, w_destructor)
+
 
 class W_CDataMem(W_CData):
-    """This is the base class used for cdata objects that own and free
-    their memory.  Used directly by the results of cffi.cast('int', x)
-    or other primitive explicitly-casted types.  It is further subclassed
-    by W_CDataNewOwning."""
+    """This is used only by the results of cffi.cast('int', x)
+    or other primitive explicitly-casted types."""
     _attrs_ = []
 
-    def __init__(self, space, size, ctype):
-        cdata = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw', zero=True)
+    def __init__(self, space, ctype):
+        cdata = lltype.malloc(rffi.CCHARP.TO, ctype.size, flavor='raw',
+                              zero=False)
         W_CData.__init__(self, space, cdata, ctype)
 
     @rgc.must_be_light_finalizer
@@ -380,36 +383,65 @@
         lltype.free(self._ptr, flavor='raw')
 
 
-class W_CDataNewOwning(W_CDataMem):
-    """This is the class used for the cata objects created by newp()."""
-    _attrs_ = []
+class W_CDataNewOwning(W_CData):
+    """This is the abstract base class used for cdata objects created
+    by newp().  They create and free their own memory according to an
+    allocator."""
+
+    # the 'length' is either >= 0 for arrays, or -1 for pointers.
+    _attrs_ = ['length']
+    _immutable_fields_ = ['length']
+
+    def __init__(self, space, cdata, ctype, length=-1):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.length = length
 
     def _repr_extra(self):
         return self._repr_extra_owning()
 
-
-class W_CDataNewOwningLength(W_CDataNewOwning):
-    """Subclass with an explicit length, for allocated instances of
-    the C type 'foo[]'."""
-    _attrs_ = ['length']
-    _immutable_fields_ = ['length']
-
-    def __init__(self, space, size, ctype, length):
-        W_CDataNewOwning.__init__(self, space, size, ctype)
-        self.length = length
-
     def _sizeof(self):
-        from pypy.module._cffi_backend import ctypearray
         ctype = self.ctype
-        assert isinstance(ctype, ctypearray.W_CTypeArray)
-        return self.length * ctype.ctitem.size
+        if self.length >= 0:
+            from pypy.module._cffi_backend import ctypearray
+            assert isinstance(ctype, ctypearray.W_CTypeArray)
+            return self.length * ctype.ctitem.size
+        else:
+            return ctype.size
 
     def get_array_length(self):
         return self.length
 
 
+class W_CDataNewStd(W_CDataNewOwning):
+    """Subclass using the standard allocator, lltype.malloc()/lltype.free()"""
+    _attrs_ = []
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        lltype.free(self._ptr, flavor='raw')
+
+
+class W_CDataNewNonStdNoFree(W_CDataNewOwning):
+    """Subclass using a non-standard allocator, no free()"""
+    _attrs_ = ['w_raw_cdata']
+
+class W_CDataNewNonStdFree(W_CDataNewNonStdNoFree):
+    """Subclass using a non-standard allocator, with a free()"""
+    _attrs_ = ['w_free']
+
+    def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space,
+                                     W_CDataNewNonStdFree.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataNewNonStdFree)
+        self.space.call_function(self.w_free, self.w_raw_cdata)
+
+
 class W_CDataPtrToStructOrUnion(W_CData):
-    """This subclass is used for the pointer returned by new('struct foo').
+    """This subclass is used for the pointer returned by new('struct foo *').
     It has a strong reference to a W_CDataNewOwning that really owns the
     struct, which is the object returned by the app-level expression 'p[0]'.
     But it is not itself owning any memory, although its repr says so;
@@ -483,6 +515,26 @@
             self.length, self.space.type(self.w_keepalive).name)
 
 
+class W_CDataGCP(W_CData):
+    """For ffi.gc()."""
+    _attrs_ = ['w_original_cdata', 'w_destructor']
+    _immutable_fields_ = ['w_original_cdata', 'w_destructor']
+
+    def __init__(self, space, cdata, ctype, w_original_cdata, w_destructor):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.w_original_cdata = w_original_cdata
+        self.w_destructor = w_destructor
+
+    def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space, W_CDataGCP.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataGCP)
+        self.space.call_function(self.w_destructor, self.w_original_cdata)
+
+
 W_CData.typedef = TypeDef(
     '_cffi_backend.CData',
     __module__ = '_cffi_backend',
diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -36,7 +36,10 @@
                         self.libname)
         try:
             cdata = dlsym(self.libhandle, name)
+            found = bool(cdata)
         except KeyError:
+            found = False
+        if not found:
             raise oefmt(self.ffi.w_FFIError,
                         "symbol '%s' not found in library '%s'",
                         name, self.libname)
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/pypy/module/_cffi_backend/cgc.py b/pypy/module/_cffi_backend/cgc.py
deleted file mode 100644
--- a/pypy/module/_cffi_backend/cgc.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from rpython.rlib import jit
-
-
- at jit.dont_look_inside
-def gc_weakrefs_build(ffi, w_cdata, w_destructor):
-    from pypy.module._weakref import interp__weakref
-
-    space = ffi.space
-    if ffi.w_gc_wref_remove is None:
-        ffi.gc_wref_dict = {}
-        ffi.w_gc_wref_remove = space.getattr(space.wrap(ffi),
-                                             space.wrap("__gc_wref_remove"))
-
-    w_new_cdata = w_cdata.ctype.cast(w_cdata)
-    assert w_new_cdata is not w_cdata
-
-    w_ref = interp__weakref.make_weakref_with_callback(
-        space,
-        space.gettypefor(interp__weakref.W_Weakref),
-        w_new_cdata,
-        ffi.w_gc_wref_remove)
-
-    ffi.gc_wref_dict[w_ref] = (w_destructor, w_cdata)
-    return w_new_cdata
-
-
-def gc_wref_remove(ffi, w_ref):
-    (w_destructor, w_cdata) = ffi.gc_wref_dict.pop(w_ref)
-    ffi.space.call_function(w_destructor, w_cdata)
diff --git a/pypy/module/_cffi_backend/cglob.py b/pypy/module/_cffi_backend/cglob.py
--- a/pypy/module/_cffi_backend/cglob.py
+++ b/pypy/module/_cffi_backend/cglob.py
@@ -1,24 +1,66 @@
+from pypy.interpreter.error import oefmt
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend import newtype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 
 class W_GlobSupport(W_Root):
-    def __init__(self, space, w_ctype, ptr):
+    _immutable_fields_ = ['w_ctype', 'ptr', 'fetch_addr']
+
+    def __init__(self, space, name, w_ctype, ptr=lltype.nullptr(rffi.CCHARP.TO),
+                 fetch_addr=lltype.nullptr(rffi.VOIDP.TO)):
         self.space = space
+        self.name = name
         self.w_ctype = w_ctype
         self.ptr = ptr
+        self.fetch_addr = fetch_addr
+
+    def fetch_global_var_addr(self):
+        if self.ptr:
+            result = self.ptr
+        else:
+            if not we_are_translated():
+                FNPTR = rffi.CCallback([], rffi.VOIDP)
+                fetch_addr = rffi.cast(FNPTR, self.fetch_addr)
+                result = fetch_addr()
+            else:
+                # careful in translated versions: we need to call fetch_addr,
+                # but in a GIL-releasing way.  The easiest is to invoke a
+                # llexternal() helper.
+                result = pypy__cffi_fetch_var(self.fetch_addr)
+            result = rffi.cast(rffi.CCHARP, result)
+        if not result:
+            from pypy.module._cffi_backend import ffi_obj
+            ffierror = ffi_obj.get_ffi_error(self.space)
+            raise oefmt(ffierror, "global variable '%s' is at address NULL",
+                        self.name)
+        return result
 
     def read_global_var(self):
-        return self.w_ctype.convert_to_object(self.ptr)
+        return self.w_ctype.convert_to_object(self.fetch_global_var_addr())
 
     def write_global_var(self, w_newvalue):
-        self.w_ctype.convert_from_object(self.ptr, w_newvalue)
+        self.w_ctype.convert_from_object(self.fetch_global_var_addr(),
+                                         w_newvalue)
 
     def address(self):
         w_ctypeptr = newtype.new_pointer_type(self.space, self.w_ctype)
-        return W_CData(self.space, self.ptr, w_ctypeptr)
+        return W_CData(self.space, self.fetch_global_var_addr(), w_ctypeptr)
 
 W_GlobSupport.typedef = TypeDef("FFIGlobSupport")
 W_GlobSupport.typedef.acceptable_as_base_class = False
+
+
+eci = ExternalCompilationInfo(post_include_bits=["""
+static void *pypy__cffi_fetch_var(void *fetch_addr) {
+    return ((void*(*)(void))fetch_addr)();
+}
+"""])
+
+pypy__cffi_fetch_var = rffi.llexternal(
+    "pypy__cffi_fetch_var", [rffi.VOIDP], rffi.VOIDP,
+    compilation_info=eci)
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -28,7 +28,7 @@
     def _alignof(self):
         return self.ctitem.alignof()
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         datasize = self.size
         #
@@ -40,12 +40,10 @@
             except OverflowError:
                 raise OperationError(space.w_OverflowError,
                     space.wrap("array size would overflow a ssize_t"))
-            #
-            cdata = cdataobj.W_CDataNewOwningLength(space, datasize,
-                                                    self, length)
+        else:
+            length = self.length
         #
-        else:
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+        cdata = allocator.allocate(space, datasize, self, length)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -55,7 +55,7 @@
     def pack_list_of_items(self, cdata, w_ob):
         return False
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         raise oefmt(space.w_TypeError,
                     "expected a pointer or array ctype, got '%s'", self.name)
@@ -90,6 +90,16 @@
     def _convert_error(self, expected, w_got):
         space = self.space
         if isinstance(w_got, cdataobj.W_CData):
+            if self.name == w_got.ctype.name:
+                # in case we'd give the error message "initializer for
+                # ctype 'A' must be a pointer to same type, not cdata
+                # 'B'", but with A=B, then give instead a different error
+                # message to try to clear up the confusion
+                return oefmt(space.w_TypeError,
+                             "initializer for ctype '%s' appears indeed to "
+                             "be '%s', but the types are different (check "
+                             "that you are not e.g. mixing up different ffi "
+                             "instances)", self.name, w_got.ctype.name)
             return oefmt(space.w_TypeError,
                          "initializer for ctype '%s' must be a %s, not cdata "
                          "'%s'", self.name, expected, w_got.ctype.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -63,7 +63,7 @@
             value = self._cast_result(value)
         else:
             value = self._cast_generic(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         self.write_raw_integer_data(w_cdata, value)
         return w_cdata
 
@@ -353,7 +353,7 @@
             value = self.cast_unicode(w_ob)
         else:
             value = space.float_w(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         if not isinstance(self, W_CTypePrimitiveLongDouble):
             w_cdata.write_raw_float_data(value)
         else:
@@ -446,7 +446,7 @@
         return self.space.wrap(value)
 
     def convert_to_object(self, cdata):
-        w_cdata = cdataobj.W_CDataMem(self.space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(self.space, self)
         with w_cdata as ptr:
             self._copy_longdouble(cdata, ptr)
         return w_cdata
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -187,7 +187,7 @@
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
         space = self.space
         ctitem = self.ctitem
@@ -207,14 +207,14 @@
                     datasize = ctitem.convert_struct_from_object(
                         lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
             #
-            cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
+            cdatastruct = allocator.allocate(space, datasize, ctitem)
             ptr = cdatastruct.unsafe_escaping_ptr()
             cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
                                                        self, cdatastruct)
         else:
             if self.is_char_or_unichar_ptr_or_array():
                 datasize *= 2       # forcefully add a null character
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+            cdata = allocator.allocate(space, datasize, self)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -81,10 +81,9 @@
     def copy_and_convert_to_object(self, source):
         space = self.space
         self.check_complete()
-        ob = cdataobj.W_CDataNewOwning(space, self.size, self)
-        with ob as target:
-            misc._raw_memcopy(source, target, self.size)
-        return ob
+        ptr = lltype.malloc(rffi.CCHARP.TO, self.size, flavor='raw', zero=False)
+        misc._raw_memcopy(source, ptr, self.size)
+        return cdataobj.W_CDataNewStd(space, ptr, self)
 
     def typeoffsetof_field(self, fieldname, following):
         self.force_lazy_struct()
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -10,8 +10,8 @@
 from pypy.module._cffi_backend import parse_c_type, realize_c_type
 from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray
 from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle
-from pypy.module._cffi_backend import cbuffer, func, cgc, wrapper
-from pypy.module._cffi_backend import cffi_opcode
+from pypy.module._cffi_backend import cbuffer, func, wrapper
+from pypy.module._cffi_backend import cffi_opcode, allocator
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend.cdataobj import W_CData
 
@@ -44,6 +44,10 @@
 
 
 class W_FFIObject(W_Root):
+    ACCEPT_STRING = ACCEPT_STRING
+    ACCEPT_CTYPE  = ACCEPT_CTYPE
+    ACCEPT_CDATA  = ACCEPT_CDATA
+
     w_gc_wref_remove = None
 
     @jit.dont_look_inside
@@ -276,8 +280,9 @@
 
 
     @unwrap_spec(w_python_callable=WrappedDefault(None),
-                 w_error=WrappedDefault(None))
-    def descr_callback(self, w_cdecl, w_python_callable, w_error):
+                 w_error=WrappedDefault(None),
+                 w_onerror=WrappedDefault(None))
+    def descr_callback(self, w_cdecl, w_python_callable, w_error, w_onerror):
         """\
 Return a callback object or a decorator making such a callback object.
 'cdecl' must name a C function pointer type.  The callback invokes the
@@ -290,14 +295,16 @@
         space = self.space
         if not space.is_none(w_python_callable):
             return ccallback.W_CDataCallback(space, w_ctype,
-                                             w_python_callable, w_error)
+                                             w_python_callable, w_error,
+                                             w_onerror)
         else:
             # decorator mode: returns a single-argument function
-            return space.appexec([w_ctype, w_error],
-            """(ctype, error):
+            return space.appexec([w_ctype, w_error, w_onerror],
+            """(ctype, error, onerror):
                 import _cffi_backend
                 return lambda python_callable: (
-                    _cffi_backend.callback(ctype, python_callable, error))""")
+                    _cffi_backend.callback(ctype, python_callable,
+                                           error, onerror))""")
 
 
     def descr_cast(self, w_arg, w_ob):
@@ -341,10 +348,7 @@
 Later, when this new cdata object is garbage-collected,
 'destructor(old_cdata_object)' will be called."""
         #
-        return cgc.gc_weakrefs_build(self, w_cdata, w_destructor)
-
-    def descr___gc_wref_remove(self, w_ref):
-        return cgc.gc_wref_remove(self, w_ref)
+        return w_cdata.with_gc(w_destructor)
 
 
     @unwrap_spec(replace_with=str)
@@ -411,7 +415,31 @@
 pointer to the memory somewhere else, e.g. into another structure."""
         #
         w_ctype = self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CTYPE)
-        return w_ctype.newp(w_init)
+        return w_ctype.newp(w_init, allocator.default_allocator)
+
+
+    @unwrap_spec(w_alloc=WrappedDefault(None),
+                 w_free=WrappedDefault(None),
+                 should_clear_after_alloc=int)
+    def descr_new_allocator(self, w_alloc, w_free,
+                            should_clear_after_alloc=1):
+        """\
+Return a new allocator, i.e. a function that behaves like ffi.new()
+but uses the provided low-level 'alloc' and 'free' functions.
+
+'alloc' is called with the size as argument.  If it returns NULL, a
+MemoryError is raised.  'free' is called with the result of 'alloc'
+as argument.  Both can be either Python function or directly C
+functions.  If 'free' is None, then no free function is called.
+If both 'alloc' and 'free' are None, the default is used.
+
+If 'should_clear_after_alloc' is set to False, then the memory
+returned by 'alloc' is assumed to be already cleared (or you are
+fine with garbage); otherwise CFFI will clear it.
+        """
+        #
+        return allocator.new_allocator(self, w_alloc, w_free,
+                                       should_clear_after_alloc)
 
 
     def descr_new_handle(self, w_arg):
@@ -539,12 +567,17 @@
 
 
 @jit.dont_look_inside
-def W_FFIObject___new__(space, w_subtype, __args__):
-    r = space.allocate_instance(W_FFIObject, w_subtype)
+def make_plain_ffi_object(space, w_ffitype=None):
+    if w_ffitype is None:
+        w_ffitype = space.gettypefor(W_FFIObject)
+    r = space.allocate_instance(W_FFIObject, w_ffitype)
     # get in 'src_ctx' a NULL which translation doesn't consider to be constant
     src_ctx = rffi.cast(parse_c_type.PCTX, 0)
     r.__init__(space, src_ctx)
-    return space.wrap(r)
+    return r
+
+def W_FFIObject___new__(space, w_subtype, __args__):
+    return space.wrap(make_plain_ffi_object(space, w_subtype))
 
 def make_CData(space):
     return space.gettypefor(W_CData)
@@ -578,7 +611,6 @@
                                      W_FFIObject.set_errno,
                                      doc=W_FFIObject.doc_errno,
                                      cls=W_FFIObject),
-        __gc_wref_remove = interp2app(W_FFIObject.descr___gc_wref_remove),
         addressof   = interp2app(W_FFIObject.descr_addressof),
         alignof     = interp2app(W_FFIObject.descr_alignof),
         buffer      = interp2app(W_FFIObject.descr_buffer),
@@ -592,6 +624,7 @@
         getctype    = interp2app(W_FFIObject.descr_getctype),
         integer_const = interp2app(W_FFIObject.descr_integer_const),
         new         = interp2app(W_FFIObject.descr_new),
+        new_allocator = interp2app(W_FFIObject.descr_new_allocator),
         new_handle  = interp2app(W_FFIObject.descr_new_handle),
         offsetof    = interp2app(W_FFIObject.descr_offsetof),
         sizeof      = interp2app(W_FFIObject.descr_sizeof),
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,13 +1,13 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.module._cffi_backend import ctypeobj, cdataobj
+from pypy.module._cffi_backend import ctypeobj, cdataobj, allocator
 
 
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_init=WrappedDefault(None))
 def newp(space, w_ctype, w_init):
-    return w_ctype.newp(w_init)
+    return w_ctype.newp(w_init, allocator.default_allocator)
 
 # ____________________________________________________________
 
@@ -18,9 +18,9 @@
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
-def callback(space, w_ctype, w_callable, w_error=None):
+def callback(space, w_ctype, w_callable, w_error=None, w_onerror=None):
     from pypy.module._cffi_backend.ccallback import W_CDataCallback
-    return W_CDataCallback(space, w_ctype, w_callable, w_error)
+    return W_CDataCallback(space, w_ctype, w_callable, w_error, w_onerror)
 
 # ____________________________________________________________
 
@@ -105,3 +105,9 @@
                     "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
+
+# ____________________________________________________________
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData)
+def gcp(space, w_cdata, w_destructor):
+    return w_cdata.with_gc(w_destructor)
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -102,6 +102,8 @@
                 #
             elif op == cffi_opcode.OP_GLOBAL_VAR:
                 # A global variable of the exact type specified here
+                # (nowadays, only used by the ABI mode or backend
+                # compatibility; see OP_GLOBAL_F for the API mode
                 w_ct = realize_c_type.realize_c_type(
                     self.ffi, self.ctx.c_types, getarg(g.c_type_op))
                 g_size = rffi.cast(lltype.Signed, g.c_size_or_direct_fn)
@@ -113,7 +115,13 @@
                 ptr = rffi.cast(rffi.CCHARP, g.c_address)
                 if not ptr:   # for dlopen() style
                     ptr = self.cdlopen_fetch(attr)
-                w_result = cglob.W_GlobSupport(space, w_ct, ptr)
+                w_result = cglob.W_GlobSupport(space, attr, w_ct, ptr=ptr)
+                #
+            elif op == cffi_opcode.OP_GLOBAL_VAR_F:
+                w_ct = realize_c_type.realize_c_type(
+                    self.ffi, self.ctx.c_types, getarg(g.c_type_op))
+                w_result = cglob.W_GlobSupport(space, attr, w_ct,
+                                               fetch_addr=g.c_address)
                 #
             elif (op == cffi_opcode.OP_CONSTANT_INT or
                   op == cffi_opcode.OP_ENUM):
@@ -131,6 +139,9 @@
                     realize_c_type.FUNCPTR_FETCH_CHARP,
                     g.c_address)
                 if w_ct.size <= 0:
+                    raise oefmt(self.ffi.w_FFIError,
+                                "constant '%s' is of type '%s', "
+                                "whose size is not known", attr, w_ct.name)
                     raise oefmt(space.w_SystemError,
                                 "constant has no known size")
                 if not fetch_funcptr:   # for dlopen() style
@@ -172,7 +183,7 @@
             w_value = self._build_attr(attr)
             if w_value is None:
                 if is_getattr and attr == '__all__':
-                    return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
+                    return self.dir1(ignore_global_vars=True)
                 if is_getattr and attr == '__dict__':
                     return self.full_dict_copy()
                 if is_getattr and attr == '__name__':
@@ -206,14 +217,18 @@
     def descr_dir(self):
         return self.dir1()
 
-    def dir1(self, ignore_type=-1):
+    def dir1(self, ignore_global_vars=False):
         space = self.space
         total = rffi.getintfield(self.ctx, 'c_num_globals')
         g = self.ctx.c_globals
         names_w = []
         for i in range(total):
-            if getop(g[i].c_type_op) != ignore_type:
-                names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
+            if ignore_global_vars:
+                op = getop(g[i].c_type_op)
+                if (op == cffi_opcode.OP_GLOBAL_VAR or
+                    op == cffi_opcode.OP_GLOBAL_VAR_F):
+                    continue
+            names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
     def full_dict_copy(self):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1170,6 +1170,14 @@
     BShort = new_primitive_type("short")
     BFunc = new_function_type((BShort,), BShort, False)
     f = callback(BFunc, Zcb1, -42)
+    #
+    seen = []
+    oops_result = None
+    def oops(*args):
+        seen.append(args)
+        return oops_result
+    ff = callback(BFunc, Zcb1, -42, oops)
+    #
     orig_stderr = sys.stderr
     orig_getline = linecache.getline
     try:
@@ -1195,6 +1203,59 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        assert len(seen) == 0
+        assert ff(bigvalue) == -42
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = 81
+        assert ff(bigvalue) == 81
+        oops_result = None
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = "xy"     # not None and not an int!
+        assert ff(bigvalue) == -42
+        oops_result = None
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+TypeError: $integer$
+""")
+        #
+        sys.stderr = cStringIO.StringIO()
+        seen = "not a list"    # this makes the oops() function crash
+        assert ff(bigvalue) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+Traceback (most recent call last):
+  File "$", line $, in oops
+    $
+AttributeError: 'str' object has no attribute 'append'
+""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline
@@ -3341,6 +3402,29 @@
     py.test.raises(RuntimeError, "p[42]")
     py.test.raises(RuntimeError, "p[42] = -1")
 
+def test_mixup():
+    BStruct1 = new_struct_type("foo")
+    BStruct2 = new_struct_type("foo")   # <= same name as BStruct1
+    BStruct3 = new_struct_type("bar")
+    BStruct1Ptr = new_pointer_type(BStruct1)
+    BStruct2Ptr = new_pointer_type(BStruct2)
+    BStruct3Ptr = new_pointer_type(BStruct3)
+    BStruct1PtrPtr = new_pointer_type(BStruct1Ptr)
+    BStruct2PtrPtr = new_pointer_type(BStruct2Ptr)
+    BStruct3PtrPtr = new_pointer_type(BStruct3Ptr)
+    pp1 = newp(BStruct1PtrPtr)
+    pp2 = newp(BStruct2PtrPtr)
+    pp3 = newp(BStruct3PtrPtr)
+    pp1[0] = pp1[0]
+    e = py.test.raises(TypeError, "pp3[0] = pp1[0]")
+    assert str(e.value).startswith("initializer for ctype 'bar *' must be a ")
+    assert str(e.value).endswith(", not cdata 'foo *'")
+    e = py.test.raises(TypeError, "pp2[0] = pp1[0]")
+    assert str(e.value) == ("initializer for ctype 'foo *' appears indeed to "
+                            "be 'foo *', but the types are different (check "
+                            "that you are not e.g. mixing up different ffi "
+                            "instances)")
+
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.2"
+    assert __version__ == "1.2.0"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -114,6 +114,18 @@
         assert ffi.callback("int(int)", lambda x: x + "", -66)(10) == -66
         assert ffi.callback("int(int)", lambda x: x + "", error=-66)(10) == -66
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myerror(exc, val, tb):
+            seen.append(exc)
+        cb = ffi.callback("int(int)", lambda x: x + "", onerror=myerror)
+        assert cb(10) == 0
+        cb = ffi.callback("int(int)", lambda x:int(1E100), -66, onerror=myerror)
+        assert cb(10) == -66
+        assert seen == [TypeError, OverflowError]
+
     def test_ffi_callback_decorator(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
@@ -122,6 +134,37 @@
         assert deco(lambda x: x + "")(10) == -66
         assert deco(lambda x: x + 42)(10) == 52
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def oops(*args):
+            seen.append(args)
+
+        @ffi.callback("int(int)", onerror=oops)
+        def fn1(x):
+            return x + ""
+        assert fn1(10) == 0
+
+        @ffi.callback("int(int)", onerror=oops, error=-66)
+        def fn2(x):
+            return x + ""
+        assert fn2(10) == -66
+
+        assert len(seen) == 2
+        exc, val, tb = seen[0]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn1"
+        exc, val, tb = seen[1]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn2"
+        del seen[:]
+        #
+        raises(TypeError, ffi.callback, "int(int)",
+               lambda x: x, onerror=42)   # <- not callable
+
     def test_ffi_getctype(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
@@ -228,3 +271,99 @@
             import gc
             gc.collect()
         assert seen == [1]
+
+    def test_ffi_new_allocator_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        alloc1 = ffi.new_allocator()
+        alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
+        for retry in range(100):
+            p1 = alloc1("int[10]")
+            p2 = alloc2("int[10]")
+            combination = 0
+            for i in range(10):
+                assert p1[i] == 0
+                combination |= p2[i]
+                p1[i] = -42
+                p2[i] = -43
+            if combination != 0:
+                break
+            del p1, p2
+            import gc; gc.collect()
+        else:
+            raise AssertionError("cannot seem to get an int[10] not "
+                                 "completely cleared")
+
+    def test_ffi_new_allocator_2(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        def myfree(raw):
+            seen.append(raw)
+        alloc1 = ffi.new_allocator(myalloc, myfree)
+        alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+                                   should_clear_after_alloc=False)
+        p1 = alloc1("int[10]")
+        p2 = alloc2("int[]", 10)
+        assert seen == [40, 40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert ffi.typeof(p2) == ffi.typeof("int[]")
+        assert ffi.sizeof(p2) == 40
+        assert p1[5] == 0
+        assert p2[6] == ord('X') * 0x01010101
+        raw1 = ffi.cast("char *", p1)
+        raw2 = ffi.cast("char *", p2)
+        del p1, p2
+        retries = 0
+        while len(seen) != 4:
+            retries += 1
+            assert retries <= 5
+            import gc; gc.collect()
+        assert seen == [40, 40, raw1, raw2]
+        assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+        assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+    def test_ffi_new_allocator_3(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", "X" * size)
+        alloc1 = ffi.new_allocator(myalloc)    # no 'free'
+        p1 = alloc1("int[10]")
+        assert seen == [40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert p1[5] == 0
+
+    def test_ffi_new_allocator_4(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(TypeError, ffi.new_allocator, free=lambda x: None)
+        #
+        def myalloc2(size):
+            raise LookupError
+        alloc2 = ffi.new_allocator(myalloc2)
+        raises(LookupError, alloc2, "int[5]")
+        #
+        def myalloc3(size):
+            return 42
+        alloc3 = ffi.new_allocator(myalloc3)
+        e = raises(TypeError, alloc3, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata object (got int)"
+        #
+        def myalloc4(size):
+            return ffi.cast("int", 42)
+        alloc4 = ffi.new_allocator(myalloc4)
+        e = raises(TypeError, alloc4, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+        #
+        def myalloc5(size):
+            return ffi.NULL
+        alloc5 = ffi.new_allocator(myalloc5)
+        raises(MemoryError, alloc5, "int[5]")
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -16,8 +16,8 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 0, 4):
-        py.test.skip("system cffi module needs to be at least 1.0.4")
+    if cffi.__version_info__ < (1, 2, 0):
+        py.test.skip("system cffi module needs to be at least 1.2.0")
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -500,28 +500,33 @@
             "int foo(int x) { return x + 32; }")
         assert lib.foo(10) == 42
 
-    def test_bad_size_of_global_1(self):
-        ffi, lib = self.prepare(
-            "short glob;",
-            "test_bad_size_of_global_1",
-            "long glob;")
-        raises(ffi.error, getattr, lib, "glob")
-
-    def test_bad_size_of_global_2(self):
-        ffi, lib = self.prepare(
-            "int glob[10];",
-            "test_bad_size_of_global_2",
-            "int glob[9];")
-        e = raises(ffi.error, getattr, lib, "glob")
-        assert str(e.value) == ("global variable 'glob' should be 40 bytes "
-                                "according to the cdef, but is actually 36")
-
-    def test_unspecified_size_of_global(self):
+    def test_unspecified_size_of_global_1(self):
         ffi, lib = self.prepare(
             "int glob[];",
-            "test_unspecified_size_of_global",
+            "test_unspecified_size_of_global_1",
             "int glob[10];")
-        lib.glob    # does not crash
+        assert ffi.typeof(lib.glob) == ffi.typeof("int *")
+
+    def test_unspecified_size_of_global_2(self):
+        ffi, lib = self.prepare(
+            "int glob[][5];",
+            "test_unspecified_size_of_global_2",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+    def test_unspecified_size_of_global_3(self):
+        ffi, lib = self.prepare(
+            "int glob[][...];",
+            "test_unspecified_size_of_global_3",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+    def test_unspecified_size_of_global_4(self):
+        ffi, lib = self.prepare(
+            "int glob[...][...];",
+            "test_unspecified_size_of_global_4",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
     def test_include_1(self):
         ffi1, lib1 = self.prepare(
@@ -869,11 +874,22 @@
             """)
         assert lib.almost_forty_two == 42.25
 
+    def test_constant_of_unknown_size(self):
+        ffi, lib = self.prepare(
+            "typedef ... opaque_t;"
+            "const opaque_t CONSTANT;",
+            'test_constant_of_unknown_size',
+            "typedef int opaque_t;"
+            "const int CONSTANT = 42;")
+        e = raises(ffi.error, getattr, lib, 'CONSTANT')
+        assert str(e.value) == ("constant 'CONSTANT' is of "
+                                "type 'opaque_t', whose size is not known")
+
     def test_variable_of_unknown_size(self):
         ffi, lib = self.prepare("""
             typedef ... opaque_t;
             opaque_t globvar;
-        """, 'test_constant_of_unknown_size', """
+        """, 'test_variable_of_unknown_size', """
             typedef char opaque_t[6];
             opaque_t globvar = "hello";
         """)
@@ -1012,3 +1028,51 @@
         assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
         assert lib.__name__ == repr(lib)
+
+    def test_macro_var_callback(self):
+        ffi, lib = self.prepare(
+            "int my_value; int *(*get_my_value)(void);",
+            'test_macro_var_callback',
+            "int *(*get_my_value)(void);\n"
+            "#define my_value (*get_my_value())")
+        #
+        values = ffi.new("int[50]")
+        def it():
+            for i in range(50):
+                yield i
+        it = it()
+        #
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            return values + it.next()
+        lib.get_my_value = get_my_value
+        #
+        values[0] = 41
+        assert lib.my_value == 41            # [0]
+        p = ffi.addressof(lib, 'my_value')   # [1]
+        assert p == values + 1
+        assert p[-1] == 41
+        assert p[+1] == 0
+        lib.my_value = 42                    # [2]
+        assert values[2] == 42
+        assert p[-1] == 41
+        assert p[+1] == 42
+        #
+        # if get_my_value raises or returns nonsense, the exception is printed
+        # to stderr like with any callback, but then the C expression 'my_value'
+        # expand to '*NULL'.  We assume here that '&my_value' will return NULL
+        # without segfaulting, and check for NULL when accessing the variable.
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            raise LookupError
+        lib.get_my_value = get_my_value
+        raises(ffi.error, getattr, lib, 'my_value')
+        raises(ffi.error, setattr, lib, 'my_value', 50)
+        raises(ffi.error, ffi.addressof, lib, 'my_value')
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            return "hello"
+        lib.get_my_value = get_my_value
+        raises(ffi.error, getattr, lib, 'my_value')
+        e = raises(ffi.error, setattr, lib, 'my_value', 50)
+        assert str(e.value) == "global variable 'my_value' is at address NULL"
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -9,6 +9,7 @@
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
+from pypy.module._cffi_backend import allocator
 
 
 class W_FunctionWrapper(W_Root):
@@ -74,7 +75,8 @@
             # then internally allocate the struct and pass a pointer to it as
             # a first argument.
             if locs[0] == 'R':
-                w_result_cdata = ctype.fargs[0].newp(space.w_None)
+                w_result_cdata = ctype.fargs[0].newp(space.w_None,
+                                                    allocator.nonzero_allocator)
                 args_w = [w_result_cdata] + args_w
                 prepare_args(space, rawfunctype, args_w, 1)
                 #
@@ -116,7 +118,7 @@
             # the equivalent of ffi.new()
             if space.is_w(w_arg, space.w_None):
                 continue
-            w_arg = farg.newp(w_arg)
+            w_arg = farg.newp(w_arg, allocator.default_allocator)
         args_w[i] = w_arg
 
 
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -23,11 +23,16 @@
 # running make inside the src dir
 DYNAMIC_VMPROF = False
 
+if sys.platform.startswith('linux'):
+    libs = ['dl']
+else:
+    libs = []
+
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['vmprof.h', 'trampoline.h'],
     separate_module_files = [SRC.join('trampoline.vmprof.s')],
-    libraries = ['dl'],
+    libraries = libs,
     
     post_include_bits=["""
         int pypy_vmprof_init(void);
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
--- a/pypy/module/_vmprof/src/config.h
+++ b/pypy/module/_vmprof/src/config.h
@@ -1,2 +1,6 @@
 #define HAVE_SYS_UCONTEXT_H
+#if defined(__FreeBSD__) || defined(__APPLE__)
+#define PC_FROM_UCONTEXT uc_mcontext.mc_rip
+#else
 #define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
+#endif
diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
--- a/pypy/module/_vmprof/src/getpc.h
+++ b/pypy/module/_vmprof/src/getpc.h
@@ -132,7 +132,7 @@
   }
 };
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   // See comment above struct CallUnrollInfo.  Only try instruction
   // flow matching if both eip and esp looks reasonable.
   const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
@@ -168,7 +168,7 @@
 typedef int ucontext_t;
 #endif
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
   return NULL;
 }
@@ -178,7 +178,7 @@
 // the right value for your system, and add it to the list in
 // configure.ac (or set it manually in your config.h).
 #else
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
 }
 
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -33,6 +33,9 @@
 //#include <libunwind.h>
 
 #include "vmprof.h"
+#if defined(__FreeBSD__) || defined(__APPLE__)
+#define sighandler_t sig_t
+#endif
 
 #define _unused(x) ((void)x)
 
@@ -259,13 +262,31 @@
 	int marker = MARKER_TRAILER;
 	write(profile_file, &marker, 1);
 
+#ifdef __linux__
     // copy /proc/PID/maps to the end of the profile file
     sprintf(buf, "/proc/%d/maps", getpid());
-    src = fopen(buf, "r");    
+    src = fopen(buf, "r");
+    if (!src) {
+        vmprof_error = "error opening proc maps";
+        return -1;
+    }
     while ((size = fread(buf, 1, BUFSIZ, src))) {
         write(profile_file, buf, size);
     }
     fclose(src);
+#else
+    // freebsd and mac
+    sprintf(buf, "procstat -v %d", getpid());
+    src = popen(buf, "r");
+    if (!src) {
+        vmprof_error = "error calling procstat";
+        return -1;
+    }
+    while ((size = fread(buf, 1, BUFSIZ, src))) {
+        write(profile_file, buf, size);
+    }
+    pclose(src);
+#endif
     close(profile_file);
 	return 0;
 }
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -338,11 +338,14 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            else:
-                # this only runs if fromlist_w != ['*']
-                for w_name in fromlist_w:
-                    if try_getattr(space, w_mod, w_name) is None:
-                        return None
+                else:
+                    fromlist_w = []
+                    # "from x import *" with x already imported and no x.__all__
+                    # always succeeds without doing more imports.  It will
+                    # just copy everything from x.__dict__ as it is now.
+            for w_name in fromlist_w:
+                if try_getattr(space, w_mod, w_name) is None:
+                    return None
         return w_mod
     return first
 
@@ -380,12 +383,12 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     fromlist_w = space.fixedview(w_all)
-            else:
-                # this only runs if fromlist_w != ['*']
-                for w_name in fromlist_w:
-                    if try_getattr(space, w_mod, w_name) is None:
-                        load_part(space, w_path, prefix, space.str0_w(w_name),
-                                  w_mod, tentative=1)
+                else:
+                    fromlist_w = []
+            for w_name in fromlist_w:
+                if try_getattr(space, w_mod, w_name) is None:
+                    load_part(space, w_path, prefix, space.str0_w(w_name),
+                              w_mod, tentative=1)
         return w_mod
     else:
         return first
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -72,6 +72,14 @@
              b          = "insubpackage = 1",
              )
     setuppkg("pkg.pkg2", a='', b='')
+    setuppkg("pkg.withall",
+             __init__  = "__all__ = ['foobar']",
+             foobar    = "found = 123")
+    setuppkg("pkg.withoutall",
+             __init__  = "",
+             foobar    = "found = 123")
+    setuppkg("pkg.bogusall",
+             __init__  = "__all__ = 42")
     setuppkg("pkg_r", inpkg = "import x.y")
     setuppkg("pkg_r.x", y='')
     setuppkg("x")
@@ -758,6 +766,33 @@
         import imp
         raises(ValueError, imp.load_module, "", "", "", [1, 2, 3, 4])
 
+    def test_import_star_finds_submodules_with___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withall import *" in d
+            assert d["foobar"].found == 123
+
+    def test_import_star_does_not_find_submodules_without___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withoutall import *" in d
+            assert "foobar" not in d
+        import pkg.withoutall.foobar     # <- import it here only
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withoutall import *" in d
+            assert d["foobar"].found == 123
+
+    def test_import_star_with_bogus___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            try:
+                exec "from pkg.bogusall import *" in {}
+            except TypeError:
+                pass    # 'int' object does not support indexing
+            else:
+                raise AssertionError("should have failed")
+
+<<<<<<< local
     def test_source_encoding(self):
         import imp
         import encoded
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -196,7 +196,12 @@
                     "'%T' object is not iterable", self)
 
     def descr_str(self, space):
-        return space.wrap(self.get_dtype(space).itemtype.str_format(self, add_quotes=False))
+        tp = self.get_dtype(space).itemtype
+        return space.wrap(tp.str_format(self, add_quotes=False))
+
+    def descr_repr(self, space):
+        tp = self.get_dtype(space).itemtype
+        return space.wrap(tp.str_format(self, add_quotes=True))
 
     def descr_format(self, space, w_spec):
         return space.format(self.item(space), w_spec)
@@ -607,16 +612,25 @@
         return W_StringBox(arr, 0, arr.dtype)

From noreply at buildbot.pypy.org  Fri Jul 17 04:45:02 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 04:45:02 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove forgotten merge conflict marker and
 fix failing tests.
Message-ID: <20150717024502.039DD1C1192@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78562:34a546ca0da4
Date: 2015-07-17 04:20 +0200
http://bitbucket.org/pypy/pypy/changeset/34a546ca0da4/

Log:	Remove forgotten merge conflict marker and fix failing tests.

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -769,30 +769,29 @@
     def test_import_star_finds_submodules_with___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
             d = {}
-            exec "from pkg.withall import *" in d
+            exec("from pkg.withall import *", d)
             assert d["foobar"].found == 123
 
     def test_import_star_does_not_find_submodules_without___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
             d = {}
-            exec "from pkg.withoutall import *" in d
+            exec("from pkg.withoutall import *", d)
             assert "foobar" not in d
         import pkg.withoutall.foobar     # <- import it here only
         for case in ["not-imported-yet", "already-imported"]:
             d = {}
-            exec "from pkg.withoutall import *" in d
+            exec("from pkg.withoutall import *", d)
             assert d["foobar"].found == 123
 
     def test_import_star_with_bogus___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
             try:
-                exec "from pkg.bogusall import *" in {}
+                exec("from pkg.bogusall import *", {})
             except TypeError:
                 pass    # 'int' object does not support indexing
             else:
                 raise AssertionError("should have failed")
 
-<<<<<<< local
     def test_source_encoding(self):
         import imp
         import encoded

From noreply at buildbot.pypy.org  Fri Jul 17 04:45:03 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 04:45:03 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150717024503.2CFB61C1192@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78563:e3d0cafafa1a
Date: 2015-07-17 04:21 +0200
http://bitbucket.org/pypy/pypy/changeset/e3d0cafafa1a/

Log:	2to3

diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -537,9 +537,9 @@
             yield 0
 
         x = f()
-        x.next()
+        next(x)
         try:
-            x.next()
+            next(x)
         except StopIteration:
             y = pickle.loads(pickle.dumps(x))
         assert 'finished' in y.__name__

From noreply at buildbot.pypy.org  Fri Jul 17 04:45:04 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 04:45:04 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150717024504.5EABD1C1192@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78564:6da82c1b733b
Date: 2015-07-17 04:23 +0200
http://bitbucket.org/pypy/pypy/changeset/6da82c1b733b/

Log:	2to3

diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -300,7 +300,7 @@
         seen = []
         def myalloc(size):
             seen.append(size)
-            return ffi.new("char[]", "X" * size)
+            return ffi.new("char[]", b"X" * size)
         def myfree(raw):
             seen.append(raw)
         alloc1 = ffi.new_allocator(myalloc, myfree)
@@ -333,7 +333,7 @@
         seen = []
         def myalloc(size):
             seen.append(size)
-            return ffi.new("char[]", "X" * size)
+            return ffi.new("char[]", b"X" * size)
         alloc1 = ffi.new_allocator(myalloc)    # no 'free'
         p1 = alloc1("int[10]")
         assert seen == [40]

From noreply at buildbot.pypy.org  Fri Jul 17 04:45:05 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 04:45:05 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Mark this as an implementation detail.
Message-ID: <20150717024505.8EF5B1C1192@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78565:b8ef0188e57e
Date: 2015-07-17 04:45 +0200
http://bitbucket.org/pypy/pypy/changeset/b8ef0188e57e/

Log:	Mark this as an implementation detail.

diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -1,6 +1,6 @@
 # Minimal tests for dis module
 
-from test.support import run_unittest, captured_stdout
+from test.support import run_unittest, captured_stdout, check_impl_detail
 import unittest
 import sys
 import dis
@@ -209,7 +209,8 @@
     def test_disassemble_str(self):
         self.do_disassembly_test(expr_str, dis_expr_str)
         self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str)
-        self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str)
+        if check_impl_detail():
+            self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str)
 
 code_info_code_info = """\
 Name:              code_info

From noreply at buildbot.pypy.org  Fri Jul 17 06:46:45 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Jul 2015 06:46:45 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: more failing tests
Message-ID: <20150717044645.9753A1C1192@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78566:b5da748a0419
Date: 2015-07-15 23:42 +0300
http://bitbucket.org/pypy/pypy/changeset/b5da748a0419/

Log:	more failing tests

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -490,6 +490,8 @@
                 assert False,'self._ptr_size unknown'
         # Issue gh-2798
         if '__pypy__' in sys.builtin_module_names:
+            a = np.array(['a'], dtype="O")
+            raises(NotImplementedError, a.astype, ("O", [("name", "O")]))
             skip("(base_dtype, new_dtype) dtype specification discouraged")
         a = np.array(['a'], dtype="O").astype(("O", [("name", "O")]))
         assert a[0] == 'a'
@@ -1407,6 +1409,24 @@
                      "('bottom', [('bleft', ('>f4', (8, 64)), (1,)), "
                      "('bright', '>f4', (8, 36))])])")
 
+        # If the sticky aligned flag is set to True, it makes the
+        # str() function use a dict representation with an 'aligned' flag
+        dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
+                                ('rtile', '>f4', (64, 36))],
+                                (3,)),
+                       ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
+                                   ('bright', '>f4', (8, 36))])],
+                       align=True)
+        assert_equal(str(dt),
+                    "{'names':['top','bottom'], "
+                     "'formats':[([('tiles', ('>f4', (64, 64)), (1,)), "
+                                  "('rtile', '>f4', (64, 36))], (3,)),"
+                                 "[('bleft', ('>f4', (8, 64)), (1,)), "
+                                  "('bright', '>f4', (8, 36))]], "
+                     "'offsets':[0,76800], "
+                     "'itemsize':80000, "
+                     "'aligned':True}")
+
         dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'],
                         'offsets': [0, 1, 2],
                         'titles': ['Red pixel', 'Green pixel', 'Blue pixel']},

From noreply at buildbot.pypy.org  Fri Jul 17 06:46:46 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Jul 2015 06:46:46 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: use a single ''.join()
 instead of lots of string concatenation
Message-ID: <20150717044646.B61991C1192@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78567:9692f8a54feb
Date: 2015-07-16 21:30 +0300
http://bitbucket.org/pypy/pypy/changeset/9692f8a54feb/

Log:	use a single ''.join() instead of lots of string concatenation

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -10,6 +10,7 @@
 from rpython.rlib.objectmodel import (
         specialize, compute_hash, we_are_translated, enforceargs)
 from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
+from rpython.rlib.rstring import StringBuilder
 from pypy.module.micronumpy import types, boxes, support, constants as NPY
 from .base import W_NDimArray
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
@@ -228,10 +229,10 @@
             # so return a string instead of a dict. Also, numpy formats
             # the lists without spaces between elements, so we cannot simply
             # do str(names)
-            names = "'names':["
-            formats = "'formats':["
-            offsets = "'offsets':["
-            titles = "'titles':["
+            names = ["'names':["]
+            formats = ["'formats':["]
+            offsets = ["'offsets':["]
+            titles = ["'titles':["]
             use_titles = False
             show_offsets = False
             offsets_n = []
@@ -239,16 +240,16 @@
             for name, title in self.names:
                 offset, subdtype = self.fields[name]
                 if subdtype.is_record():
-                    substr = space.str_w(space.str(subdtype.descr_get_descr(
-                                                space, style='substr'))) + ","
+                    substr = [space.str_w(space.str(subdtype.descr_get_descr(
+                                                space, style='substr'))), ","]
                 elif subdtype.subdtype is not None:
-                    substr = "'" + subdtype.subdtype.get_str(ignore='') + "',"
+                    substr = ["'", subdtype.subdtype.get_str(ignore=''), "',"]
                 else:
-                    substr = "'" + subdtype.get_str(ignore='') + "',"
+                    substr = ["'", subdtype.get_str(ignore=''), "',"]
                 formats += substr
-                offsets += str(offset) + ','
-                names += "'" + name + "',"
-                titles += "'" + str(title) + "',"
+                offsets += [str(offset),  ',']
+                names += ["'", name, "',"]
+                titles += ["'", str(title), "',"]
                 if title is not None:
                     use_titles = True
                 if total != offset:
@@ -264,10 +265,13 @@
             for i in range(len(offsets_n)):
                 if offsets_n[i] != self.alignment * i:
                     show_offsets = True
-            formats = formats[:-1] + ']'
-            offsets = offsets[:-1] + ']'
-            names = names[:-1] + ']'
-            titles = titles[:-1] + ']'
+            if use_titles and not show_offsets: 
+                return self.descr_get_descr(space, style='descr')
+            # replace the last , with a ]
+            formats[-1] = formats[-1][:-1] + ']'
+            offsets[-1] = offsets[-1][:-1] + ']'
+            names[-1] = names[-1][:-1] + ']'
+            titles[-1] = titles[-1][:-1] + ']'
             if self.alignment < 2:
                 suffix = "}"
             elif style == 'str':
@@ -276,17 +280,12 @@
                 suffix = '}'
             else:
                 suffix = "}, align=True"
-            if use_titles and not show_offsets: 
-                return self.descr_get_descr(space, style='descr')
-            elif use_titles:
-                return space.wrap('{' + names + ', ' + formats + ', ' +
-                            offsets + ', ' + titles + ", 'itemsize':" + 
-                            str(self.elsize) + suffix)
-            else:
-                return space.wrap('{' + names + ', ' + formats + ', ' +
-                            offsets + ', ' + "'itemsize':" + str(self.elsize) +
-                            suffix)
-            
+            s_as_list = ['{'] + names + [', '] + formats + [', '] + offsets + [', ']
+            if use_titles:
+                s_as_list += titles + [', ']
+                    
+            s_as_list += ["'itemsize':", str(self.elsize), suffix]
+            return space.wrap(''.join(s_as_list))
         else:
             descr = []
             total = 0

From noreply at buildbot.pypy.org  Fri Jul 17 06:46:47 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Jul 2015 06:46:47 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: fix failing tests,
 even more logic for creating repr(dtype)
Message-ID: <20150717044647.D94321C1192@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78568:1b6a885833e3
Date: 2015-07-17 01:46 +0300
http://bitbucket.org/pypy/pypy/changeset/1b6a885833e3/

Log:	fix failing tests, even more logic for creating repr(dtype)

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -224,7 +224,7 @@
         if not self.is_record():
             return space.newlist([space.newtuple([space.wrap(""),
                                                   self.descr_get_str(space, simple=simple)])])
-        elif (self.alignment > 1 and style != 'descr') or force_dict:
+        elif (self.alignment > 1 and not style.startswith('descr')) or force_dict:
             # we need to force a sorting order for the keys,
             # so return a string instead of a dict. Also, numpy formats
             # the lists without spaces between elements, so we cannot simply
@@ -241,9 +241,13 @@
                 offset, subdtype = self.fields[name]
                 if subdtype.is_record():
                     substr = [space.str_w(space.str(subdtype.descr_get_descr(
-                                                space, style='substr'))), ","]
+                                                space, style='descr_subdtype'))), ","]
                 elif subdtype.subdtype is not None:
-                    substr = ["'", subdtype.subdtype.get_str(ignore=''), "',"]
+                    substr = ["(", space.str_w(space.str(
+                        subdtype.subdtype.descr_get_descr(space, style='descr_subdtype'))),
+                        ', ',
+                        space.str_w(space.repr(space.newtuple([space.wrap(s) for s in subdtype.shape]))),
+                        "),"]
                 else:
                     substr = ["'", subdtype.get_str(ignore=''), "',"]
                 formats += substr
@@ -272,7 +276,7 @@
             offsets[-1] = offsets[-1][:-1] + ']'
             names[-1] = names[-1][:-1] + ']'
             titles[-1] = titles[-1][:-1] + ']'
-            if self.alignment < 2:
+            if self.alignment < 2 or style.endswith('subdtype'):
                 suffix = "}"
             elif style == 'str':
                 suffix = ", 'aligned':True}"
@@ -292,9 +296,9 @@
             for name, title in self.names:
                 offset, subdtype = self.fields[name]
                 show_offsets = False
-                if total != offset:
+                if total != offset and len(subdtype.shape) < 1:
                     # whoops, need to use other format
-                    return self.descr_get_descr(space, style=style, force_dict=True)
+                    return self.descr_get_descr(space, style=style + '_subdtype', force_dict=True)
                 total += subdtype.elsize
                 ignore = '|'
                 if title:
@@ -311,7 +315,7 @@
                 if subdtype.shape != []:
                     subdescr.append(subdtype.descr_get_shape(space))
                 descr.append(space.newtuple(subdescr[:]))
-            if self.alignment >= 0:
+            if self.alignment >= 0 and not style.endswith('subdtype'):
                 return space.wrap(space.str_w(space.repr(space.newlist(descr))) + ', align=True')                 
             return space.newlist(descr)
 
@@ -711,8 +715,6 @@
         assert isinstance(subdtype, W_Dtype)
         if alignment >= 0:
             maxalign = max(subdtype.alignment, maxalign)
-            if not subdtype.is_record():
-                maxalign = max(subdtype.elsize, maxalign)
             delta = subdtype.alignment
             # Set offset to the next power-of-two above delta
             delta = (delta + maxalign -1) & (-maxalign)
@@ -937,6 +939,9 @@
     shape_w = space.fixedview(w_shape)
     if len(shape_w) < 1:
         return None
+    elif len(shape_w) == 1 and space.isinstance_w(shape_w[0], space.w_tuple):
+        # (base_dtype, new_dtype) dtype spectification
+        return None
     shape = []
     for w_dim in shape_w:
         try:
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1417,7 +1417,7 @@
                        ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
                                    ('bright', '>f4', (8, 36))])],
                        align=True)
-        assert_equal(str(dt),
+        assert str(dt) == (
                     "{'names':['top','bottom'], "
                      "'formats':[([('tiles', ('>f4', (64, 64)), (1,)), "
                                   "('rtile', '>f4', (64, 36))], (3,)),"

From noreply at buildbot.pypy.org  Fri Jul 17 06:46:49 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Jul 2015 06:46:49 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: merge default into branch
Message-ID: <20150717044649.53FEF1C1192@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78569:191ef045b0b6
Date: 2015-07-17 01:46 +0300
http://bitbucket.org/pypy/pypy/changeset/191ef045b0b6/

Log:	merge default into branch

diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
--- a/pypy/module/_vmprof/src/getpc.h
+++ b/pypy/module/_vmprof/src/getpc.h
@@ -132,7 +132,7 @@
   }
 };
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   // See comment above struct CallUnrollInfo.  Only try instruction
   // flow matching if both eip and esp looks reasonable.
   const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
@@ -168,7 +168,7 @@
 typedef int ucontext_t;
 #endif
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
   return NULL;
 }
@@ -178,7 +178,7 @@
 // the right value for your system, and add it to the list in
 // configure.ac (or set it manually in your config.h).
 #else
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
 }
 
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -262,13 +262,31 @@
 	int marker = MARKER_TRAILER;
 	write(profile_file, &marker, 1);
 
+#ifdef __linux__
     // copy /proc/PID/maps to the end of the profile file
     sprintf(buf, "/proc/%d/maps", getpid());
-    src = fopen(buf, "r");    
+    src = fopen(buf, "r");
+    if (!src) {
+        vmprof_error = "error opening proc maps";
+        return -1;
+    }
     while ((size = fread(buf, 1, BUFSIZ, src))) {
         write(profile_file, buf, size);
     }
     fclose(src);
+#else
+    // freebsd and mac
+    sprintf(buf, "procstat -v %d", getpid());
+    src = popen(buf, "r");
+    if (!src) {
+        vmprof_error = "error calling procstat";
+        return -1;
+    }
+    while ((size = fread(buf, 1, BUFSIZ, src))) {
+        write(profile_file, buf, size);
+    }
+    pclose(src);
+#endif
     close(profile_file);
 	return 0;
 }
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1031,6 +1031,48 @@
         A.__dict__['x'] = 5
         assert A.x == 5
 
+    def test_we_already_got_one_1(self):
+        # Issue #2079: highly obscure: CPython complains if we say
+        # ``__slots__="__dict__"`` and there is already a __dict__...
+        # but from the "best base" only.  If the __dict__ comes from
+        # another base, it doesn't complain.  Shrug and copy the logic.
+        class A(object):
+            __slots__ = ()
+        class B(object):
+            pass
+        class C(A, B):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(A, B):     # "best base" is A
+            __slots__ = ("__weakref__",)
+        try:
+            class E(B, A):   # "best base" is B
+                __slots__ = ("__dict__",)
+        except TypeError, e:
+            assert 'we already got one' in str(e)
+        else:
+            raise AssertionError("TypeError not raised")
+        try:
+            class F(B, A):   # "best base" is B
+                __slots__ = ("__weakref__",)
+        except TypeError, e:
+            assert 'we already got one' in str(e)
+        else:
+            raise AssertionError("TypeError not raised")
+
+    def test_we_already_got_one_2(self):
+        class A(object):
+            __slots__ = ()
+        class B:
+            pass
+        class C(A, B):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(A, B):     # "best base" is A
+            __slots__ = ("__weakref__",)
+        class C(B, A):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(B, A):     # "best base" is A
+            __slots__ = ("__weakref__",)
+
 
 class AppTestWithMethodCacheCounter:
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1022,7 +1022,7 @@
     w_self.nslots = w_bestbase.nslots
     return hasoldstylebase
 
-def create_all_slots(w_self, hasoldstylebase):
+def create_all_slots(w_self, hasoldstylebase, w_bestbase):
     space = w_self.space
     dict_w = w_self.dict_w
     if '__slots__' not in dict_w:
@@ -1040,12 +1040,12 @@
         for w_slot_name in slot_names_w:
             slot_name = space.str_w(w_slot_name)
             if slot_name == '__dict__':
-                if wantdict or w_self.hasdict:
+                if wantdict or w_bestbase.hasdict:
                     raise oefmt(space.w_TypeError,
                                 "__dict__ slot disallowed: we already got one")
                 wantdict = True
             elif slot_name == '__weakref__':
-                if wantweakref or w_self.weakrefable:
+                if wantweakref or w_bestbase.weakrefable:
                     raise oefmt(space.w_TypeError,
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
@@ -1106,7 +1106,7 @@
         w_self.flag_abstract |= w_base.flag_abstract
 
     hasoldstylebase = copy_flags_from_bases(w_self, w_bestbase)
-    create_all_slots(w_self, hasoldstylebase)
+    create_all_slots(w_self, hasoldstylebase, w_bestbase)
 
     ensure_common_attributes(w_self)
 

From noreply at buildbot.pypy.org  Fri Jul 17 06:46:50 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Jul 2015 06:46:50 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: document branch to be
	merged
Message-ID: <20150717044650.773A51C1192@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78570:c386a754e4f4
Date: 2015-07-17 01:50 +0300
http://bitbucket.org/pypy/pypy/changeset/c386a754e4f4/

Log:	document branch to be merged

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -37,3 +37,9 @@
 .. branch: unicode-dtype
 
 Partial implementation of unicode dtype and unicode scalars.
+
+.. branch: dtypes-compatibility
+
+Improve compatibility with numpy dtypes; handle offsets to create unions,
+fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
+allow subclassing dtype

From noreply at buildbot.pypy.org  Fri Jul 17 06:46:51 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Jul 2015 06:46:51 +0200 (CEST)
Subject: [pypy-commit] pypy dtypes-compatability: close branch
Message-ID: <20150717044651.8C5291C1192@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: dtypes-compatability
Changeset: r78571:29f6c9e399e1
Date: 2015-07-17 01:51 +0300
http://bitbucket.org/pypy/pypy/changeset/29f6c9e399e1/

Log:	close branch


From noreply at buildbot.pypy.org  Fri Jul 17 06:46:52 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Jul 2015 06:46:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge dtypes-compatability,
 which almost finishes dtype attributes, creation, and display
Message-ID: <20150717044652.E8E521C1192@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78572:c4992c458ed4
Date: 2015-07-17 01:52 +0300
http://bitbucket.org/pypy/pypy/changeset/c4992c458ed4/

Log:	merge dtypes-compatability, which almost finishes dtype attributes,
	creation, and display

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -37,3 +37,9 @@
 .. branch: unicode-dtype
 
 Partial implementation of unicode dtype and unicode scalars.
+
+.. branch: dtypes-compatibility
+
+Improve compatibility with numpy dtypes; handle offsets to create unions,
+fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
+allow subclassing dtype
diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -8,6 +8,7 @@
     w__commastring = None
     w_array_repr = None
     w_array_str = None
+    w__usefields = None
 
     def __init__(self, space):
         pass
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -563,7 +563,7 @@
         elif space.isinstance_w(w_item, space.w_int):
             indx = space.int_w(w_item)
             try:
-                item = self.dtype.names[indx]
+                item = self.dtype.names[indx][0]
             except IndexError:
                 if indx < 0:
                     indx += len(self.dtype.names)
@@ -596,7 +596,7 @@
         try:
             ofs, dtype = self.dtype.fields[item]
         except KeyError:
-            raise oefmt(space.w_IndexError, "222only integers, slices (`:`), "
+            raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
                 "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
                 "boolean arrays are valid indices")
         dtype.store(self.arr, self.ofs, ofs,
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -47,6 +47,9 @@
     def lookup(self, name):
         return self.getdictvalue(self, name)
 
+    def getname(self, space):
+        return self.name
+
 class FakeSpace(ObjSpace):
     w_ValueError = W_TypeObject("ValueError")
     w_TypeError = W_TypeObject("TypeError")
diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -92,6 +92,21 @@
 ARRAY_ELEMENTSTRIDES = 0x0080 # strides  are units of the dtype element size
 ARRAY_NOTSWAPPED  = 0x0200 #native byte order
 
+#dtype flags
+ITEM_REFCOUNT   = 0x01
+ITEM_HASOBJECT  = 0x01
+LIST_PICKLE     = 0x02
+ITEM_IS_POINTER = 0x04
+NEEDS_INIT      = 0x08
+NEEDS_PYAPI     = 0x10
+USE_GETITEM     = 0x20
+USE_SETITEM     = 0x40
+ALIGNED_STRUCT  = 0x80
+FROM_FIELDS     = NEEDS_INIT | LIST_PICKLE | ITEM_REFCOUNT | NEEDS_PYAPI
+OBJECT_DTYPE_FLAGS = (LIST_PICKLE | USE_GETITEM | ITEM_IS_POINTER |
+                        ITEM_REFCOUNT | NEEDS_INIT | NEEDS_PYAPI)
+
+
 LITTLE = '<'
 BIG = '>'
 NATIVE = '='
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -10,6 +10,7 @@
 from rpython.rlib.objectmodel import (
         specialize, compute_hash, we_are_translated, enforceargs)
 from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
+from rpython.rlib.rstring import StringBuilder
 from pypy.module.micronumpy import types, boxes, support, constants as NPY
 from .base import W_NDimArray
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
@@ -54,11 +55,11 @@
 class W_Dtype(W_Root):
     _immutable_fields_ = [
         "itemtype?", "w_box_type", "byteorder?", "names?", "fields?",
-        "elsize?", "alignment?", "shape?", "subdtype?", "base?"]
+        "elsize?", "alignment?", "shape?", "subdtype?", "base?", "flags?"]
 
     @enforceargs(byteorder=SomeChar())
     def __init__(self, itemtype, w_box_type, byteorder=NPY.NATIVE, names=[],
-                 fields={}, elsize=None, shape=[], subdtype=None):
+                 fields={}, elsize=-1, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.w_box_type = w_box_type
         if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType):
@@ -66,16 +67,21 @@
         self.byteorder = byteorder
         self.names = names
         self.fields = fields
-        if elsize is None:
+        if elsize < 0:
             elsize = itemtype.get_element_size()
         self.elsize = elsize
-        self.alignment = itemtype.alignment
         self.shape = shape
         self.subdtype = subdtype
+        self.flags = 0
+        self.metadata = None
+        if isinstance(itemtype, types.ObjectType):
+            self.flags = NPY.OBJECT_DTYPE_FLAGS
         if not subdtype:
             self.base = self
+            self.alignment = itemtype.get_element_size()
         else:
             self.base = subdtype.base
+            self.alignment = subdtype.itemtype.get_element_size()
 
     @property
     def num(self):
@@ -172,51 +178,149 @@
         return dtype
 
     def get_name(self):
-        name = self.w_box_type.name
-        if name.startswith('numpy.'):
-            name = name[6:]
+        name = self.w_box_type.getname(self.itemtype.space)
         if name.endswith('_'):
             name = name[:-1]
         return name
 
-    def descr_get_name(self, space):
-        name = self.get_name()
+    def descr_get_name(self, space, quote=False):
+        if quote:
+            name = "'" + self.get_name() + "'"
+        else:
+            name = self.get_name()
         if self.is_flexible() and self.elsize != 0:
             return space.wrap(name + str(self.elsize * 8))
         return space.wrap(name)
 
-    def descr_get_str(self, space):
+    def descr_get_str(self, space, ignore='|', simple=True):
+        if not simple and self.fields and len(self.fields) > 0:
+            return self.descr_get_descr(space)
+        total = 0
+        for s in self.shape:
+            total += s
+        if not simple and total > 0:
+            return space.newtuple(
+                [space.wrap(self.subdtype.get_str(ignore='')), 
+                 space.newtuple([space.wrap(s) for s in self.shape]),
+                ])
+        return space.wrap(self.get_str(ignore=ignore))
+
+    def get_str(self, ignore='|'):
         basic = self.kind
         endian = self.byteorder
         size = self.elsize
         if endian == NPY.NATIVE:
             endian = NPY.NATBYTE
+        elif endian == NPY.IGNORE:
+            endian = ignore
         if self.num == NPY.UNICODE:
             size >>= 2
-        return space.wrap("%s%s%s" % (endian, basic, size))
+        return "%s%s%s" % (endian, basic, size)
 
-    def descr_get_descr(self, space):
+    def descr_get_descr(self, space, style='descr', force_dict=False):
+        simple = False
+        if style == 'descr':
+            simple = True
         if not self.is_record():
             return space.newlist([space.newtuple([space.wrap(""),
-                                                  self.descr_get_str(space)])])
+                                                  self.descr_get_str(space, simple=simple)])])
+        elif (self.alignment > 1 and not style.startswith('descr')) or force_dict:
+            # we need to force a sorting order for the keys,
+            # so return a string instead of a dict. Also, numpy formats
+            # the lists without spaces between elements, so we cannot simply
+            # do str(names)
+            names = ["'names':["]
+            formats = ["'formats':["]
+            offsets = ["'offsets':["]
+            titles = ["'titles':["]
+            use_titles = False
+            show_offsets = False
+            offsets_n = []
+            total = 0
+            for name, title in self.names:
+                offset, subdtype = self.fields[name]
+                if subdtype.is_record():
+                    substr = [space.str_w(space.str(subdtype.descr_get_descr(
+                                                space, style='descr_subdtype'))), ","]
+                elif subdtype.subdtype is not None:
+                    substr = ["(", space.str_w(space.str(
+                        subdtype.subdtype.descr_get_descr(space, style='descr_subdtype'))),
+                        ', ',
+                        space.str_w(space.repr(space.newtuple([space.wrap(s) for s in subdtype.shape]))),
+                        "),"]
+                else:
+                    substr = ["'", subdtype.get_str(ignore=''), "',"]
+                formats += substr
+                offsets += [str(offset),  ',']
+                names += ["'", name, "',"]
+                titles += ["'", str(title), "',"]
+                if title is not None:
+                    use_titles = True
+                if total != offset:
+                    show_offsets = True
+                total += subdtype.elsize
+                # make sure offsets_n is sorted
+                i = 0
+                for i in range(len(offsets_n)):
+                    if offset < offsets_n[i]:
+                        break
+                offsets_n.insert(i, offset)
+            total = 0
+            for i in range(len(offsets_n)):
+                if offsets_n[i] != self.alignment * i:
+                    show_offsets = True
+            if use_titles and not show_offsets: 
+                return self.descr_get_descr(space, style='descr')
+            # replace the last , with a ]
+            formats[-1] = formats[-1][:-1] + ']'
+            offsets[-1] = offsets[-1][:-1] + ']'
+            names[-1] = names[-1][:-1] + ']'
+            titles[-1] = titles[-1][:-1] + ']'
+            if self.alignment < 2 or style.endswith('subdtype'):
+                suffix = "}"
+            elif style == 'str':
+                suffix = ", 'aligned':True}"
+            elif style == 'substr':
+                suffix = '}'
+            else:
+                suffix = "}, align=True"
+            s_as_list = ['{'] + names + [', '] + formats + [', '] + offsets + [', ']
+            if use_titles:
+                s_as_list += titles + [', ']
+                    
+            s_as_list += ["'itemsize':", str(self.elsize), suffix]
+            return space.wrap(''.join(s_as_list))
         else:
             descr = []
-            for name in self.names:
-                subdtype = self.fields[name][1]
-                subdescr = [space.wrap(name)]
+            total = 0
+            for name, title in self.names:
+                offset, subdtype = self.fields[name]
+                show_offsets = False
+                if total != offset and len(subdtype.shape) < 1:
+                    # whoops, need to use other format
+                    return self.descr_get_descr(space, style=style + '_subdtype', force_dict=True)
+                total += subdtype.elsize
+                ignore = '|'
+                if title:
+                    subdescr = [space.newtuple([space.wrap(title), space.wrap(name)])]
+                    ignore = ''
+                else:
+                    subdescr = [space.wrap(name)]
                 if subdtype.is_record():
-                    subdescr.append(subdtype.descr_get_descr(space))
+                    subdescr.append(subdtype.descr_get_descr(space, style))
                 elif subdtype.subdtype is not None:
-                    subdescr.append(subdtype.subdtype.descr_get_str(space))
+                    subdescr.append(subdtype.subdtype.descr_get_str(space, simple=False))
                 else:
-                    subdescr.append(subdtype.descr_get_str(space))
+                    subdescr.append(subdtype.descr_get_str(space, ignore=ignore, simple=False))
                 if subdtype.shape != []:
                     subdescr.append(subdtype.descr_get_shape(space))
                 descr.append(space.newtuple(subdescr[:]))
+            if self.alignment >= 0 and not style.endswith('subdtype'):
+                return space.wrap(space.str_w(space.repr(space.newlist(descr))) + ', align=True')                 
             return space.newlist(descr)
 
     def descr_get_hasobject(self, space):
-        return space.w_False
+        return space.wrap(self.is_object())
 
     def descr_get_isbuiltin(self, space):
         if self.fields is None:
@@ -238,19 +342,28 @@
     def descr_get_shape(self, space):
         return space.newtuple([space.wrap(dim) for dim in self.shape])
 
+    def descr_get_flags(self, space):
+        return space.wrap(self.flags)
+
     def descr_get_fields(self, space):
         if not self.fields:
             return space.w_None
         w_fields = space.newdict()
-        for name, (offset, subdtype) in self.fields.iteritems():
-            space.setitem(w_fields, space.wrap(name),
+        for name, title in self.names:
+            offset, subdtype = self.fields[name]
+            if title is not None:
+                w_nt = space.newtuple([space.wrap(name), space.wrap(title)]) 
+                space.setitem(w_fields, w_nt,
+                          space.newtuple([subdtype, space.wrap(offset)]))
+            else:
+                space.setitem(w_fields, space.wrap(name),
                           space.newtuple([subdtype, space.wrap(offset)]))
         return w_fields
 
     def descr_get_names(self, space):
         if not self.fields:
             return space.w_None
-        return space.newtuple([space.wrap(name) for name in self.names])
+        return space.newtuple([space.wrap(name[0]) for name in self.names])
 
     def descr_set_names(self, space, w_names):
         if not self.fields:
@@ -262,23 +375,43 @@
                         "with a sequence of length %d",
                         len(self.names))
         names = []
-        for w_name in space.fixedview(w_names):
+        names_w = space.fixedview(w_names)
+        for i in range(len(names_w)):
+            w_name = names_w[i]
+            title = self.names[i][1]
             if not space.isinstance_w(w_name, space.w_str):
                 raise oefmt(space.w_ValueError,
                             "item #%d of names is of type %T and not string",
                             len(names), w_name)
-            names.append(space.str_w(w_name))
+            names.append((space.str_w(w_name), title))
         fields = {}
         for i in range(len(self.names)):
-            if names[i] in fields:
+            if names[i][0] in fields:
                 raise oefmt(space.w_ValueError, "Duplicate field names given.")
-            fields[names[i]] = self.fields[self.names[i]]
+            fields[names[i][0]] = self.fields[self.names[i][0]]
+            if self.names[i][1] is not None:
+                fields[self.names[i][1]] = self.fields[self.names[i][0]]
         self.fields = fields
         self.names = names
 
     def descr_del_names(self, space):
-        raise OperationError(space.w_AttributeError, space.wrap(
-            "Cannot delete dtype names attribute"))
+        raise oefmt(space.w_AttributeError, 
+            "Cannot delete dtype names attribute")
+
+    def descr_get_metadata(self, space):
+        if self.metadata is None:
+            return space.w_None
+        return self.metadata
+
+    def descr_set_metadata(self, space, w_metadata):
+        if w_metadata is None:
+            return
+        if not space.isinstance_w(w_metadata, space.w_dict):
+            raise oefmt(space.w_TypeError, "argument 4 must be dict, not str")
+        self.metadata = w_metadata
+
+    def descr_del_metadata(self, space):
+        self.metadata = None
 
     def eq(self, space, w_other):
         w_other = space.call_function(space.gettypefor(W_Dtype), w_other)
@@ -331,7 +464,8 @@
                 y = intmask((1000003 * y) ^ self.alignment)
             return intmask((1000003 * x) ^ y)
         if self.fields:
-            for name, (offset, subdtype) in self.fields.iteritems():
+            for name in self.fields.keys():
+                offset, subdtype = self.fields[name]
                 assert isinstance(subdtype, W_Dtype)
                 y = intmask(1000003 * (0x345678 ^ compute_hash(name)))
                 y = intmask(1000003 * (y ^ compute_hash(offset)))
@@ -349,7 +483,7 @@
 
     def descr_str(self, space):
         if self.fields:
-            return space.str(self.descr_get_descr(space))
+            return space.str(self.descr_get_descr(space, style='str'))
         elif self.subdtype is not None:
             return space.str(space.newtuple([
                 self.subdtype.descr_get_str(space),
@@ -362,7 +496,7 @@
 
     def descr_repr(self, space):
         if self.fields:
-            r = self.descr_get_descr(space)
+            r = self.descr_get_descr(space, style='repr')
         elif self.subdtype is not None:
             r = space.newtuple([self.subdtype.descr_get_str(space),
                                 self.descr_get_shape(space)])
@@ -375,9 +509,11 @@
                 size = self.elsize
                 if self.num == NPY.UNICODE:
                     size >>= 2
-                r = space.wrap(byteorder + self.char + str(size))
+                r = space.wrap("'" + byteorder + self.char + str(size) + "'")
             else:
-                r = self.descr_get_name(space)
+                r = self.descr_get_name(space, quote=True)
+        if space.isinstance_w(r, space.w_str):
+            return space.wrap("dtype(%s)" % space.str_w(r))
         return space.wrap("dtype(%s)" % space.str_w(space.repr(r)))
 
     def descr_getitem(self, space, w_item):
@@ -389,7 +525,7 @@
         elif space.isinstance_w(w_item, space.w_int):
             indx = space.int_w(w_item)
             try:
-                item = self.names[indx]
+                item,title = self.names[indx]
             except IndexError:
                 raise oefmt(space.w_IndexError,
                     "Field index %d out of range.", indx)
@@ -436,14 +572,17 @@
         values = self.descr_get_fields(space)
         if self.is_flexible():
             w_size = space.wrap(self.elsize)
-            alignment = space.wrap(self.alignment)
+            if self.alignment > 2:
+                w_alignment = space.wrap(self.alignment)
+            else:
+                w_alignment = space.wrap(1)
         else:
             w_size = space.wrap(-1)
-            alignment = space.wrap(-1)
-        flags = space.wrap(0)
+            w_alignment = space.wrap(-1)
+        w_flags = space.wrap(self.flags)
 
         data = space.newtuple([version, space.wrap(endian), subdescr,
-                               names, values, w_size, alignment, flags])
+                               names, values, w_size, w_alignment, w_flags])
         return space.newtuple([w_class, builder_args, data])
 
     def descr_setstate(self, space, w_data):
@@ -465,6 +604,9 @@
         w_fields = space.getitem(w_data, space.wrap(4))
         size = space.int_w(space.getitem(w_data, space.wrap(5)))
         alignment = space.int_w(space.getitem(w_data, space.wrap(6)))
+        if alignment < 2:
+            alignment = -1
+        flags = space.int_w(space.getitem(w_data, space.wrap(7)))
 
         if (w_names == space.w_None) != (w_fields == space.w_None):
             raise oefmt(space.w_ValueError, "inconsistent fields and names in Numpy dtype unpickling")
@@ -492,20 +634,21 @@
             self.names = []
             self.fields = {}
             for w_name in space.fixedview(w_names):
+                # XXX what happens if there is a title in the pickled dtype?
                 name = space.str_w(w_name)
                 value = space.getitem(w_fields, w_name)
 
                 dtype = space.getitem(value, space.wrap(0))
+                offset = space.int_w(space.getitem(value, space.wrap(1)))
+                self.names.append((name, None))
                 assert isinstance(dtype, W_Dtype)
-                offset = space.int_w(space.getitem(value, space.wrap(1)))
-
-                self.names.append(name)
                 self.fields[name] = offset, dtype
             self.itemtype = types.RecordType(space)
 
         if self.is_flexible():
             self.elsize = size
             self.alignment = alignment
+        self.flags = flags
 
     @unwrap_spec(new_order=str)
     def descr_newbyteorder(self, space, new_order=NPY.SWAP):
@@ -526,17 +669,24 @@
 
 
 @specialize.arg(2)
-def dtype_from_list(space, w_lst, simple, align=False):
+def dtype_from_list(space, w_lst, simple, alignment, offsets=None, itemsize=0):
     lst_w = space.listview(w_lst)
     fields = {}
-    offset = 0
-    names = []
-    maxalign = 0
+    use_supplied_offsets = True
+    if offsets is None:
+        use_supplied_offsets = False
+        offsets = [0] * len(lst_w)
+    maxalign = alignment 
+    fldnames = [''] * len(lst_w)
+    subdtypes = [None] * len(lst_w)
+    titles = [None] * len(lst_w)
+    total = 0
     for i in range(len(lst_w)):
         w_elem = lst_w[i]
         if simple:
-            subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_elem)
-            fldname = 'f%d' % i
+            subdtype = make_new_dtype(space, space.gettypefor(W_Dtype), w_elem,
+                                    maxalign)
+            fldnames[i] = 'f%d' % i
         else:
             w_shape = space.newtuple([])
             if space.len_w(w_elem) == 3:
@@ -545,52 +695,213 @@
                     w_shape = space.newtuple([w_shape])
             else:
                 w_fldname, w_flddesc = space.fixedview(w_elem, 2)
-            subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc, w_shape=w_shape)
-            fldname = space.str_w(w_fldname)
-            if fldname == '':
-                fldname = 'f%d' % i
-            if fldname in fields:
+            subdtype = make_new_dtype(space, space.gettypefor(W_Dtype),
+                                    w_flddesc, maxalign, w_shape=w_shape)
+            if space.isinstance_w(w_fldname, space.w_tuple):
+                fldlist = space.listview(w_fldname)
+                fldnames[i] = space.str_w(fldlist[0])
+                if space.is_w(fldlist[1], space.w_None):
+                    titles[i] = None
+                else:
+                    titles[i] = space.str_w(fldlist[1])
+                if len(fldlist) != 2:
+                    raise oefmt(space.w_TypeError, "data type not understood")
+            elif space.isinstance_w(w_fldname, space.w_str): 
+                fldnames[i] = space.str_w(w_fldname)
+            else:
+                raise oefmt(space.w_TypeError, "data type not understood")
+            if fldnames[i] == '':
+                fldnames[i] = 'f%d' % i
+        assert isinstance(subdtype, W_Dtype)
+        if alignment >= 0:
+            maxalign = max(subdtype.alignment, maxalign)
+            delta = subdtype.alignment
+            # Set offset to the next power-of-two above delta
+            delta = (delta + maxalign -1) & (-maxalign)
+            if not use_supplied_offsets:
+                if delta > offsets[i]:
+                    for j in range(i):
+                        offsets[j+1] = delta + offsets[j]
+                if  i + 1 < len(offsets) and offsets[i + 1] == 0:
+                    offsets[i + 1] = offsets[i] + max(delta, subdtype.elsize)
+        elif not use_supplied_offsets:
+            if  i + 1 < len(offsets) and offsets[i + 1] == 0:
+                offsets[i+1] = offsets[i] + subdtype.elsize
+        subdtypes[i] = subdtype
+        if use_supplied_offsets:
+            sz = subdtype.elsize
+        else:
+            sz = max(maxalign, subdtype.elsize)
+        if offsets[i] + sz > total:
+            total = offsets[i] + sz
+    # padding?
+    if alignment >= 0 and total % maxalign:
+        total = total // maxalign * maxalign + maxalign
+    names = []
+    for i in range(len(subdtypes)):
+        subdtype = subdtypes[i]
+        assert isinstance(subdtype, W_Dtype)
+        if alignment >=0 and subdtype.is_record():
+            subdtype.alignment = maxalign
+        if fldnames[i] in fields:
+            raise oefmt(space.w_ValueError, "two fields with the same name")
+        if maxalign > 1 and offsets[i] % subdtype.alignment:
+            raise oefmt(space.w_ValueError, "offset %d for NumPy dtype with "
+                    "fields is not divisible by the field alignment %d "
+                    "with align=True", offsets[i], maxalign)
+        fields[fldnames[i]] = offsets[i], subdtype
+        if titles[i] is not None:
+            if titles[i] in fields:
                 raise oefmt(space.w_ValueError, "two fields with the same name")
-        assert isinstance(subdtype, W_Dtype)
-        fields[fldname] = (offset, subdtype)
-        offset += subdtype.elsize
-        maxalign = max(subdtype.elsize, maxalign)
-        names.append(fldname)
+            fields[titles[i]] = offsets[i], subdtype
+        names.append((fldnames[i], titles[i]))
+    if itemsize > 1:
+        if total > itemsize:
+            raise oefmt(space.w_ValueError,
+                     "NumPy dtype descriptor requires %d bytes, cannot"
+                     " override to smaller itemsize of %d", total, itemsize)
+        if alignment >= 0 and itemsize % maxalign:
+            raise oefmt(space.w_ValueError,
+                    "NumPy dtype descriptor requires alignment of %d bytes, "
+                    "which is not divisible into the specified itemsize %d",
+                    maxalign, itemsize) 
+        total = itemsize
+    retval = W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
+                   names=names, fields=fields, elsize=total)
+    if alignment >=0:
+        retval.alignment = maxalign
+    else:
+        retval.alignment = -1
+    retval.flags |= NPY.NEEDS_PYAPI
+    return retval
+
+def _get_val_or_none(space, w_dict, key):
+    w_key = space.wrap(key)
+    try:
+        w_val = space.getitem(w_dict, w_key)
+    except OperationError as e:
+        if e.match(space, space.w_KeyError):
+            return None
+        else:
+            raise
+    return w_val
+
+def _get_list_or_none(space, w_dict, key):
+    w_val = _get_val_or_none(space, w_dict, key)
+    if w_val is None:
+        return None
+    if space.isinstance_w(w_val, space.w_set):
+        raise oefmt(space.w_TypeError, "'set' object does not support indexing")
+    return space.listview(w_val)
+
+def _usefields(space, w_dict, align):
+    # Only for testing, a shortened version of the real _usefields
+    allfields = []
+    for fname in w_dict.iterkeys().iterator:
+        obj = _get_list_or_none(space, w_dict, fname)
+        num = space.int_w(obj[1])
+        if align:
+            alignment = 0
+        else:
+            alignment = -1
+        format = dtype_from_spec(space, obj[0], alignment=alignment)
+        if len(obj) > 2:
+            title = space.wrap(obj[2])
+        else:
+            title = space.w_None
+        allfields.append((space.wrap(fname), format, num, title))
+    allfields.sort(key=lambda x: x[2])
+    names   = [space.newtuple([x[0], x[3]]) for x in allfields]
+    formats = [x[1] for x in allfields]
+    offsets = [x[2] for x in allfields]
+    aslist = []
     if align:
-        # Set offset to the next power-of-two above offset
-        offset = (offset + maxalign -1) & (-maxalign)
-    return W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
-                   names=names, fields=fields, elsize=offset)
+        alignment = 0
+    else:
+        alignment = -1
+    for i in range(len(names)):
+        aslist.append(space.newtuple([space.wrap(names[i]), space.wrap(formats[i])]))
+    return dtype_from_list(space, space.newlist(aslist), False, alignment, offsets=offsets)
+    
+def dtype_from_dict(space, w_dict, alignment):
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    assert isinstance(w_dict, W_DictMultiObject)
+    names_w = _get_list_or_none(space, w_dict, 'names')
+    formats_w = _get_list_or_none(space, w_dict, 'formats') 
+    offsets_w = _get_list_or_none(space, w_dict, 'offsets')
+    titles_w = _get_list_or_none(space, w_dict, 'titles')
+    metadata_w = _get_val_or_none(space, w_dict, 'metadata')
+    aligned_w = _get_val_or_none(space, w_dict, 'align')
+    itemsize_w = _get_val_or_none(space, w_dict, 'itemsize')
+    if names_w is None or formats_w is None:
+        if we_are_translated():
+            return get_appbridge_cache(space).call_method(space,
+                'numpy.core._internal', '_usefields', Arguments(space, 
+                                [w_dict, space.wrap(alignment >= 0)]))
+        else:
+            return _usefields(space, w_dict, alignment >= 0)
+    n = len(names_w)
+    if (n != len(formats_w) or 
+        (offsets_w is not None and n != len(offsets_w)) or
+        (titles_w is not None and n != len(titles_w))):
+        raise oefmt(space.w_ValueError, "'names', 'formats', 'offsets', and "
+            "'titles' dicct entries must have the same length")
+    if aligned_w is not None:
+        if space.isinstance_w(aligned_w, space.w_bool) and space.is_true(aligned_w):
+            if alignment < 0:
+                alignment = 0 
+        else:
+            raise oefmt(space.w_ValueError,
+                    "NumPy dtype descriptor includes 'aligned' entry, "
+                    "but its value is neither True nor False");
+    if offsets_w is None:
+        offsets = None
+    else:
+        offsets = [space.int_w(i) for i in offsets_w]
+    if titles_w is not None:
+        _names_w = []
+        for i in range(min(len(names_w), len(titles_w))):
+            _names_w.append(space.newtuple([names_w[i], titles_w[i]]))
+        names_w = _names_w
+    aslist = []
+    if itemsize_w is None:
+        itemsize = 0
+    else:
+        itemsize = space.int_w(itemsize_w)
+    for i in range(min(len(names_w), len(formats_w))):
+        aslist.append(space.newtuple([names_w[i], formats_w[i]]))
+    retval = dtype_from_list(space, space.newlist(aslist), False, alignment,
+                             offsets=offsets, itemsize=itemsize)
+    if metadata_w is not None:
+        retval.descr_set_metadata(space, metadata_w)
+    retval.flags |= NPY.NEEDS_PYAPI
+    return retval 
 
-
-def dtype_from_dict(space, w_dict):
-    raise OperationError(space.w_NotImplementedError, space.wrap(
-        "dtype from dict"))
-
-
-def dtype_from_spec(space, w_spec):
+def dtype_from_spec(space, w_spec, alignment):
 
     if we_are_translated():
         w_lst = get_appbridge_cache(space).call_method(space,
             'numpy.core._internal', '_commastring', Arguments(space, [w_spec]))
     else:
-        # testing, handle manually
-        if space.eq_w(w_spec, space.wrap('u4,u4,u4')):
-            w_lst = space.newlist([space.wrap('u4')]*3)
-        if space.eq_w(w_spec, space.wrap('u4,u4,u4')):
-            w_lst = space.newlist([space.wrap('u4')]*3)
-        else:
-            raise oefmt(space.w_RuntimeError,
-                    "cannot parse w_spec")
+        # handle only simple cases for testing
+        if space.isinstance_w(w_spec, space.w_str):
+            spec = [s.strip() for s in space.str_w(w_spec).split(',')]
+            w_lst = space.newlist([space.wrap(s) for s in spec]) 
+        elif space.isinstance_w(w_spec, space.w_list):
+            w_lst = w_spec
     if not space.isinstance_w(w_lst, space.w_list) or space.len_w(w_lst) < 1:
         raise oefmt(space.w_RuntimeError,
                     "_commastring is not returning a list with len >= 1")
     if space.len_w(w_lst) == 1:
         return descr__new__(space, space.gettypefor(W_Dtype),
-                            space.getitem(w_lst, space.wrap(0)))
+                            space.getitem(w_lst, space.wrap(0)), align=alignment>0)
     else:
-        return dtype_from_list(space, w_lst, True)
-
+        try:
+            return dtype_from_list(space, w_lst, True, alignment)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return dtype_from_list(space, w_lst, False, alignment)
+            raise
 
 def _check_for_commastring(s):
     if s[0] in string.digits or s[0] in '<>=|' and s[1] in string.digits:
@@ -608,30 +919,82 @@
             sqbracket -= 1
     return False
 
- at unwrap_spec(align=bool)
-def descr__new__(space, w_subtype, w_dtype, align=False, w_copy=None, w_shape=None):
-    # align and w_copy are necessary for pickling
+def _set_metadata_and_copy(space, w_metadata, dtype, copy=False):
     cache = get_dtype_cache(space)
+    assert isinstance(dtype, W_Dtype)
+    if copy or (dtype in cache.builtin_dtypes and w_metadata is not None):
+        dtype = W_Dtype(dtype.itemtype, dtype.w_box_type, dtype.byteorder)
+    if w_metadata is not None:
+        dtype.descr_set_metadata(space, w_metadata)
+    return dtype
 
-    if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or
-                                space.len_w(w_shape) > 0):
-        subdtype = descr__new__(space, w_subtype, w_dtype, align, w_copy)
+def _get_shape(space, w_shape):
+    if w_shape is None:
+        return None
+    if space.isinstance_w(w_shape, space.w_int):
+        dim = space.int_w(w_shape)
+        if dim == 1:
+            return None
+        return [dim]
+    shape_w = space.fixedview(w_shape)
+    if len(shape_w) < 1:
+        return None
+    elif len(shape_w) == 1 and space.isinstance_w(shape_w[0], space.w_tuple):
+        # (base_dtype, new_dtype) dtype spectification
+        return None
+    shape = []
+    for w_dim in shape_w:
+        try:
+            dim = space.int_w(w_dim)
+        except OperationError as e:
+            if e.match(space, space.w_OverflowError):
+                raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple.")
+            else:
+                raise
+        if dim > 2 ** 32 -1:
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                  "dimension does not fit into a C int.")
+        elif dim < 0:
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                  "dimension smaller than zero.")
+        shape.append(dim)
+    return shape
+
+ at unwrap_spec(align=bool, copy=bool)
+def descr__new__(space, w_subtype, w_dtype, align=False, copy=False,
+                 w_shape=None, w_metadata=None):
+    if align:
+        alignment = 0
+    else:
+        alignment = -1
+    return make_new_dtype(space, w_subtype, w_dtype, alignment, copy=copy,
+                          w_shape=w_shape, w_metadata=w_metadata)
+
+def make_new_dtype(space, w_subtype, w_dtype, alignment, copy=False, w_shape=None, w_metadata=None):
+    cache = get_dtype_cache(space)
+    shape = _get_shape(space, w_shape)
+    if shape is not None:
+        subdtype = make_new_dtype(space, w_subtype, w_dtype, alignment, copy, w_metadata=w_metadata)
         assert isinstance(subdtype, W_Dtype)
-        size = 1
-        if space.isinstance_w(w_shape, space.w_int):
-            w_shape = space.newtuple([w_shape])
-        shape = []
-        for w_dim in space.fixedview(w_shape):
-            dim = space.int_w(w_dim)
-            shape.append(dim)
-            size *= dim
-        if size == 1:
-            return subdtype
+        size = support.product(shape)
         size *= subdtype.elsize
-        return W_Dtype(types.VoidType(space),
-                       space.gettypefor(boxes.W_VoidBox),
-                       shape=shape, subdtype=subdtype, elsize=size)
-
+        if size >= 2 ** 31:
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                  "dtype size in bytes must fit into a C int.")
+        return _set_metadata_and_copy(space, w_metadata,
+               W_Dtype(types.VoidType(space), space.gettypefor(boxes.W_VoidBox),
+                       shape=shape, subdtype=subdtype, elsize=size))
+    elif w_shape is not None and not space.isinstance_w(w_shape, space.w_int):
+        spec = space.listview(w_shape)
+        if len(spec) > 0:
+            # this is (base_dtype, new_dtype) so just make it a union by setting both
+            # parts' offset to 0
+            try:
+                dtype1 = make_new_dtype(space, w_subtype, w_shape, alignment)
+            except:
+                raise
+            raise oefmt(space.w_NotImplementedError, 
+                "(base_dtype, new_dtype) dtype spectification discouraged, not implemented")
     if space.is_none(w_dtype):
         return cache.w_float64dtype
     if space.isinstance_w(w_dtype, w_subtype):
@@ -641,7 +1004,8 @@
     if space.isinstance_w(w_dtype, space.w_str):
         name = space.str_w(w_dtype)
         if _check_for_commastring(name):
-            return dtype_from_spec(space, w_dtype)
+            return _set_metadata_and_copy(space, w_metadata,
+                                dtype_from_spec(space, w_dtype, alignment))
         cname = name[1:] if name[0] == NPY.OPPBYTE else name
         try:
             dtype = cache.dtypes_by_name[cname]
@@ -655,26 +1019,34 @@
             return variable_dtype(space, name)
         raise oefmt(space.w_TypeError, 'data type "%s" not understood', name)
     elif space.isinstance_w(w_dtype, space.w_list):
-        return dtype_from_list(space, w_dtype, False, align=align)
+        return _set_metadata_and_copy( space, w_metadata,
+                        dtype_from_list(space, w_dtype, False, alignment), copy)
     elif space.isinstance_w(w_dtype, space.w_tuple):
         w_dtype0 = space.getitem(w_dtype, space.wrap(0))
         w_dtype1 = space.getitem(w_dtype, space.wrap(1))
-        subdtype = descr__new__(space, w_subtype, w_dtype0, align, w_copy)
+        subdtype = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy)
         assert isinstance(subdtype, W_Dtype)
         if subdtype.elsize == 0:
             name = "%s%d" % (subdtype.kind, space.int_w(w_dtype1))
-            return descr__new__(space, w_subtype, space.wrap(name), align, w_copy)
-        return descr__new__(space, w_subtype, w_dtype0, align, w_copy, w_shape=w_dtype1)
+            retval = make_new_dtype(space, w_subtype, space.wrap(name), alignment, copy)
+        else:
+            retval = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy, w_shape=w_dtype1)
+        return _set_metadata_and_copy(space, w_metadata, retval, copy)
     elif space.isinstance_w(w_dtype, space.w_dict):
-        return dtype_from_dict(space, w_dtype)
+        return _set_metadata_and_copy(space, w_metadata,
+                dtype_from_dict(space, w_dtype, alignment), copy)
     for dtype in cache.builtin_dtypes:
         if dtype.num in cache.alternate_constructors and \
                 w_dtype in cache.alternate_constructors[dtype.num]:
-            return dtype
+            return _set_metadata_and_copy(space, w_metadata, dtype, copy)
         if w_dtype is dtype.w_box_type:
-            return dtype
+            return _set_metadata_and_copy(space, w_metadata, dtype, copy)
+        if space.isinstance_w(w_dtype, space.w_type) and \
+           space.is_true(space.issubtype(w_dtype, dtype.w_box_type)):
+            return _set_metadata_and_copy( space, w_metadata,
+                            W_Dtype(dtype.itemtype, w_dtype, elsize=0), copy)
     if space.isinstance_w(w_dtype, space.w_type):
-        return cache.w_objectdtype
+        return _set_metadata_and_copy(space, w_metadata, cache.w_objectdtype, copy)
     raise oefmt(space.w_TypeError, "data type not understood")
 
 
@@ -702,6 +1074,11 @@
     names = GetSetProperty(W_Dtype.descr_get_names,
                            W_Dtype.descr_set_names,
                            W_Dtype.descr_del_names),
+    metadata = GetSetProperty(W_Dtype.descr_get_metadata,
+                           #W_Dtype.descr_set_metadata,
+                           #W_Dtype.descr_del_metadata,
+                            ),
+    flags = GetSetProperty(W_Dtype.descr_get_flags),
 
     __eq__ = interp2app(W_Dtype.descr_eq),
     __ne__ = interp2app(W_Dtype.descr_ne),
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -16,7 +16,7 @@
 
     def apply(self, space, orig_arr):
         arr = orig_arr.implementation
-        ofs, subdtype = arr.dtype.fields[self.name]
+        ofs, subdtype = arr.dtype.fields[self.name][:2]
         # ofs only changes start
         # create a view of the original array by extending
         # the shape, strides, backstrides of the array
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -92,6 +92,7 @@
         assert d == np.dtype('i8')
         assert d.shape == ()
         d = np.dtype((np.int64, 1,))
+        assert d.shape == ()
         assert d == np.dtype('i8')
         assert d.shape == ()
         d = np.dtype((np.int64, 4))
@@ -111,6 +112,7 @@
         assert "int8" == dtype("int8")
         raises(TypeError, lambda: dtype("int8") == 3)
         assert dtype(bool) == bool
+        assert dtype('f8') != dtype(('f8', (1,)))
 
     def test_dtype_cmp(self):
         from numpy import dtype
@@ -342,10 +344,10 @@
         raises(TypeError, type, "Foo", (dtype,), {})
 
     def test_can_subclass(self):
-        import numpy
-        class xyz(numpy.void):
+        import numpy as np
+        class xyz(np.void):
             pass
-        assert True
+        assert np.dtype(xyz).name == 'xyz'
 
     def test_index(self):
         import numpy as np
@@ -413,7 +415,7 @@
         assert loads(dumps(a.dtype)) == a.dtype
         assert np.dtype('bool').__reduce__() == (dtype, ('b1', 0, 1), (3, '|', None, None, None, -1, -1, 0))
         assert np.dtype('|V16').__reduce__() == (dtype, ('V16', 0, 1), (3, '|', None, None, None, 16, 1, 0))
-        assert np.dtype(('<f8', 2)).__reduce__() == (dtype, ('V16', 0, 1), (3, '|', (dtype('float64'), (2,)), None, None, 16, 1, 0))
+        assert np.dtype(('<f8', 2)).__reduce__() == (dtype, ('V16', 0, 1), (3, '|', (dtype('float64'), (2,)), None, None, 16, 8, 0))
 
     def test_newbyteorder(self):
         import numpy as np
@@ -480,13 +482,21 @@
         class O(object):
             pass
         for o in [object, O]:
-            print np.dtype(o).byteorder
             if self.ptr_size == 4:
                 assert np.dtype(o).str == '|O4'
             elif self.ptr_size == 8:
                 assert np.dtype(o).str == '|O8'
             else:
                 assert False,'self._ptr_size unknown'
+        # Issue gh-2798
+        if '__pypy__' in sys.builtin_module_names:
+            a = np.array(['a'], dtype="O")
+            raises(NotImplementedError, a.astype, ("O", [("name", "O")]))
+            skip("(base_dtype, new_dtype) dtype specification discouraged")
+        a = np.array(['a'], dtype="O").astype(("O", [("name", "O")]))
+        assert a[0] == 'a'
+        assert a == 'a'
+        assert a['name'].dtype == a.dtype
 
 class AppTestTypes(BaseAppTestDtypes):
     def test_abstract_types(self):
@@ -686,16 +696,8 @@
                                       numpy.integer, numpy.number,
                                       numpy.generic, object]
         import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            # These tests pass "by chance" on numpy, things that are larger than
-            # platform long (i.e. a python int), don't get put in a normal box,
-            # instead they become an object array containing a long, we don't have
-            # yet, so these can't pass.
-            assert numpy.uint64(9223372036854775808) == 9223372036854775808
-            assert numpy.uint64(18446744073709551615) == 18446744073709551615
-        else:
-            raises(OverflowError, numpy.int64, 9223372036854775808)
-            raises(OverflowError, numpy.int64, 18446744073709551615)
+        raises(OverflowError, numpy.int64, 9223372036854775808)
+        raises(OverflowError, numpy.int64, 18446744073709551615)
         raises(OverflowError, numpy.uint64, 18446744073709551616)
         assert numpy.uint64((2<<63) - 1) == (2<<63) - 1
 
@@ -1136,10 +1138,7 @@
         import sys
         d = {'names': ['r','g','b','a'],
              'formats': [np.uint8, np.uint8, np.uint8, np.uint8]}
-        if '__pypy__' not in sys.builtin_module_names:
-            dt = np.dtype(d)
-        else:
-            raises(NotImplementedError, np.dtype, d)
+        dt = np.dtype(d)
 
     def test_create_subarrays(self):
         from numpy import dtype
@@ -1272,7 +1271,7 @@
                      ('x', 'y', 'z', 'value'),
                      {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0),
                       'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12),
-                      }, 20, 1, 0))
+                      }, 20, 1, 16))
 
         new_d = loads(dumps(d))
 
@@ -1291,6 +1290,182 @@
 
         assert new_d.itemsize == d.itemsize == 76
 
+    def test_shape_invalid(self):
+        import numpy as np
+        # Check that the shape is valid.
+        max_int = 2 ** (8 * 4 - 1)
+        max_intp = 2 ** (8 * np.dtype('intp').itemsize - 1) - 1
+        # Too large values (the datatype is part of this)
+        raises(ValueError, np.dtype, [('a', 'f4', max_int // 4 + 1)])
+        raises(ValueError, np.dtype, [('a', 'f4', max_int + 1)])
+        raises(ValueError, np.dtype, [('a', 'f4', (max_int, 2))])
+        # Takes a different code path (fails earlier:
+        raises(ValueError, np.dtype, [('a', 'f4', max_intp + 1)])
+        # Negative values
+        raises(ValueError, np.dtype, [('a', 'f4', -1)])
+        raises(ValueError, np.dtype, [('a', 'f4', (-1, -1))])
+
+    def test_aligned_size(self):
+        import numpy as np
+        # Check that structured dtypes get padded to an aligned size
+        dt = np.dtype('i4, i1', align=True)
+        assert dt.itemsize == 8
+        dt = np.dtype([('f0', 'i4'), ('f1', 'i1')], align=True)
+        assert dt.itemsize == 8
+        dt = np.dtype({'names':['f0', 'f1'],
+                       'formats':['i4', 'u1'],
+                       'offsets':[0, 4]}, align=True)
+        assert dt.itemsize == 8
+        dt = np.dtype({'f0': ('i4', 0), 'f1':('u1', 4)}, align=True)
+        assert dt.itemsize == 8
+        assert dt.alignment == 4
+        assert str(dt) == "{'names':['f0','f1'], 'formats':['<i4','u1'], 'offsets':[0,4], 'itemsize':8, 'aligned':True}"
+        dt = np.dtype([('f1', 'u1'), ('f0', 'i4')], align=True)
+        assert str(dt) == "{'names':['f1','f0'], 'formats':['u1','<i4'], 'offsets':[0,4], 'itemsize':8, 'aligned':True}"
+        # Nesting should preserve that alignment
+        dt1 = np.dtype([('f0', 'i4'),
+                       ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
+                       ('f2', 'i1')], align=True)
+        assert dt1.alignment == 4
+        assert dt1['f1'].itemsize == 12
+        assert dt1.itemsize == 20
+        dt2 = np.dtype({'names':['f0', 'f1', 'f2'],
+                       'formats':['i4',
+                                  [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')],
+                                  'i1'],
+                       'offsets':[0, 4, 16]}, align=True)
+        assert dt2.itemsize == 20
+        dt3 = np.dtype({'f0': ('i4', 0),
+                       'f1': ([('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')], 4),
+                       'f2': ('i1', 16)}, align=True)
+        assert dt3.itemsize == 20
+        assert dt1 == dt2
+        answer = "{'names':['f0','f1','f2'], " + \
+                    "'formats':['<i4',{'names':['f1','f2','f3'], " + \
+                                      "'formats':['i1','<i4','i1'], " + \
+                                      "'offsets':[0,4,8], 'itemsize':12}," + \
+                                 "'i1'], " + \
+                    "'offsets':[0,4,16], 'itemsize':20, 'aligned':True}"
+        assert str(dt3) == answer
+        assert dt2 == dt3
+        # Nesting should preserve packing
+        dt1 = np.dtype([('f0', 'i4'),
+                       ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
+                       ('f2', 'i1')], align=False)
+        assert dt1.itemsize == 11
+        dt2 = np.dtype({'names':['f0', 'f1', 'f2'],
+                       'formats':['i4',
+                                  [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')],
+                                  'i1'],
+                       'offsets':[0, 4, 10]}, align=False)
+        assert dt2.itemsize == 11
+        dt3 = np.dtype({'f0': ('i4', 0),
+                       'f1': ([('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')], 4),
+                       'f2': ('i1', 10)}, align=False)
+        assert dt3.itemsize == 11
+        assert dt1 == dt2
+        assert dt2 == dt3
+
+    def test_bad_param(self):
+        import numpy as np
+        # Can't give a size that's too small
+        raises(ValueError, np.dtype,
+                        {'names':['f0', 'f1'],
+                         'formats':['i4', 'i1'],
+                         'offsets':[0, 4],
+                         'itemsize':4})
+        # If alignment is enabled, the alignment (4) must divide the itemsize
+        raises(ValueError, np.dtype,
+                        {'names':['f0', 'f1'],
+                         'formats':['i4', 'i1'],
+                         'offsets':[0, 4],
+                         'itemsize':9}, align=True)
+        # If alignment is enabled, the individual fields must be aligned
+        raises(ValueError, np.dtype,
+                        {'names':['f0', 'f1'],
+                         'formats':['i1', 'f4'],
+                         'offsets':[0, 2]}, align=True)
+        dt = np.dtype(np.double)
+        attr = ["subdtype", "descr", "str", "name", "base", "shape",
+                "isbuiltin", "isnative", "isalignedstruct", "fields",
+                "metadata", "hasobject"]
+        for s in attr:
+            raises(AttributeError, delattr, dt, s)
+
+        raises(TypeError, np.dtype,
+            dict(names=set(['A', 'B']), formats=['f8', 'i4']))
+        raises(TypeError, np.dtype,
+            dict(names=['A', 'B'], formats=set(['f8', 'i4'])))
+
+    def test_complex_dtype_repr(self):
+        import numpy as np
+        dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
+                                ('rtile', '>f4', (64, 36))], (3,)),
+                       ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
+                                   ('bright', '>f4', (8, 36))])])
+        assert repr(dt) == (
+                     "dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)), "
+                     "('rtile', '>f4', (64, 36))], (3,)), "
+                     "('bottom', [('bleft', ('>f4', (8, 64)), (1,)), "
+                     "('bright', '>f4', (8, 36))])])")
+
+        # If the sticky aligned flag is set to True, it makes the
+        # str() function use a dict representation with an 'aligned' flag
+        dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
+                                ('rtile', '>f4', (64, 36))],
+                                (3,)),
+                       ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
+                                   ('bright', '>f4', (8, 36))])],
+                       align=True)
+        assert str(dt) == (
+                    "{'names':['top','bottom'], "
+                     "'formats':[([('tiles', ('>f4', (64, 64)), (1,)), "
+                                  "('rtile', '>f4', (64, 36))], (3,)),"
+                                 "[('bleft', ('>f4', (8, 64)), (1,)), "
+                                  "('bright', '>f4', (8, 36))]], "
+                     "'offsets':[0,76800], "
+                     "'itemsize':80000, "
+                     "'aligned':True}")
+
+        dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'],
+                        'offsets': [0, 1, 2],
+                        'titles': ['Red pixel', 'Green pixel', 'Blue pixel']},
+                        align=True)
+        assert repr(dt) == (
+                    "dtype([(('Red pixel', 'r'), 'u1'), "
+                    "(('Green pixel', 'g'), 'u1'), "
+                    "(('Blue pixel', 'b'), 'u1')], align=True)")
+
+        dt = np.dtype({'names': ['rgba', 'r', 'g', 'b'],
+                       'formats': ['<u4', 'u1', 'u1', 'u1'],
+                       'offsets': [0, 0, 1, 2],
+                       'titles': ['Color', 'Red pixel',
+                                  'Green pixel', 'Blue pixel']}, align=True)
+        assert repr(dt) == (
+                    "dtype({'names':['rgba','r','g','b'],"
+                    " 'formats':['<u4','u1','u1','u1'],"
+                    " 'offsets':[0,0,1,2],"
+                    " 'titles':['Color','Red pixel',"
+                              "'Green pixel','Blue pixel'],"
+                    " 'itemsize':4}, align=True)")
+
+        dt = np.dtype({'names': ['r', 'b'], 'formats': ['u1', 'u1'],
+                        'offsets': [0, 2],
+                        'titles': ['Red pixel', 'Blue pixel'],
+                        'itemsize': 4})
+        assert repr(dt) == (
+                    "dtype({'names':['r','b'], "
+                    "'formats':['u1','u1'], "
+                    "'offsets':[0,2], "
+                    "'titles':['Red pixel','Blue pixel'], "
+                    "'itemsize':4})")
+        if 'datetime64' not in dir(np):
+            skip('datetime dtype not available')
+        dt = np.dtype([('a', '<M8[D]'), ('b', '<m8[us]')])
+        assert repr(dt) == (
+                    "dtype([('a', '<M8[D]'), ('b', '<m8[us]')])")
+
+
 class AppTestNotDirect(BaseNumpyAppTest):
     def setup_class(cls):
         BaseNumpyAppTest.setup_class.im_func(cls)
@@ -1332,4 +1507,45 @@
         assert a[0] == 1
         assert (a + a)[1] == 4
 
+class AppTestMonsterType(BaseNumpyAppTest):
+    """Test deeply nested subtypes."""
+    def test1(self):
+        import numpy as np
+        simple1 = np.dtype({'names': ['r', 'b'], 'formats': ['u1', 'u1'],
+            'titles': ['Red pixel', 'Blue pixel']})
+        a = np.dtype([('yo', int), ('ye', simple1),
+            ('yi', np.dtype((int, (3, 2))))])
+        b = np.dtype([('yo', int), ('ye', simple1),
+            ('yi', np.dtype((int, (3, 2))))])
+        assert a == b
 
+        c = np.dtype([('yo', int), ('ye', simple1),
+            ('yi', np.dtype((a, (3, 2))))])
+        d = np.dtype([('yo', int), ('ye', simple1),
+            ('yi', np.dtype((a, (3, 2))))])
+        assert c == d
+
+
+class AppTestMetadata(BaseNumpyAppTest):
+    def test_no_metadata(self):
+        import numpy as np
+        d = np.dtype(int)
+        assert d.metadata is None
+
+    def test_metadata_takes_dict(self):
+        import numpy as np
+        d = np.dtype(int, metadata={'datum': 1})
+        assert d.metadata == {'datum': 1}
+
+    def test_metadata_rejects_nondict(self):
+        import numpy as np
+        raises(TypeError, np.dtype, int, metadata='datum')
+        raises(TypeError, np.dtype, int, metadata=1)
+        raises(TypeError, np.dtype, int, metadata=None)
+
+    def test_nested_metadata(self):
+        import numpy as np
+        d = np.dtype([('a', np.dtype(int, metadata={'datum': 1}))])
+        assert d['a'].metadata == {'datum': 1}
+
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2396,7 +2396,7 @@
         dt = item.arr.dtype
         ret_unwrapped = []
         for name in dt.names:
-            ofs, dtype = dt.fields[name]
+            ofs, dtype = dt.fields[name[0]]
             # XXX: code duplication with W_VoidBox.descr_getitem()
             if isinstance(dtype.itemtype, VoidType):
                 read_val = dtype.itemtype.readarray(item.arr, ofs, 0, dtype)
@@ -2431,14 +2431,14 @@
                 return w_item
             else:
                 # match up the field names
-                items_w = [None] * len(dtype.fields)
-                for i in range(len(dtype.fields)):
+                items_w = [None] * len(dtype.names)
+                for i in range(len(dtype.names)):
                     name = dtype.names[i]
                     if name in w_item.dtype.names:
-                        items_w[i] = w_item.descr_getitem(space, space.wrap(name))
+                        items_w[i] = w_item.descr_getitem(space, space.wrap(name[0]))
         elif w_item is not None:
             if space.isinstance_w(w_item, space.w_tuple):
-                if len(dtype.fields) != space.len_w(w_item):
+                if len(dtype.names) != space.len_w(w_item):
                     raise OperationError(space.w_ValueError, space.wrap(
                         "size of tuple must match number of fields."))
                 items_w = space.fixedview(w_item)
@@ -2449,12 +2449,12 @@
                             "expected a readable buffer object")
             else:
                 # XXX support initializing from readable buffers
-                items_w = [w_item] * len(dtype.fields)
+                items_w = [w_item] * len(dtype.names)
         else:
             items_w = [None] * len(dtype.fields)
         arr = VoidBoxStorage(dtype.elsize, dtype)
-        for i in range(len(dtype.fields)):
-            ofs, subdtype = dtype.fields[dtype.names[i]]
+        for i in range(len(dtype.names)):
+            ofs, subdtype = dtype.fields[dtype.names[i][0]]
             try:
                 w_box = subdtype.coerce(space, items_w[i])
             except IndexError:
@@ -2492,7 +2492,7 @@
         items = []
         dtype = box.dtype
         for name in dtype.names:
-            ofs, subdtype = dtype.fields[name]
+            ofs, subdtype = dtype.fields[name[0]]
             subbox = subdtype.read(box.arr, box.ofs, ofs)
             items.append(subdtype.itemtype.to_builtin_type(space, subbox))
         return space.newtuple(items)
@@ -2503,7 +2503,7 @@
         pieces = ["("]
         first = True
         for name in box.dtype.names:
-            ofs, subdtype = box.dtype.fields[name]
+            ofs, subdtype = box.dtype.fields[name[0]]
             if first:
                 first = False
             else:
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -72,6 +72,10 @@
     def get_module(self):
         return w_some_obj()
 
+
+    def getname(self, space):
+        return self.name
+
 def w_some_obj():
     if NonConstant(False):
         return W_Root()

From noreply at buildbot.pypy.org  Fri Jul 17 06:46:54 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Jul 2015 06:46:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: whoops
Message-ID: <20150717044654.186CE1C1192@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78573:a1f4fa173a0d
Date: 2015-07-17 02:00 +0300
http://bitbucket.org/pypy/pypy/changeset/a1f4fa173a0d/

Log:	whoops

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -38,7 +38,7 @@
 
 Partial implementation of unicode dtype and unicode scalars.
 
-.. branch: dtypes-compatibility
+.. branch: dtypes-compatability
 
 Improve compatibility with numpy dtypes; handle offsets to create unions,
 fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,

From noreply at buildbot.pypy.org  Fri Jul 17 11:48:18 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Jul 2015 11:48:18 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: work some more on short
	preamble
Message-ID: <20150717094818.750DE1C1334@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78574:b7aec727abac
Date: 2015-07-16 15:17 +0200
http://bitbucket.org/pypy/pypy/changeset/b7aec727abac/

Log:	work some more on short preamble

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -9,7 +9,8 @@
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers
+from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\
+     AbstractResOp
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp.optimizeopt import info
 
@@ -53,12 +54,11 @@
                                              descr):
         assert self._lazy_setfield is None
         for i, info in enumerate(self.cached_infos):
-            structbox = self.cached_structs[i]
-            op = info._fields[descr.get_index()]
-            op = optimizer.get_box_replacement(op)
+            structbox = optimizer.get_box_replacement(self.cached_structs[i])
+            op = optimizer.get_box_replacement(info._fields[descr.get_index()])
             opnum = OpHelpers.getfield_for_descr(descr)
             getfield_op = ResOperation(opnum, [structbox], descr=descr)
-            shortboxes.add_potential(op, getfield_op)
+            shortboxes.add_heap_op(op, getfield_op)
         return
         for structvalue in self._cached_fields_getfield_op.keys():
             op = self._cached_fields_getfield_op[structvalue]
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -221,9 +221,9 @@
         ops = self.optimizer._newoperations
         for i, op in enumerate(ops):
             if op.is_always_pure():
-                sb.add_potential(op)
+                sb.add_pure_op(op)
             if op.is_ovf() and ops[i + 1].getopnum() == rop.GUARD_NO_OVERFLOW:
-                sb.add_potential(op)
+                sb.add_pure_op(op)
         for i in self.call_pure_positions:
             yyy
             op = ops[i]
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -1,55 +1,119 @@
 
 from rpython.jit.metainterp.resoperation import ResOperation, OpHelpers,\
-     AbstractInputArg
+     AbstractInputArg, rop
 from rpython.jit.metainterp.history import Const
+from rpython.jit.metainterp.optimizeopt import info
 
+class InputArgPlaceholder(AbstractInputArg):
+    def __repr__(self):
+        return "placeholder"
 
+placeholder = InputArgPlaceholder()
 
 class ShortBoxes(object):
     def __init__(self):
-        self.potential_ops = []
-        self.produced_short_boxes = {}
-        self.extra_same_as = []
+        self.potential_ops = {}
+        #self.extra_same_as = []
 
     def create_short_boxes(self, optimizer, inputargs):
+        self.produced_short_boxes = {}
+        self.const_short_boxes = []
+        self.short_inputargs = []
         for box in inputargs:
-            self.produced_short_boxes[box] = None
+            renamed = OpHelpers.inputarg_from_tp(box.type)
+            self.produced_short_boxes[box] = (placeholder, renamed)
+            self.short_inputargs.append(renamed)
 
         optimizer.produce_potential_short_preamble_ops(self)
 
-        self.short_boxes = []
-        # short boxes is a list of (op, preamble_op)
-        # where op can be
-        # anything, but the preamble_op has to be either pure
-        # or a heap cache op
+        short_boxes = []
 
-        for op, preamble_op in self.potential_ops:
-            self.produce_short_preamble_op(op, preamble_op)
-        self.produced_short_boxes = None
-        return self.short_boxes
+        for op, getfield_op in self.potential_ops.items():
+            self.add_op_to_short(op, getfield_op)
+        #
+        for op, (getfield_op, preamble_op) in self.produced_short_boxes.iteritems():
+            if getfield_op is not placeholder:
+                short_boxes.append((op, preamble_op))
+        return short_boxes + self.const_short_boxes
 
-    def add_to_short(self, op, short_op):
-        self.short_boxes.append((op, short_op))
-        self.produced_short_boxes[op] = None
+    def produce_short_inputargs(self):
+        return self.short_inputargs
 
-    def produce_short_preamble_op(self, op, preamble_op):
+    def produce_arg(self, op):
+        if op in self.produced_short_boxes:
+            return self.produced_short_boxes[op][1]
+        elif isinstance(op, Const):
+            return op
+        elif op in self.potential_ops:
+            return self.add_op_to_short(op, self.potential_ops[op])
+        else:
+            return None
+
+    def add_op_to_short(self, op, sop):
+        if sop:
+            preamble_arg = self.produce_arg(sop.getarg(0))
+            if preamble_arg is None:
+                return False
+            preamble_op = ResOperation(sop.getopnum(), [preamble_arg],
+                                       descr=sop.getdescr())
+        else:
+            arglist = []
+            for arg in op.getarglist():
+                newarg = self.produce_arg(arg)
+                if newarg is None:
+                    return False
+                arglist.append(newarg)
+            preamble_op = op.copy_and_change(op.getopnum(), args=arglist)
+        self.produced_short_boxes[op] = (sop, preamble_op)
+        return True
+
+    def add_pure_op(self, op):
+        assert not self.potential_ops.get(op, None)
+        self.potential_ops[op] = None
+
+    def add_heap_op(self, op, getfield_op):
+        assert not self.potential_ops.get(op, None)
+        if isinstance(op, Const):
+            self.const_short_boxes.append((op, getfield_op))
+            return # we should not be called from anywhere
+        self.potential_ops[op] = getfield_op
+
+class EmptyInfo(info.AbstractInfo):
+    pass
+
+empty_info = EmptyInfo()
+
+class ShortPreambleBuilder(object):
+    def __init__(self, short_boxes, short_inputargs, exported_infos):
+        self.producable_ops = {}
+        for op, preamble_op in short_boxes:
+            self.producable_ops[op] = preamble_op
+            preamble_op.set_forwarded(exported_infos.get(op, empty_info))
+        self.short = []
+        self.used_boxes = []
+        self.short_inputargs = short_inputargs
+
+    def use_box(self, box):
+        preamble_op = self.producable_ops.get(box, None)
+        if preamble_op is None:
+            return
+        del self.producable_ops[box]
         for arg in preamble_op.getarglist():
             if isinstance(arg, Const):
                 pass
-            elif arg in self.produced_short_boxes:
+            elif arg.get_forwarded() is None:
                 pass
             else:
-                return # can't produce
-        if op in self.produced_short_boxes:
-            opnum = OpHelpers.same_as_for_type(op.type)
-            same_as_op = ResOperation(opnum, [op])
-            self.extra_same_as.append(same_as_op)
-            self.add_to_short(same_as_op, preamble_op)
-        else:
-            self.add_to_short(op, preamble_op)
+                xxx
+        self.short.append(preamble_op)
+        info = preamble_op.get_forwarded()
+        if info is not empty_info:
+            info.make_guards(preamble_op, self.short)
+            if info.is_constant():
+                return
+        self.used_boxes.append(preamble_op)
 
-    def add_potential(self, op, short_preamble_op=None):
-        if short_preamble_op is None:
-            self.potential_ops.append((op, op))
-        else:
-            self.potential_ops.append((op, short_preamble_op))
+    def build_short_preamble(self):
+        label_op = ResOperation(rop.LABEL, self.short_inputargs[:])
+        jump_op = ResOperation(rop.JUMP, self.used_boxes)
+        return [label_op] + self.short + [jump_op]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_short.py b/rpython/jit/metainterp/optimizeopt/test/test_short.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_short.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_short.py
@@ -2,6 +2,7 @@
 """ Short preamble tests
 """
 
+import py
 from rpython.jit.metainterp.resoperation import InputArgInt, ResOperation, rop
 from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes
 from rpython.jit.metainterp.history import AbstractDescr
@@ -16,10 +17,9 @@
     def produce_potential_short_preamble_ops(self, sb):
         for op in self.oplist:
             if isinstance(op, tuple):
-                op, r = op
+                sb.add_heap_op(*op)
             else:
-                op, r = op, op
-            sb.add_potential(op, r)
+                sb.add_pure_op(op)
 
 class TestShortBoxes(object):
     def test_pure_ops(self):
@@ -27,16 +27,16 @@
         i1 = InputArgInt()
         op = ResOperation(rop.INT_ADD, [i0, i1])
         sb = ShortBoxes()
-        sb.create_short_boxes(Opt([op]), [i0, i1])
-        assert sb.short_boxes == [(op, op)]
+        short_boxes = sb.create_short_boxes(Opt([op]), [i0, i1])
+        assert short_boxes == [(op, None)]
 
     def test_pure_ops_does_not_work(self):
         i0 = InputArgInt()
         i1 = InputArgInt()
         op = ResOperation(rop.INT_ADD, [i0, i1])
         sb = ShortBoxes()
-        sb.create_short_boxes(Opt([op]), [i0])
-        assert sb.short_boxes == []
+        short_boxes = sb.create_short_boxes(Opt([op]), [i0])
+        assert short_boxes == []
 
     def test_multiple_similar_ops(self):
         """ This can happen e.g. if heap cache and pure ops produce
@@ -49,6 +49,7 @@
         we store both in short preamble (in case someone else who inlines
         the short preamble does not share them)
         """
+        py.test.skip("l8r")
         i0 = InputArgInt()
         i1 = InputArgInt()
         op = ResOperation(rop.INT_ADD, [i0, i1])
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -6,10 +6,12 @@
 
 from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest,\
      LLtypeMixin
+from rpython.jit.metainterp.optimizeopt.util import equaloplists
 from rpython.jit.metainterp.history import (TreeLoop, ConstInt,
                                             JitCellToken, TargetToken)
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      InputArgRef
+from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortPreambleBuilder
 from rpython.jit.metainterp.compile import LoopCompileData
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN, LEVEL_KNOWNCLASS,\
@@ -51,6 +53,16 @@
         preamble.inputargs = start_state.renamed_inputargs
         return start_state, loop, preamble
 
+    def compare_short(self, short, expected_short):
+        expected_short = self.parse(expected_short,
+                                    postprocess=self.postprocess)
+        remap = {}
+        exp = ([ResOperation(rop.LABEL, expected_short.inputargs)] +
+               expected_short.operations)
+        for k, v in zip(short[0].getarglist(), expected_short.inputargs):
+            remap[v] = k
+        equaloplists(short, exp, remap=remap)
+
 class TestUnroll(BaseTestUnroll):
     def test_simple(self):
         loop = """
@@ -69,7 +81,25 @@
         # we have exported values for i1, which happens to be an inputarg
         assert es.inputarg_mapping[0][1].getint() == 1
         assert isinstance(es.inputarg_mapping[0][1], ConstInt)
-        assert es.short_boxes == []
+        sb = ShortPreambleBuilder(es.short_boxes, es.short_inputargs,
+                                  es.exported_infos)
+        sp = sb.build_short_preamble()
+        exp = """
+        [i0]
+        jump()
+        """
+        self.compare_short(sp, exp)
+        sb = ShortPreambleBuilder(es.short_boxes, es.short_inputargs,
+                                  es.exported_infos)
+        sb.use_box(es.short_boxes[0][0])
+        assert len(es.short_boxes) == 1
+        exp = """
+        [i0]
+        i1 = int_add(i0, 1)
+        guard_value(i1, 1) []
+        jump()
+        """
+        self.compare_short(sb.build_short_preamble(), exp)
 
     def test_not_constant(self):
         loop = """
@@ -82,7 +112,7 @@
         assert isinstance(vs.state[0], NotVirtualStateInfo)
         assert vs.state[0].level == LEVEL_UNKNOWN
         op = preamble.operations[0]
-        assert es.short_boxes == [(op, op)]
+        assert es.short_boxes[0][0] == op
 
     def test_guard_class(self):
         loop = """
@@ -131,7 +161,18 @@
         jump(p0, i0)
         """
         es, loop, preamble = self.optimize(loop)
-        assert es.short_boxes[0][0] == preamble.operations[0]
+        op = preamble.operations[0]
+        assert len(es.short_boxes) == 1
+        assert es.short_boxes[0][0] is op
+        sb = ShortPreambleBuilder(es.short_boxes, es.short_inputargs,
+                                  es.exported_infos)
+        sb.use_box(preamble.operations[0])
+        exp_short = """
+        [p0, i1]
+        i0 = getfield_gc_i(p0, descr=valuedescr)
+        jump(i0)
+        """
+        self.compare_short(sb.build_short_preamble(), exp_short)
 
     def test_int_is_true(self):
         loop = """
@@ -142,14 +183,34 @@
         """
         es, loop, preamble = self.optimize(loop)
         op = preamble.operations[0]
-        assert es.short_boxes == [(op,op)]
+        assert es.short_boxes[0][0] is op
         assert es.exported_infos[op].is_constant()
 
     def test_only_setfield(self):
         loop = """
+        [p0, p1]
+        setfield_gc(p0, 5, descr=valuedescr)
+        setfield_gc(p1, 5, descr=nextdescr)
+        jump(p0, p1)
+        """
+        es, loop, preamble = self.optimize(loop)
+        p0, p1 = preamble.inputargs
+        assert es.short_boxes[0][0].getint() == 5
+        assert es.short_boxes[1][0].getint() == 5
+        assert es.short_boxes[0][1].getarg(0) is p0
+        assert es.short_boxes[1][1].getarg(0) is p1
+
+    def test_double_getfield_plus_pure(self):
+        loop = """
         [p0]
-        setfield_gc(p0, 5, descr=valuedescr)
+        pc = getfield_gc_pure_r(p0, descr=nextdescr)
+        escape_n(p0) # that should flush the caches
+        p1 = getfield_gc_r(pc, descr=nextdescr)
+        i0 = getfield_gc_i(p1, descr=valuedescr)
         jump(p0)
         """
         es, loop, preamble = self.optimize(loop)
-        assert es.short_boxes[0][0] == preamble.operations[0].getarg(1)
+        assert len(es.short_boxes) == 3
+        # both getfields are available as
+        # well as getfield_gc_pure
+        
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,7 +1,8 @@
 import sys
 
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
-from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes
+from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes,\
+     ShortPreambleBuilder
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
@@ -228,22 +229,23 @@
         end_args = [self.get_box_replacement(a) for a in original_label_args]
         virtual_state = self.get_virtual_state(end_args)
         sb = ShortBoxes()
-        sb.create_short_boxes(self.optimizer, [self.get_box_replacement(a)
-                                            for a in start_label.getarglist()])
+        short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs)
         inparg_mapping = [(start_label.getarg(i), end_args[i])
                           for i in range(len(end_args)) if
                           start_label.getarg(i) is not end_args[i]]
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
-        for box, _ in sb.short_boxes:
+        for box, _ in short_boxes:
             if not isinstance(box, Const):
                 infos[box] = self.optimizer.getinfo(box)
         label_args = virtual_state.make_inputargs(end_args, self.optimizer)
+        short_inputargs = sb.produce_short_inputargs()
         self.optimizer._clean_optimization_info(end_args)
         self.optimizer._clean_optimization_info(start_label.getarglist())
         return ExportedState(label_args, inparg_mapping, virtual_state, infos,
-                             sb.short_boxes, renamed_inputargs)
+                             short_boxes, renamed_inputargs,
+                             short_inputargs)
 
 
         inputargs = virtual_state.make_inputargs(jump_args, self.optimizer)
@@ -302,22 +304,41 @@
                 self.optimizer.setinfo_from_preamble(source, info)
         # import the optimizer state, starting from boxes that can be produced
         # by short preamble
-        for op, preamble_op in exported_state.short_boxes:
-            if not isinstance(op, Const):
-                self.ops_to_import[op] = preamble_op
-            if preamble_op.is_always_pure():
-                self.pure(op.getopnum(), PreambleOp(op, preamble_op,
+        self.short_preamble_producer = ShortPreambleProducer()
+        for op, getfield_op in exported_state.short_boxes:
+            if getfield_op is None:
+                optpure = self.optimizer.optpure
+                if optpure is None:
+                    continue
+                self.pure(op.getopnum(), PreambleOp(op, None,
                                 exported_state.exported_infos.get(op, None)))
             else:
-                assert preamble_op.is_getfield()
                 optheap = self.optimizer.optheap
                 if optheap is None:
                     continue
-                opinfo = self.optimizer.ensure_ptr_info_arg0(preamble_op)
+                opinfo = self.optimizer.ensure_ptr_info_arg0(getfield_op)
                 pre_info = exported_state.exported_infos.get(op, None)
-                pop = PreambleOp(op, preamble_op, pre_info)
+                pop = PreambleOp(op, None, pre_info)
                 assert not opinfo.is_virtual()
-                opinfo._fields[preamble_op.getdescr().get_index()] = pop
+                opinfo._fields[getfield_op.getdescr().get_index()] = pop
+            #if not isinstance(op, Const):
+            #    self.ops_to_import[op] = None
+            # XXX think later about the short preamble
+            #if not isinstance(op, Const):
+            #    self.ops_to_import[op] = preamble_op
+            #if preamble_op.is_always_pure():
+            #    self.pure(op.getopnum(), PreambleOp(op, preamble_op,
+            #                    exported_state.exported_infos.get(op, None)))
+            #else:
+            #    assert preamble_op.is_getfield()
+            #    optheap = self.optimizer.optheap
+            #    if optheap is None:
+            #        continue
+            #    opinfo = self.optimizer.ensure_ptr_info_arg0(preamble_op)
+            #    pre_info = exported_state.exported_infos.get(op, None)
+            #    pop = PreambleOp(op, preamble_op, pre_info)
+            #    assert not opinfo.is_virtual()
+            #    opinfo._fields[preamble_op.getdescr().get_index()] = pop
 
         return
         self.inputargs = targetop.getarglist()
@@ -774,13 +795,16 @@
                       of virtuals at this label
     * short boxes - a mapping op -> preamble_op
     * renamed_inputargs - the start label arguments in optimized version
+    * short_inputargs - the renamed inputargs for short preamble
     """
     
     def __init__(self, end_args, inputarg_mapping, virtual_state,
-                 exported_infos, short_boxes, renamed_inputargs):
+                 exported_infos, short_boxes, renamed_inputargs,
+                 short_inputargs):
         self.end_args = end_args
         self.inputarg_mapping = inputarg_mapping
         self.virtual_state = virtual_state
         self.exported_infos = exported_infos
         self.short_boxes = short_boxes
         self.renamed_inputargs = renamed_inputargs
+        self.short_inputargs = short_inputargs
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -12,6 +12,7 @@
     _repr_memo = weakref.WeakKeyDictionary()
     is_info_class = False
     _attrs_ = ()
+    namespace = None
 
     def _get_hash_(self):
         return compute_identity_hash(self)
@@ -185,7 +186,10 @@
             return name
 
     def __repr__(self):
-        return self.repr(self._repr_memo)
+        r = self.repr(self._repr_memo)
+        if self.namespace is not None:
+            return "<" + self.namespace + ">" + r
+        return r
 
     def getopname(self):
         try:

From noreply at buildbot.pypy.org  Fri Jul 17 11:48:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Jul 2015 11:48:19 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: dance around to the music
Message-ID: <20150717094819.CC3941C1334@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78575:8a00142bfe8a
Date: 2015-07-17 11:45 +0200
http://bitbucket.org/pypy/pypy/changeset/8a00142bfe8a/

Log:	dance around to the music

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -30,6 +30,7 @@
 
 class Optimization(object):
     next_optimization = None
+    potential_extra_ops = None
 
     def __init__(self):
         pass # make rpython happy
@@ -302,16 +303,15 @@
     def force_box(self, op):
         op = self.get_box_replacement(op)
         info = op.get_forwarded()
-        if self.optunroll and self.optunroll.ops_to_import:
-            # XXX hack, use stuff on info somehow, a bit on the hard side
-            #     but doable :-)
+        if self.optunroll and self.optunroll.potential_extra_ops:
+            # XXX hack
             try:
-                self.optunroll.ops_to_import[op]
+                 preamble_op = self.optunroll.potential_extra_ops.popitem(op)
             except KeyError:
                 pass
             else:
-                self.optunroll.extra_label_args.append(op)
-                del self.optunroll.ops_to_import[op]
+                sb = self.optunroll.short_preamble_producer
+                sb.add_preamble_op(op, preamble_op)
         if info is not None:
             return info.force_box(op, self)
         return op
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -33,8 +33,17 @@
         #
         for op, (getfield_op, preamble_op) in self.produced_short_boxes.iteritems():
             if getfield_op is not placeholder:
-                short_boxes.append((op, preamble_op))
-        return short_boxes + self.const_short_boxes
+                if getfield_op is None:
+                    short_boxes.append((op, None, preamble_op))
+                else:
+                    short_boxes.append((op, getfield_op.getarg(0), preamble_op))
+        for op, getfield_op in self.const_short_boxes:
+            preamble_arg = self.produce_arg(getfield_op.getarg(0))
+            if preamble_arg is not None:
+                short_boxes.append((op, getfield_op.getarg(0),
+                    getfield_op.copy_and_change(getfield_op.getopnum(),
+                                                [preamble_arg])))
+        return short_boxes
 
     def produce_short_inputargs(self):
         return self.short_inputargs
@@ -53,7 +62,7 @@
         if sop:
             preamble_arg = self.produce_arg(sop.getarg(0))
             if preamble_arg is None:
-                return False
+                return None
             preamble_op = ResOperation(sop.getopnum(), [preamble_arg],
                                        descr=sop.getdescr())
         else:
@@ -61,11 +70,11 @@
             for arg in op.getarglist():
                 newarg = self.produce_arg(arg)
                 if newarg is None:
-                    return False
+                    return None
                 arglist.append(newarg)
             preamble_op = op.copy_and_change(op.getopnum(), args=arglist)
         self.produced_short_boxes[op] = (sop, preamble_op)
-        return True
+        return preamble_op
 
     def add_pure_op(self, op):
         assert not self.potential_ops.get(op, None)
@@ -84,16 +93,24 @@
 empty_info = EmptyInfo()
 
 class ShortPreambleBuilder(object):
-    def __init__(self, short_boxes, short_inputargs, exported_infos):
+    def __init__(self, short_boxes, short_inputargs, exported_infos,
+                 optimizer=None):
         self.producable_ops = {}
-        for op, preamble_op in short_boxes:
+        for op, sop, preamble_op in short_boxes:
             self.producable_ops[op] = preamble_op
-            preamble_op.set_forwarded(exported_infos.get(op, empty_info))
+            if isinstance(op, Const):
+                info = optimizer.getinfo(op)
+            else:
+                info = exported_infos[op]
+                if info is None:
+                    info = empty_info
+            preamble_op.set_forwarded(info)
         self.short = []
         self.used_boxes = []
+        self.short_preamble_jump = []
         self.short_inputargs = short_inputargs
 
-    def use_box(self, box):
+    def use_box(self, box, optimizer=None):
         preamble_op = self.producable_ops.get(box, None)
         if preamble_op is None:
             return
@@ -104,16 +121,25 @@
             elif arg.get_forwarded() is None:
                 pass
             else:
-                xxx
+                self.short.append(arg)
+                info = arg.get_forwarded()
+                if info is not empty_info:
+                    info.make_guards(arg, self.short)
+                arg.set_forwarded(None)
+                self.force_info_from(arg)
         self.short.append(preamble_op)
         info = preamble_op.get_forwarded()
+        preamble_op.set_forwarded(None)
         if info is not empty_info:
             info.make_guards(preamble_op, self.short)
-            if info.is_constant():
-                return
-        self.used_boxes.append(preamble_op)
+        if optimizer is not None:
+            optimizer.setinfo_from_preamble(box, info)
+
+    def add_preamble_op(self, op, preamble_op):
+        self.used_boxes.append(op)
+        self.short_preamble_jump.append(preamble_op)        
 
     def build_short_preamble(self):
         label_op = ResOperation(rop.LABEL, self.short_inputargs[:])
-        jump_op = ResOperation(rop.JUMP, self.used_boxes)
+        jump_op = ResOperation(rop.JUMP, self.short_preamble_jump)
         return [label_op] + self.short + [jump_op]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -191,7 +191,7 @@
         short = """
         [i2]
         p3 = cast_int_to_ptr(i2)
-        #jump(i2) <- think about it
+        jump()
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
@@ -450,7 +450,7 @@
         [i0]
         i1 = int_is_true(i0)
         guard_value(i1, 1) []
-        #jump(i0) <- xxx
+        jump()
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
@@ -851,7 +851,8 @@
         p2sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        jump(i1, p2, p2sub)
+        i3 = same_as(i1)
+        jump(i1, p2, i3)
         """
         expected = """
         [i1, p2, i3]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -214,3 +214,17 @@
         # both getfields are available as
         # well as getfield_gc_pure
         
+    def test_p123_anti_nested(self):
+        loop = """
+        [i1, p2, p3]
+        p3sub = getfield_gc_r(p3, descr=nextdescr)
+        i3 = getfield_gc_i(p3sub, descr=valuedescr)
+        escape_n(i3)
+        p1 = new_with_vtable(descr=nodesize)
+        p2sub = new_with_vtable(descr=nodesize2)
+        setfield_gc(p2sub, i1, descr=valuedescr)
+        setfield_gc(p2, p2sub, descr=nextdescr)
+        jump(i1, p1, p2)
+        """
+        es, loop, preamble = self.optimize(loop)
+        xxx
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -27,10 +27,9 @@
     See force_op_from_preamble for details how the extra things are put.
     """
     
-    def __init__(self, op, preamble_op, info):
+    def __init__(self, op, preamble_op):
         self.op = op
         self.preamble_op = preamble_op
-        self.info = info
 
     def getarg(self, i):
         return self.op.getarg(i)
@@ -42,10 +41,8 @@
 class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
         op = preamble_op.op
-        self.optunroll.short.append(preamble_op.preamble_op)
-        if preamble_op.info:
-            self.setinfo_from_preamble(op, preamble_op.info)
-            preamble_op.info.make_guards(op, self.optunroll.short)
+        self.optunroll.short_preamble_producer.use_box(op, self)
+        self.potential_extra_ops[op] = preamble_op
         return op
 
     def setinfo_from_preamble(self, op, preamble_info):
@@ -108,29 +105,36 @@
         return exported_state, self.optimizer._newoperations
 
     def optimize_peeled_loop(self, start_label, end_jump, ops, state):
-        self.short = []
-        self.extra_label_args = []
         self._check_no_forwarding([[start_label, end_jump], ops])
         self.import_state(start_label, state)
+        self.optimizer.potential_extra_ops = {}
         self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops,
                                              rename_inputargs=False)
         jump_args = [self.get_box_replacement(op)
                      for op in end_jump.getarglist()]
-        args_from_extras = [self.get_box_replacement(op) for op in
-                            self.extra_label_args]
         jump_args = state.virtual_state.make_inputargs(jump_args,
-                    self.optimizer, force_boxes=True) + args_from_extras
+                                    self.optimizer, force_boxes=True)
+        jump_args += self.inline_short_preamble(jump_args)
         
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer._newoperations.append(jump_op)
-        return (UnrollInfo(self.make_short_preamble(start_label.getarglist()),
-                           self.extra_label_args),
+        return (UnrollInfo(self.short_preamble_producer.build_short_preamble(),
+                           self.short_preamble_producer.used_boxes),
                 self.optimizer._newoperations)
 
-    def make_short_preamble(self, args):
-        label = ResOperation(rop.LABEL, args)
-        short = [label] + self.short
-        return short
+    def inline_short_preamble(self, jump_args):
+        sb = self.short_preamble_producer
+        assert len(sb.short_inputargs) == len(jump_args)
+        for i in range(len(jump_args)):
+            sb.short_inputargs[i].set_forwarded(None)
+            self.make_equal_to(sb.short_inputargs[i], jump_args[i])
+        for op in sb.short:
+            self.optimizer.send_extra_operation(op)
+        res = [self.optimizer.get_box_replacement(op) for op in
+                sb.short_preamble_jump]
+        for op in sb.short_inputargs:
+            op.set_forwarded(None)
+        return res
 
     def random_garbage(self):
         # WTF is the rest of this function
@@ -236,7 +240,7 @@
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
-        for box, _ in short_boxes:
+        for box, _, _ in short_boxes:
             if not isinstance(box, Const):
                 infos[box] = self.optimizer.getinfo(box)
         label_args = virtual_state.make_inputargs(end_args, self.optimizer)
@@ -295,7 +299,8 @@
     def import_state(self, targetop, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
-        self.ops_to_import = {}
+        #self.ops_to_import = {}
+        virtual_state = exported_state.virtual_state
         for source, target in exported_state.inputarg_mapping:
             if source is not target:
                 source.set_forwarded(target)
@@ -304,23 +309,27 @@
                 self.optimizer.setinfo_from_preamble(source, info)
         # import the optimizer state, starting from boxes that can be produced
         # by short preamble
-        self.short_preamble_producer = ShortPreambleProducer()
-        for op, getfield_op in exported_state.short_boxes:
-            if getfield_op is None:
+        short_inpargs = virtual_state.make_inputargs(
+            exported_state.short_inputargs, None)
+        self.short_preamble_producer = ShortPreambleBuilder(
+            exported_state.short_boxes, short_inpargs,
+            exported_state.exported_infos, self.optimizer)
+        for op, structop, preamble_op in exported_state.short_boxes:
+            if structop is None:
                 optpure = self.optimizer.optpure
                 if optpure is None:
                     continue
-                self.pure(op.getopnum(), PreambleOp(op, None,
-                                exported_state.exported_infos.get(op, None)))
+                self.pure(op.getopnum(), PreambleOp(op, preamble_op))
             else:
                 optheap = self.optimizer.optheap
                 if optheap is None:
                     continue
-                opinfo = self.optimizer.ensure_ptr_info_arg0(getfield_op)
-                pre_info = exported_state.exported_infos.get(op, None)
-                pop = PreambleOp(op, None, pre_info)
+                g = preamble_op.copy_and_change(preamble_op.getopnum(),
+                                                args=[structop])
+                opinfo = self.optimizer.ensure_ptr_info_arg0(g)
+                pop = PreambleOp(op, preamble_op)
                 assert not opinfo.is_virtual()
-                opinfo._fields[getfield_op.getdescr().get_index()] = pop
+                opinfo._fields[preamble_op.getdescr().get_index()] = pop
             #if not isinstance(op, Const):
             #    self.ops_to_import[op] = None
             # XXX think later about the short preamble
@@ -776,7 +785,7 @@
     """ A state after optimizing the peeled loop, contains the following:
 
     * short_preamble - list of operations that go into short preamble
-    * extra_label_args - list of extra operations that go into the label
+    * extra_label_args - additional things to put in the label
     """
     def __init__(self, short_preamble, extra_label_args):
         self.short_preamble = short_preamble
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -133,10 +133,11 @@
         raise NotImplementedError
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
-        info = optimizer.getptrinfo(box)
-        box = optimizer.get_box_replacement(box)
-        if info is None or not info.is_virtual():
-            raise BadVirtualState()
+        if optimizer is not None:
+            info = optimizer.getptrinfo(box)
+            box = optimizer.get_box_replacement(box)
+            if info is None or not info.is_virtual():
+                raise BadVirtualState()
         for i in range(len(self.fielddescrs)):
             state = self.fieldstate[i]
             if not state:
@@ -427,14 +428,15 @@
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals
-        box = optimizer.get_box_replacement(box)
-        if box.type == 'r':
-            info = optimizer.getptrinfo(box)
-            if info and info.is_virtual():
-                if force_boxes:
-                    info.force_box(box, optimizer)
-                else:
-                    raise BadVirtualState
+        if optimizer is not None:
+            box = optimizer.get_box_replacement(box)
+            if box.type == 'r':
+                info = optimizer.getptrinfo(box)
+                if info and info.is_virtual():
+                    if force_boxes:
+                        info.force_box(box, optimizer)
+                    else:
+                        raise BadVirtualState
         boxes[self.position_in_notvirtuals] = box
 
     def _enum(self, virtual_state):

From noreply at buildbot.pypy.org  Fri Jul 17 11:48:21 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Jul 2015 11:48:21 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: detect recursion
Message-ID: <20150717094821.198061C1334@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult-unroll
Changeset: r78576:677da62dbd4f
Date: 2015-07-17 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/677da62dbd4f/

Log:	detect recursion

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -27,6 +27,7 @@
         optimizer.produce_potential_short_preamble_ops(self)
 
         short_boxes = []
+        self.boxes_in_production = {}
 
         for op, getfield_op in self.potential_ops.items():
             self.add_op_to_short(op, getfield_op)
@@ -51,6 +52,8 @@
     def produce_arg(self, op):
         if op in self.produced_short_boxes:
             return self.produced_short_boxes[op][1]
+        elif op in self.boxes_in_production:
+            return None
         elif isinstance(op, Const):
             return op
         elif op in self.potential_ops:
@@ -59,21 +62,25 @@
             return None
 
     def add_op_to_short(self, op, sop):
-        if sop:
-            preamble_arg = self.produce_arg(sop.getarg(0))
-            if preamble_arg is None:
-                return None
-            preamble_op = ResOperation(sop.getopnum(), [preamble_arg],
-                                       descr=sop.getdescr())
-        else:
-            arglist = []
-            for arg in op.getarglist():
-                newarg = self.produce_arg(arg)
-                if newarg is None:
+        self.boxes_in_production[op] = None
+        try:
+            if sop:
+                preamble_arg = self.produce_arg(sop.getarg(0))
+                if preamble_arg is None:
                     return None
-                arglist.append(newarg)
-            preamble_op = op.copy_and_change(op.getopnum(), args=arglist)
-        self.produced_short_boxes[op] = (sop, preamble_op)
+                preamble_op = ResOperation(sop.getopnum(), [preamble_arg],
+                                           descr=sop.getdescr())
+            else:
+                arglist = []
+                for arg in op.getarglist():
+                    newarg = self.produce_arg(arg)
+                    if newarg is None:
+                        return None
+                    arglist.append(newarg)
+                preamble_op = op.copy_and_change(op.getopnum(), args=arglist)
+            self.produced_short_boxes[op] = (sop, preamble_op)
+        finally:
+            del self.boxes_in_production[op]
         return preamble_op
 
     def add_pure_op(self, op):

From noreply at buildbot.pypy.org  Fri Jul 17 11:52:28 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 11:52:28 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: hg merge py3k
Message-ID: <20150717095228.49B641C1334@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78577:95c0538a5119
Date: 2015-07-17 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/95c0538a5119/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 4669 lines

diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -1,6 +1,6 @@
 # Minimal tests for dis module
 
-from test.support import run_unittest, captured_stdout
+from test.support import run_unittest, captured_stdout, check_impl_detail
 import difflib
 import unittest
 import sys
@@ -253,7 +253,8 @@
     def test_disassemble_str(self):
         self.do_disassembly_test(expr_str, dis_expr_str)
         self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str)
-        self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str)
+        if check_impl_detail():
+            self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str)
 
     def test_disassemble_bytes(self):
         self.do_disassembly_test(_f.__code__.co_code, dis_f_co_code)
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.1.2
+Version: 1.2.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.1.2"
-__version_info__ = (1, 1, 2)
+__version__ = "1.2.0"
+__version_info__ = (1, 2, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -236,6 +236,30 @@
             cdecl = self._typeof(cdecl)
         return self._backend.newp(cdecl, init)
 
+    def new_allocator(self, alloc=None, free=None,
+                      should_clear_after_alloc=True):
+        """Return a new allocator, i.e. a function that behaves like ffi.new()
+        but uses the provided low-level 'alloc' and 'free' functions.
+
+        'alloc' is called with the size as argument.  If it returns NULL, a
+        MemoryError is raised.  'free' is called with the result of 'alloc'
+        as argument.  Both can be either Python function or directly C
+        functions.  If 'free' is None, then no free function is called.
+        If both 'alloc' and 'free' are None, the default is used.
+
+        If 'should_clear_after_alloc' is set to False, then the memory
+        returned by 'alloc' is assumed to be already cleared (or you are
+        fine with garbage); otherwise CFFI will clear it.
+        """
+        compiled_ffi = self._backend.FFI()
+        allocator = compiled_ffi.new_allocator(alloc, free,
+                                               should_clear_after_alloc)
+        def allocate(cdecl, init=None):
+            if isinstance(cdecl, basestring):
+                cdecl = self._typeof(cdecl)
+            return allocator(cdecl, init)
+        return allocate
+
     def cast(self, cdecl, source):
         """Similar to a C cast: returns an instance of the named C
         type initialized with the given 'source'.  The source is
@@ -286,7 +310,7 @@
         """
         return self._backend.from_buffer(self.BCharA, python_buffer)
 
-    def callback(self, cdecl, python_callable=None, error=None):
+    def callback(self, cdecl, python_callable=None, error=None, onerror=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
         The callback invokes the specified 'python_callable' (which may
@@ -298,7 +322,8 @@
             if not callable(python_callable):
                 raise TypeError("the 'python_callable' argument "
                                 "is not callable")
-            return self._backend.callback(cdecl, python_callable, error)
+            return self._backend.callback(cdecl, python_callable,
+                                          error, onerror)
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl, consider_function_as_funcptr=True)
         if python_callable is None:
@@ -327,6 +352,13 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
+        try:
+            gcp = self._backend.gcp
+        except AttributeError:
+            pass
+        else:
+            return gcp(cdata, destructor)
+        #
         with self._lock:
             try:
                 gc_weakrefs = self.gc_weakrefs
@@ -428,6 +460,8 @@
             raise TypeError("ffi.include() expects an argument that is also of"
                             " type cffi.FFI, not %r" % (
                                 type(ffi_to_include).__name__,))
+        if ffi_to_include is self:
+            raise ValueError("self.include(self)")
         with ffi_to_include._lock:
             with self._lock:
                 self._parser.include(ffi_to_include._parser)
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -989,7 +989,8 @@
     def cast(self, BType, source):
         return BType._cast_from(source)
 
-    def callback(self, BType, source, error):
+    def callback(self, BType, source, error, onerror):
+        assert onerror is None   # XXX not implemented
         return BType(source, error)
 
     typeof = type
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -633,6 +633,8 @@
 
     def include(self, other):
         for name, tp in other._declarations.items():
+            if name.startswith('anonymous $enum_$'):
+                continue   # fix for test_anonymous_enum_include
             kind = name.split(' ', 1)[0]
             if kind in ('struct', 'union', 'enum', 'anonymous'):
                 self._declare(name, tp, included=True)
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -35,9 +35,6 @@
     def is_integer_type(self):
         return False
 
-    def sizeof_enabled(self):
-        return False
-
     def get_cached_btype(self, ffi, finishlist, can_delay=False):
         try:
             BType = ffi._cached_btypes[self]
@@ -80,8 +77,7 @@
 
 
 class BasePrimitiveType(BaseType):
-    def sizeof_enabled(self):
-        return True
+    pass
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -205,9 +201,6 @@
 class FunctionPtrType(BaseFunctionType):
     _base_pattern = '(*&)(%s)'
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         result = self.result.get_cached_btype(ffi, finishlist)
         args = []
@@ -233,9 +226,6 @@
             extra = self._base_pattern
         self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra)
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True)
         return global_cache(self, ffi, 'new_pointer_type', BItem)
@@ -276,9 +266,6 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def sizeof_enabled(self):
-        return self.item.sizeof_enabled() and self.length is not None
-
     def resolve_length(self, newlength):
         return ArrayType(self.item, newlength)
 
@@ -433,9 +420,6 @@
             from . import ffiplatform
             raise ffiplatform.VerificationMissing(self._get_c_name())
 
-    def sizeof_enabled(self):
-        return self.fldtypes is not None
-
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
@@ -464,9 +448,6 @@
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
-    def sizeof_enabled(self):
-        return True     # not strictly true, but external enums are obscure
-
     def force_the_name(self, forcename):
         StructOrUnionOrEnum.force_the_name(self, forcename)
         if self.forcename is None:
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -26,6 +26,7 @@
 #define _CFFI_OP_GLOBAL_VAR     33
 #define _CFFI_OP_DLOPEN_FUNC    35
 #define _CFFI_OP_DLOPEN_CONST   37
+#define _CFFI_OP_GLOBAL_VAR_F   39
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -981,10 +981,6 @@
         if not self.target_is_python and tp.is_integer_type():
             type_op = CffiOp(OP_CONSTANT_INT, -1)
         else:
-            if not tp.sizeof_enabled():
-                raise ffiplatform.VerificationError(
-                    "constant '%s' is of type '%s', whose size is not known"
-                    % (name, tp._get_c_name()))
             if self.target_is_python:
                 const_kind = OP_DLOPEN_CONST
             else:
@@ -1069,18 +1065,36 @@
         self._do_collect_type(self._global_type(tp, name))
 
     def _generate_cpy_variable_decl(self, tp, name):
-        pass
+        prnt = self._prnt
+        tp = self._global_type(tp, name)
+        if isinstance(tp, model.ArrayType) and tp.length is None:
+            tp = tp.item
+            ampersand = ''
+        else:
+            ampersand = '&'
+        # This code assumes that casts from "tp *" to "void *" is a
+        # no-op, i.e. a function that returns a "tp *" can be called
+        # as if it returned a "void *".  This should be generally true
+        # on any modern machine.  The only exception to that rule (on
+        # uncommon architectures, and as far as I can tell) might be
+        # if 'tp' were a function type, but that is not possible here.
+        # (If 'tp' is a function _pointer_ type, then casts from "fn_t
+        # **" to "void *" are again no-ops, as far as I can tell.)
+        prnt('static ' + tp.get_c_name('*_cffi_var_%s(void)' % (name,)))
+        prnt('{')
+        prnt('  return %s(%s);' % (ampersand, name))
+        prnt('}')
+        prnt()
 
     def _generate_cpy_variable_ctx(self, tp, name):
         tp = self._global_type(tp, name)
         type_index = self._typesdict[tp]
-        type_op = CffiOp(OP_GLOBAL_VAR, type_index)
-        if tp.sizeof_enabled():
-            size = "sizeof(%s)" % (name,)
+        if self.target_is_python:
+            op = OP_GLOBAL_VAR
         else:
-            size = 0
+            op = OP_GLOBAL_VAR_F
         self._lsts["global"].append(
-            GlobalExpr(name, '&%s' % name, type_op, size))
+            GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index)))
 
     # ----------
     # emitting the opcodes for individual types
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -24,14 +24,14 @@
     "_codecs", "atexit", "gc", "_weakref", "marshal", "errno", "imp",
     "itertools", "math", "cmath", "_sre", "_pickle_support", "operator",
     "parser", "symbol", "token", "_ast", "_random", "__pypy__",
-    "_string", "_testing"
+    "_string", "_testing", "time"
 ])
 
 
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_minimal_curses",
     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -70,6 +70,20 @@
 .. _`use virtualenv (as documented here)`: getting-started.html#installing-using-virtualenv
 
 
+Module xyz does not work in the sandboxed PyPy?
+-----------------------------------------------
+
+You cannot import *any* extension module in a `sandboxed PyPy`_,
+sorry.  Even the built-in modules available are very limited.
+Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
+it is only a proof of concept.  It seriously requires someone working
+on it.  Before this occurs, it can only be used it for "pure Python"
+examples: programs that import mostly nothing (or only pure Python
+modules, recursively).
+
+.. _`sandboxed PyPy`: sandbox.html
+
+
 .. _`See below.`:
 
 Do CPython Extension modules work with PyPy?
diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -103,12 +103,15 @@
 Howto
 -----
 
-In pypy/goal::
+Grab a copy of the pypy repository_.  In the directory pypy/goal, run::
 
    ../../rpython/bin/rpython -O2 --sandbox targetpypystandalone.py
 
 If you don't have a regular PyPy installed, you should, because it's
-faster to translate, but you can also run ``python translate.py`` instead.
+faster to translate; but you can also run the same line with ``python``
+in front.
+
+.. _repository: https://bitbucket.org/pypy/pypy
 
 
 To run it, use the tools in the pypy/sandbox directory::
@@ -136,8 +139,6 @@
 Not all operations are supported; e.g. if you type os.readlink('...'),
 the controller crashes with an exception and the subprocess is killed.
 Other operations make the subprocess die directly with a "Fatal RPython
-error".  None of this is a security hole; it just means that if you try
-to run some random program, it risks getting killed depending on the
-Python built-in functions it tries to call.  This is a matter of the
-sandboxing layer being incomplete so far, but it should not really be
-a problem in practice.
+error".  None of this is a security hole.  More importantly, *most other
+built-in modules are not enabled.  Please read all the warnings in this
+page before complaining about this.  Contributions welcome.*
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -20,5 +20,20 @@
 .. branch: run-create_cffi_imports
 
 Build cffi import libraries as part of translation by monkey-patching an 
-aditional task into translation
+additional task into translation
 
+.. branch: int-float-list-strategy
+
+Use a compact strategy for Python lists that mix integers and floats,
+at least if the integers fit inside 32 bits.  These lists are now
+stored as an array of floats, like lists that contain only floats; the
+difference is that integers are stored as tagged NaNs.  (This should
+have no visible effect!  After ``lst = [42, 42.5]``, the value of
+``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
+
+.. branch: cffi-callback-onerror
+.. branch: cffi-new-allocator
+
+.. branch: unicode-dtype
+
+Partial implementation of unicode dtype and unicode scalars.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -303,7 +303,12 @@
         options = make_dict(config)
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
+        if config.objspace.usemodules._cffi_backend:
+            self.hack_for_cffi_modules(driver)
 
+        return self.get_entry_point(config)
+    
+    def hack_for_cffi_modules(self, driver):
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_c to build_cffi_imports
         # this should probably get cleaned up and merged with driver.create_exe
@@ -342,8 +347,6 @@
         driver.default_goal = 'build_cffi_imports'
         # HACKHACKHACK end
 
-        return self.get_entry_point(config)
-
     def jitpolicy(self, driver):
         from pypy.module.pypyjit.policy import PyPyJitPolicy
         from pypy.module.pypyjit.hooks import pypy_hooks
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -277,7 +277,8 @@
                                w_t, w_v, w_tb],
                 """(where, objrepr, extra_line, t, v, tb):
                     import sys, traceback
-                    sys.stderr.write('From %s%s:\\n' % (where, objrepr))
+                    if where or objrepr:
+                        sys.stderr.write('From %s%s:\\n' % (where, objrepr))
                     if extra_line:
                         sys.stderr.write(extra_line)
                     traceback.print_exception(t, v, tb)
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -15,7 +15,10 @@
         self.running = False
 
     def descr__repr__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = '<finished>'
+        else:
+            code_name = self.pycode.co_name
         addrstring = self.getaddrstring(space)
         return space.wrap("<generator object %s at 0x%s>" %
                           (code_name, addrstring))
@@ -45,6 +48,8 @@
         w_framestate, w_running = args_w
         if space.is_w(w_framestate, space.w_None):
             self.frame = None
+            self.space = space
+            self.pycode = None
         else:
             frame = instantiate(space.FrameClass)   # XXX fish
             frame.descr__setstate__(space, w_framestate)
@@ -62,9 +67,10 @@
 
     def send_ex(self, w_arg, operr=None):
         pycode = self.pycode
-        if jit.we_are_jitted() and should_not_inline(pycode):
-            generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
-                                                  operr=operr, pycode=pycode)
+        if pycode is not None:
+            if jit.we_are_jitted() and should_not_inline(pycode):
+                generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
+                                                    operr=operr, pycode=pycode)
         return self._send_ex(w_arg, operr)
 
     def _send_ex(self, w_arg, operr):
@@ -233,7 +239,10 @@
         return self.pycode
 
     def descr__name__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = '<finished>'
+        else:
+            code_name = self.pycode.co_name
         return space.wrap(code_name)
 
     # Results can be either an RPython list of W_Root, or it can be an
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -530,6 +530,22 @@
             del sys.modules['mod']
 
 
+    def test_pickle_generator_crash(self):
+        import pickle
+
+        def f():
+            yield 0
+
+        x = f()
+        next(x)
+        try:
+            next(x)
+        except StopIteration:
+            y = pickle.loads(pickle.dumps(x))
+        assert 'finished' in y.__name__
+        assert 'finished' in repr(y)
+        assert y.gi_code is None
+
 class AppTestGeneratorCloning:
 
     def setup_class(cls):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.2"
+VERSION = "1.2.0"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -0,0 +1,86 @@
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+class W_Allocator(W_Root):
+    _immutable_ = True
+
+    def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc):
+        self.ffi = ffi    # may be None
+        self.w_alloc = w_alloc
+        self.w_free = w_free
+        self.should_clear_after_alloc = should_clear_after_alloc
+
+    def allocate(self, space, datasize, ctype, length=-1):
+        from pypy.module._cffi_backend import cdataobj, ctypeptr
+        if self.w_alloc is None:
+            if self.should_clear_after_alloc:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=True)
+            else:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=False)
+            return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+        else:
+            w_raw_cdata = space.call_function(self.w_alloc,
+                                              space.wrap(datasize))
+            if not isinstance(w_raw_cdata, cdataobj.W_CData):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata object (got %T)",
+                            w_raw_cdata)
+            if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata pointer, not '%s'",
+                            w_raw_cdata.ctype.name)
+            #
+            ptr = w_raw_cdata.unsafe_escaping_ptr()
+            if not ptr:
+                raise oefmt(space.w_MemoryError, "alloc() returned NULL")
+            #
+            if self.should_clear_after_alloc:
+                rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0,
+                              rffi.cast(rffi.SIZE_T, datasize))
+            #
+            if self.w_free is None:
+                # use this class which does not have a __del__, but still
+                # keeps alive w_raw_cdata
+                res = cdataobj.W_CDataNewNonStdNoFree(space, ptr, ctype, length)
+            else:
+                res = cdataobj.W_CDataNewNonStdFree(space, ptr, ctype, length)
+                res.w_free = self.w_free
+            res.w_raw_cdata = w_raw_cdata
+            return res
+
+    @unwrap_spec(w_init=WrappedDefault(None))
+    def descr_call(self, space, w_arg, w_init):
+        ffi = self.ffi
+        assert ffi is not None
+        w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING | ffi.ACCEPT_CTYPE)
+        return w_ctype.newp(w_init, self)
+
+
+W_Allocator.typedef = TypeDef(
+        'FFIAllocator',
+        __call__ = interp2app(W_Allocator.descr_call),
+        )
+W_Allocator.typedef.acceptable_as_base_class = False
+
+
+def new_allocator(ffi, w_alloc, w_free, should_clear_after_alloc):
+    space = ffi.space
+    if space.is_none(w_alloc):
+        w_alloc = None
+    if space.is_none(w_free):
+        w_free = None
+    if w_alloc is None and w_free is not None:
+        raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'")
+    alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc))
+    return space.wrap(alloc)
+
+
+default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True)
+nonzero_allocator = W_Allocator(None, None, None,should_clear_after_alloc=False)
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -22,8 +22,9 @@
 class W_CDataCallback(W_CData):
     #_immutable_fields_ = ...
     ll_error = lltype.nullptr(rffi.CCHARP.TO)
+    w_onerror = None
 
-    def __init__(self, space, ctype, w_callable, w_error):
+    def __init__(self, space, ctype, w_callable, w_error, w_onerror):
         raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc())
         W_CData.__init__(self, space, raw_closure, ctype)
         #
@@ -31,6 +32,12 @@
             raise oefmt(space.w_TypeError,
                         "expected a callable object, not %T", w_callable)
         self.w_callable = w_callable
+        if not space.is_none(w_onerror):
+            if not space.is_true(space.callable(w_onerror)):
+                raise oefmt(space.w_TypeError,
+                            "expected a callable object for 'onerror', not %T",
+                            w_onerror)
+            self.w_onerror = w_onerror
         #
         fresult = self.getfunctype().ctitem
         size = fresult.size
@@ -161,6 +168,29 @@
 STDERR = 2
 
 
+ at jit.dont_look_inside
+def _handle_applevel_exception(space, callback, e, ll_res, extra_line):
+    callback.write_error_return_value(ll_res)
+    if callback.w_onerror is None:
+        callback.print_error(e, extra_line)
+    else:
+        try:
+            e.normalize_exception(space)
+            w_t = e.w_type
+            w_v = e.get_w_value(space)
+            w_tb = space.wrap(e.get_traceback())
+            w_res = space.call_function(callback.w_onerror,
+                                        w_t, w_v, w_tb)
+            if not space.is_none(w_res):
+                callback.convert_result(ll_res, w_res)
+        except OperationError, e2:
+            # double exception! print a double-traceback...
+            callback.print_error(e, extra_line)    # original traceback
+            e2.write_unraisable(space, '', with_traceback=True,
+                                extra_line="\nDuring the call to 'onerror', "
+                                           "another exception occurred:\n\n")
+
+
 @jit.jit_callback("CFFI")
 def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
     """ Callback specification.
@@ -178,7 +208,7 @@
         try:
             os.write(STDERR, "SystemError: invoking a callback "
                              "that was already freed\n")
-        except OSError:
+        except:
             pass
         # In this case, we don't even know how big ll_res is.  Let's assume
         # it is just a 'ffi_arg', and store 0 there.
@@ -195,9 +225,7 @@
             extra_line = "Trying to convert the result back to C:\n"
             callback.convert_result(ll_res, w_res)
         except OperationError, e:
-            # got an app-level exception
-            callback.print_error(e, extra_line)
-            callback.write_error_return_value(ll_res)
+            _handle_applevel_exception(space, callback, e, ll_res, extra_line)
         #
     except Exception, e:
         # oups! last-level attempt to recover.
@@ -205,7 +233,7 @@
             os.write(STDERR, "SystemError: callback raised ")
             os.write(STDERR, str(e))
             os.write(STDERR, "\n")
-        except OSError:
+        except:
             pass
         callback.write_error_return_value(ll_res)
     if must_leave:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -363,16 +363,19 @@
     def _sizeof(self):
         return self.ctype.size
 
+    def with_gc(self, w_destructor):
+        with self as ptr:
+            return W_CDataGCP(self.space, ptr, self.ctype, self, w_destructor)
+
 
 class W_CDataMem(W_CData):
-    """This is the base class used for cdata objects that own and free
-    their memory.  Used directly by the results of cffi.cast('int', x)
-    or other primitive explicitly-casted types.  It is further subclassed
-    by W_CDataNewOwning."""
+    """This is used only by the results of cffi.cast('int', x)
+    or other primitive explicitly-casted types."""
     _attrs_ = []
 
-    def __init__(self, space, size, ctype):
-        cdata = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw', zero=True)
+    def __init__(self, space, ctype):
+        cdata = lltype.malloc(rffi.CCHARP.TO, ctype.size, flavor='raw',
+                              zero=False)
         W_CData.__init__(self, space, cdata, ctype)
 
     @rgc.must_be_light_finalizer
@@ -380,36 +383,65 @@
         lltype.free(self._ptr, flavor='raw')
 
 
-class W_CDataNewOwning(W_CDataMem):
-    """This is the class used for the cata objects created by newp()."""
-    _attrs_ = []
+class W_CDataNewOwning(W_CData):
+    """This is the abstract base class used for cdata objects created
+    by newp().  They create and free their own memory according to an
+    allocator."""
+
+    # the 'length' is either >= 0 for arrays, or -1 for pointers.
+    _attrs_ = ['length']
+    _immutable_fields_ = ['length']
+
+    def __init__(self, space, cdata, ctype, length=-1):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.length = length
 
     def _repr_extra(self):
         return self._repr_extra_owning()
 
-
-class W_CDataNewOwningLength(W_CDataNewOwning):
-    """Subclass with an explicit length, for allocated instances of
-    the C type 'foo[]'."""
-    _attrs_ = ['length']
-    _immutable_fields_ = ['length']
-
-    def __init__(self, space, size, ctype, length):
-        W_CDataNewOwning.__init__(self, space, size, ctype)
-        self.length = length
-
     def _sizeof(self):
-        from pypy.module._cffi_backend import ctypearray
         ctype = self.ctype
-        assert isinstance(ctype, ctypearray.W_CTypeArray)
-        return self.length * ctype.ctitem.size
+        if self.length >= 0:
+            from pypy.module._cffi_backend import ctypearray
+            assert isinstance(ctype, ctypearray.W_CTypeArray)
+            return self.length * ctype.ctitem.size
+        else:
+            return ctype.size
 
     def get_array_length(self):
         return self.length
 
 
+class W_CDataNewStd(W_CDataNewOwning):
+    """Subclass using the standard allocator, lltype.malloc()/lltype.free()"""
+    _attrs_ = []
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        lltype.free(self._ptr, flavor='raw')
+
+
+class W_CDataNewNonStdNoFree(W_CDataNewOwning):
+    """Subclass using a non-standard allocator, no free()"""
+    _attrs_ = ['w_raw_cdata']
+
+class W_CDataNewNonStdFree(W_CDataNewNonStdNoFree):
+    """Subclass using a non-standard allocator, with a free()"""
+    _attrs_ = ['w_free']
+
+    def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space,
+                                     W_CDataNewNonStdFree.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataNewNonStdFree)
+        self.space.call_function(self.w_free, self.w_raw_cdata)
+
+
 class W_CDataPtrToStructOrUnion(W_CData):
-    """This subclass is used for the pointer returned by new('struct foo').
+    """This subclass is used for the pointer returned by new('struct foo *').
     It has a strong reference to a W_CDataNewOwning that really owns the
     struct, which is the object returned by the app-level expression 'p[0]'.
     But it is not itself owning any memory, although its repr says so;
@@ -483,6 +515,26 @@
             self.length, self.space.type(self.w_keepalive).name)
 
 
+class W_CDataGCP(W_CData):
+    """For ffi.gc()."""
+    _attrs_ = ['w_original_cdata', 'w_destructor']
+    _immutable_fields_ = ['w_original_cdata', 'w_destructor']
+
+    def __init__(self, space, cdata, ctype, w_original_cdata, w_destructor):
+        W_CData.__init__(self, space, cdata, ctype)
+        self.w_original_cdata = w_original_cdata
+        self.w_destructor = w_destructor
+
+    def __del__(self):
+        self.clear_all_weakrefs()
+        self.enqueue_for_destruction(self.space, W_CDataGCP.call_destructor,
+                                     'destructor of ')
+
+    def call_destructor(self):
+        assert isinstance(self, W_CDataGCP)
+        self.space.call_function(self.w_destructor, self.w_original_cdata)
+
+
 W_CData.typedef = TypeDef(
     '_cffi_backend.CData',
     __module__ = '_cffi_backend',
diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -36,7 +36,10 @@
                         self.libname)
         try:
             cdata = dlsym(self.libhandle, name)
+            found = bool(cdata)
         except KeyError:
+            found = False
+        if not found:
             raise oefmt(self.ffi.w_FFIError,
                         "symbol '%s' not found in library '%s'",
                         name, self.libname)
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/pypy/module/_cffi_backend/cgc.py b/pypy/module/_cffi_backend/cgc.py
deleted file mode 100644
--- a/pypy/module/_cffi_backend/cgc.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from rpython.rlib import jit
-
-
- at jit.dont_look_inside
-def gc_weakrefs_build(ffi, w_cdata, w_destructor):
-    from pypy.module._weakref import interp__weakref
-
-    space = ffi.space
-    if ffi.w_gc_wref_remove is None:
-        ffi.gc_wref_dict = {}
-        ffi.w_gc_wref_remove = space.getattr(space.wrap(ffi),
-                                             space.wrap("__gc_wref_remove"))
-
-    w_new_cdata = w_cdata.ctype.cast(w_cdata)
-    assert w_new_cdata is not w_cdata
-
-    w_ref = interp__weakref.make_weakref_with_callback(
-        space,
-        space.gettypefor(interp__weakref.W_Weakref),
-        w_new_cdata,
-        ffi.w_gc_wref_remove)
-
-    ffi.gc_wref_dict[w_ref] = (w_destructor, w_cdata)
-    return w_new_cdata
-
-
-def gc_wref_remove(ffi, w_ref):
-    (w_destructor, w_cdata) = ffi.gc_wref_dict.pop(w_ref)
-    ffi.space.call_function(w_destructor, w_cdata)
diff --git a/pypy/module/_cffi_backend/cglob.py b/pypy/module/_cffi_backend/cglob.py
--- a/pypy/module/_cffi_backend/cglob.py
+++ b/pypy/module/_cffi_backend/cglob.py
@@ -1,24 +1,66 @@
+from pypy.interpreter.error import oefmt
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend import newtype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 
 class W_GlobSupport(W_Root):
-    def __init__(self, space, w_ctype, ptr):
+    _immutable_fields_ = ['w_ctype', 'ptr', 'fetch_addr']
+
+    def __init__(self, space, name, w_ctype, ptr=lltype.nullptr(rffi.CCHARP.TO),
+                 fetch_addr=lltype.nullptr(rffi.VOIDP.TO)):
         self.space = space
+        self.name = name
         self.w_ctype = w_ctype
         self.ptr = ptr
+        self.fetch_addr = fetch_addr
+
+    def fetch_global_var_addr(self):
+        if self.ptr:
+            result = self.ptr
+        else:
+            if not we_are_translated():
+                FNPTR = rffi.CCallback([], rffi.VOIDP)
+                fetch_addr = rffi.cast(FNPTR, self.fetch_addr)
+                result = fetch_addr()
+            else:
+                # careful in translated versions: we need to call fetch_addr,
+                # but in a GIL-releasing way.  The easiest is to invoke a
+                # llexternal() helper.
+                result = pypy__cffi_fetch_var(self.fetch_addr)
+            result = rffi.cast(rffi.CCHARP, result)
+        if not result:
+            from pypy.module._cffi_backend import ffi_obj
+            ffierror = ffi_obj.get_ffi_error(self.space)
+            raise oefmt(ffierror, "global variable '%s' is at address NULL",
+                        self.name)
+        return result
 
     def read_global_var(self):
-        return self.w_ctype.convert_to_object(self.ptr)
+        return self.w_ctype.convert_to_object(self.fetch_global_var_addr())
 
     def write_global_var(self, w_newvalue):
-        self.w_ctype.convert_from_object(self.ptr, w_newvalue)
+        self.w_ctype.convert_from_object(self.fetch_global_var_addr(),
+                                         w_newvalue)
 
     def address(self):
         w_ctypeptr = newtype.new_pointer_type(self.space, self.w_ctype)
-        return W_CData(self.space, self.ptr, w_ctypeptr)
+        return W_CData(self.space, self.fetch_global_var_addr(), w_ctypeptr)
 
 W_GlobSupport.typedef = TypeDef("FFIGlobSupport")
 W_GlobSupport.typedef.acceptable_as_base_class = False
+
+
+eci = ExternalCompilationInfo(post_include_bits=["""
+static void *pypy__cffi_fetch_var(void *fetch_addr) {
+    return ((void*(*)(void))fetch_addr)();
+}
+"""])
+
+pypy__cffi_fetch_var = rffi.llexternal(
+    "pypy__cffi_fetch_var", [rffi.VOIDP], rffi.VOIDP,
+    compilation_info=eci)
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -28,7 +28,7 @@
     def _alignof(self):
         return self.ctitem.alignof()
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         datasize = self.size
         #
@@ -40,12 +40,10 @@
             except OverflowError:
                 raise OperationError(space.w_OverflowError,
                     space.wrap("array size would overflow a ssize_t"))
-            #
-            cdata = cdataobj.W_CDataNewOwningLength(space, datasize,
-                                                    self, length)
+        else:
+            length = self.length
         #
-        else:
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+        cdata = allocator.allocate(space, datasize, self, length)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -55,7 +55,7 @@
     def pack_list_of_items(self, cdata, w_ob):
         return False
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         space = self.space
         raise oefmt(space.w_TypeError,
                     "expected a pointer or array ctype, got '%s'", self.name)
@@ -90,6 +90,16 @@
     def _convert_error(self, expected, w_got):
         space = self.space
         if isinstance(w_got, cdataobj.W_CData):
+            if self.name == w_got.ctype.name:
+                # in case we'd give the error message "initializer for
+                # ctype 'A' must be a pointer to same type, not cdata
+                # 'B'", but with A=B, then give instead a different error
+                # message to try to clear up the confusion
+                return oefmt(space.w_TypeError,
+                             "initializer for ctype '%s' appears indeed to "
+                             "be '%s', but the types are different (check "
+                             "that you are not e.g. mixing up different ffi "
+                             "instances)", self.name, w_got.ctype.name)
             return oefmt(space.w_TypeError,
                          "initializer for ctype '%s' must be a %s, not cdata "
                          "'%s'", self.name, expected, w_got.ctype.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -63,7 +63,7 @@
             value = self._cast_result(value)
         else:
             value = self._cast_generic(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         self.write_raw_integer_data(w_cdata, value)
         return w_cdata
 
@@ -353,7 +353,7 @@
             value = self.cast_unicode(w_ob)
         else:
             value = space.float_w(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(space, self)
         if not isinstance(self, W_CTypePrimitiveLongDouble):
             w_cdata.write_raw_float_data(value)
         else:
@@ -446,7 +446,7 @@
         return self.space.wrap(value)
 
     def convert_to_object(self, cdata):
-        w_cdata = cdataobj.W_CDataMem(self.space, self.size, self)
+        w_cdata = cdataobj.W_CDataMem(self.space, self)
         with w_cdata as ptr:
             self._copy_longdouble(cdata, ptr)
         return w_cdata
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -187,7 +187,7 @@
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
-    def newp(self, w_init):
+    def newp(self, w_init, allocator):
         from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
         space = self.space
         ctitem = self.ctitem
@@ -207,14 +207,14 @@
                     datasize = ctitem.convert_struct_from_object(
                         lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
             #
-            cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
+            cdatastruct = allocator.allocate(space, datasize, ctitem)
             ptr = cdatastruct.unsafe_escaping_ptr()
             cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
                                                        self, cdatastruct)
         else:
             if self.is_char_or_unichar_ptr_or_array():
                 datasize *= 2       # forcefully add a null character
-            cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+            cdata = allocator.allocate(space, datasize, self)
         #
         if not space.is_w(w_init, space.w_None):
             with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -81,10 +81,9 @@
     def copy_and_convert_to_object(self, source):
         space = self.space
         self.check_complete()
-        ob = cdataobj.W_CDataNewOwning(space, self.size, self)
-        with ob as target:
-            misc._raw_memcopy(source, target, self.size)
-        return ob
+        ptr = lltype.malloc(rffi.CCHARP.TO, self.size, flavor='raw', zero=False)
+        misc._raw_memcopy(source, ptr, self.size)
+        return cdataobj.W_CDataNewStd(space, ptr, self)
 
     def typeoffsetof_field(self, fieldname, following):
         self.force_lazy_struct()
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -10,8 +10,8 @@
 from pypy.module._cffi_backend import parse_c_type, realize_c_type
 from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray
 from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle
-from pypy.module._cffi_backend import cbuffer, func, cgc, wrapper
-from pypy.module._cffi_backend import cffi_opcode
+from pypy.module._cffi_backend import cbuffer, func, wrapper
+from pypy.module._cffi_backend import cffi_opcode, allocator
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend.cdataobj import W_CData
 
@@ -44,6 +44,10 @@
 
 
 class W_FFIObject(W_Root):
+    ACCEPT_STRING = ACCEPT_STRING
+    ACCEPT_CTYPE  = ACCEPT_CTYPE
+    ACCEPT_CDATA  = ACCEPT_CDATA
+
     w_gc_wref_remove = None
 
     @jit.dont_look_inside
@@ -276,8 +280,9 @@
 
 
     @unwrap_spec(w_python_callable=WrappedDefault(None),
-                 w_error=WrappedDefault(None))
-    def descr_callback(self, w_cdecl, w_python_callable, w_error):
+                 w_error=WrappedDefault(None),
+                 w_onerror=WrappedDefault(None))
+    def descr_callback(self, w_cdecl, w_python_callable, w_error, w_onerror):
         """\
 Return a callback object or a decorator making such a callback object.
 'cdecl' must name a C function pointer type.  The callback invokes the
@@ -290,14 +295,16 @@
         space = self.space
         if not space.is_none(w_python_callable):
             return ccallback.W_CDataCallback(space, w_ctype,
-                                             w_python_callable, w_error)
+                                             w_python_callable, w_error,
+                                             w_onerror)
         else:
             # decorator mode: returns a single-argument function
-            return space.appexec([w_ctype, w_error],
-            """(ctype, error):
+            return space.appexec([w_ctype, w_error, w_onerror],
+            """(ctype, error, onerror):
                 import _cffi_backend
                 return lambda python_callable: (
-                    _cffi_backend.callback(ctype, python_callable, error))""")
+                    _cffi_backend.callback(ctype, python_callable,
+                                           error, onerror))""")
 
 
     def descr_cast(self, w_arg, w_ob):
@@ -341,10 +348,7 @@
 Later, when this new cdata object is garbage-collected,
 'destructor(old_cdata_object)' will be called."""
         #
-        return cgc.gc_weakrefs_build(self, w_cdata, w_destructor)
-
-    def descr___gc_wref_remove(self, w_ref):
-        return cgc.gc_wref_remove(self, w_ref)
+        return w_cdata.with_gc(w_destructor)
 
 
     @unwrap_spec(replace_with=str)
@@ -411,7 +415,31 @@
 pointer to the memory somewhere else, e.g. into another structure."""
         #
         w_ctype = self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CTYPE)
-        return w_ctype.newp(w_init)
+        return w_ctype.newp(w_init, allocator.default_allocator)
+
+
+    @unwrap_spec(w_alloc=WrappedDefault(None),
+                 w_free=WrappedDefault(None),
+                 should_clear_after_alloc=int)
+    def descr_new_allocator(self, w_alloc, w_free,
+                            should_clear_after_alloc=1):
+        """\
+Return a new allocator, i.e. a function that behaves like ffi.new()
+but uses the provided low-level 'alloc' and 'free' functions.
+
+'alloc' is called with the size as argument.  If it returns NULL, a
+MemoryError is raised.  'free' is called with the result of 'alloc'
+as argument.  Both can be either Python function or directly C
+functions.  If 'free' is None, then no free function is called.
+If both 'alloc' and 'free' are None, the default is used.
+
+If 'should_clear_after_alloc' is set to False, then the memory
+returned by 'alloc' is assumed to be already cleared (or you are
+fine with garbage); otherwise CFFI will clear it.
+        """
+        #
+        return allocator.new_allocator(self, w_alloc, w_free,
+                                       should_clear_after_alloc)
 
 
     def descr_new_handle(self, w_arg):
@@ -539,12 +567,17 @@
 
 
 @jit.dont_look_inside
-def W_FFIObject___new__(space, w_subtype, __args__):
-    r = space.allocate_instance(W_FFIObject, w_subtype)
+def make_plain_ffi_object(space, w_ffitype=None):
+    if w_ffitype is None:
+        w_ffitype = space.gettypefor(W_FFIObject)
+    r = space.allocate_instance(W_FFIObject, w_ffitype)
     # get in 'src_ctx' a NULL which translation doesn't consider to be constant
     src_ctx = rffi.cast(parse_c_type.PCTX, 0)
     r.__init__(space, src_ctx)
-    return space.wrap(r)
+    return r
+
+def W_FFIObject___new__(space, w_subtype, __args__):
+    return space.wrap(make_plain_ffi_object(space, w_subtype))
 
 def make_CData(space):
     return space.gettypefor(W_CData)
@@ -578,7 +611,6 @@
                                      W_FFIObject.set_errno,
                                      doc=W_FFIObject.doc_errno,
                                      cls=W_FFIObject),
-        __gc_wref_remove = interp2app(W_FFIObject.descr___gc_wref_remove),
         addressof   = interp2app(W_FFIObject.descr_addressof),
         alignof     = interp2app(W_FFIObject.descr_alignof),
         buffer      = interp2app(W_FFIObject.descr_buffer),
@@ -592,6 +624,7 @@
         getctype    = interp2app(W_FFIObject.descr_getctype),
         integer_const = interp2app(W_FFIObject.descr_integer_const),
         new         = interp2app(W_FFIObject.descr_new),
+        new_allocator = interp2app(W_FFIObject.descr_new_allocator),
         new_handle  = interp2app(W_FFIObject.descr_new_handle),
         offsetof    = interp2app(W_FFIObject.descr_offsetof),
         sizeof      = interp2app(W_FFIObject.descr_sizeof),
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,13 +1,13 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.module._cffi_backend import ctypeobj, cdataobj
+from pypy.module._cffi_backend import ctypeobj, cdataobj, allocator
 
 
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_init=WrappedDefault(None))
 def newp(space, w_ctype, w_init):
-    return w_ctype.newp(w_init)
+    return w_ctype.newp(w_init, allocator.default_allocator)
 
 # ____________________________________________________________
 
@@ -18,9 +18,9 @@
 # ____________________________________________________________
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
-def callback(space, w_ctype, w_callable, w_error=None):
+def callback(space, w_ctype, w_callable, w_error=None, w_onerror=None):
     from pypy.module._cffi_backend.ccallback import W_CDataCallback
-    return W_CDataCallback(space, w_ctype, w_callable, w_error)
+    return W_CDataCallback(space, w_ctype, w_callable, w_error, w_onerror)
 
 # ____________________________________________________________
 
@@ -105,3 +105,9 @@
                     "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
+
+# ____________________________________________________________
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData)
+def gcp(space, w_cdata, w_destructor):
+    return w_cdata.with_gc(w_destructor)
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -102,6 +102,8 @@
                 #
             elif op == cffi_opcode.OP_GLOBAL_VAR:
                 # A global variable of the exact type specified here
+                # (nowadays, only used by the ABI mode or backend
+                # compatibility; see OP_GLOBAL_F for the API mode
                 w_ct = realize_c_type.realize_c_type(
                     self.ffi, self.ctx.c_types, getarg(g.c_type_op))
                 g_size = rffi.cast(lltype.Signed, g.c_size_or_direct_fn)
@@ -113,7 +115,13 @@
                 ptr = rffi.cast(rffi.CCHARP, g.c_address)
                 if not ptr:   # for dlopen() style
                     ptr = self.cdlopen_fetch(attr)
-                w_result = cglob.W_GlobSupport(space, w_ct, ptr)
+                w_result = cglob.W_GlobSupport(space, attr, w_ct, ptr=ptr)
+                #
+            elif op == cffi_opcode.OP_GLOBAL_VAR_F:
+                w_ct = realize_c_type.realize_c_type(
+                    self.ffi, self.ctx.c_types, getarg(g.c_type_op))
+                w_result = cglob.W_GlobSupport(space, attr, w_ct,
+                                               fetch_addr=g.c_address)
                 #
             elif (op == cffi_opcode.OP_CONSTANT_INT or
                   op == cffi_opcode.OP_ENUM):
@@ -131,6 +139,9 @@
                     realize_c_type.FUNCPTR_FETCH_CHARP,
                     g.c_address)
                 if w_ct.size <= 0:
+                    raise oefmt(self.ffi.w_FFIError,
+                                "constant '%s' is of type '%s', "
+                                "whose size is not known", attr, w_ct.name)
                     raise oefmt(space.w_SystemError,
                                 "constant has no known size")
                 if not fetch_funcptr:   # for dlopen() style
@@ -172,7 +183,7 @@
             w_value = self._build_attr(attr)
             if w_value is None:
                 if is_getattr and attr == '__all__':
-                    return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR)
+                    return self.dir1(ignore_global_vars=True)
                 if is_getattr and attr == '__dict__':
                     return self.full_dict_copy()
                 if is_getattr and attr == '__name__':
@@ -206,14 +217,18 @@
     def descr_dir(self):
         return self.dir1()
 
-    def dir1(self, ignore_type=-1):
+    def dir1(self, ignore_global_vars=False):
         space = self.space
         total = rffi.getintfield(self.ctx, 'c_num_globals')
         g = self.ctx.c_globals
         names_w = []
         for i in range(total):
-            if getop(g[i].c_type_op) != ignore_type:
-                names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
+            if ignore_global_vars:
+                op = getop(g[i].c_type_op)
+                if (op == cffi_opcode.OP_GLOBAL_VAR or
+                    op == cffi_opcode.OP_GLOBAL_VAR_F):
+                    continue
+            names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
     def full_dict_copy(self):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1170,6 +1170,14 @@
     BShort = new_primitive_type("short")
     BFunc = new_function_type((BShort,), BShort, False)
     f = callback(BFunc, Zcb1, -42)
+    #
+    seen = []
+    oops_result = None
+    def oops(*args):
+        seen.append(args)
+        return oops_result
+    ff = callback(BFunc, Zcb1, -42, oops)
+    #
     orig_stderr = sys.stderr
     orig_getline = linecache.getline
     try:
@@ -1195,6 +1203,59 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        assert len(seen) == 0
+        assert ff(bigvalue) == -42
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = 81
+        assert ff(bigvalue) == 81
+        oops_result = None
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = "xy"     # not None and not an int!
+        assert ff(bigvalue) == -42
+        oops_result = None
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+TypeError: $integer$
+""")
+        #
+        sys.stderr = cStringIO.StringIO()
+        seen = "not a list"    # this makes the oops() function crash
+        assert ff(bigvalue) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+Traceback (most recent call last):
+  File "$", line $, in oops
+    $
+AttributeError: 'str' object has no attribute 'append'
+""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline
@@ -3341,6 +3402,29 @@
     py.test.raises(RuntimeError, "p[42]")
     py.test.raises(RuntimeError, "p[42] = -1")
 
+def test_mixup():
+    BStruct1 = new_struct_type("foo")
+    BStruct2 = new_struct_type("foo")   # <= same name as BStruct1
+    BStruct3 = new_struct_type("bar")
+    BStruct1Ptr = new_pointer_type(BStruct1)
+    BStruct2Ptr = new_pointer_type(BStruct2)
+    BStruct3Ptr = new_pointer_type(BStruct3)
+    BStruct1PtrPtr = new_pointer_type(BStruct1Ptr)
+    BStruct2PtrPtr = new_pointer_type(BStruct2Ptr)
+    BStruct3PtrPtr = new_pointer_type(BStruct3Ptr)
+    pp1 = newp(BStruct1PtrPtr)
+    pp2 = newp(BStruct2PtrPtr)
+    pp3 = newp(BStruct3PtrPtr)
+    pp1[0] = pp1[0]
+    e = py.test.raises(TypeError, "pp3[0] = pp1[0]")
+    assert str(e.value).startswith("initializer for ctype 'bar *' must be a ")
+    assert str(e.value).endswith(", not cdata 'foo *'")
+    e = py.test.raises(TypeError, "pp2[0] = pp1[0]")
+    assert str(e.value) == ("initializer for ctype 'foo *' appears indeed to "
+                            "be 'foo *', but the types are different (check "
+                            "that you are not e.g. mixing up different ffi "
+                            "instances)")
+
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.1.2"
+    assert __version__ == "1.2.0"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -114,6 +114,18 @@
         assert ffi.callback("int(int)", lambda x: x + "", -66)(10) == -66
         assert ffi.callback("int(int)", lambda x: x + "", error=-66)(10) == -66
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myerror(exc, val, tb):
+            seen.append(exc)
+        cb = ffi.callback("int(int)", lambda x: x + "", onerror=myerror)
+        assert cb(10) == 0
+        cb = ffi.callback("int(int)", lambda x:int(1E100), -66, onerror=myerror)
+        assert cb(10) == -66
+        assert seen == [TypeError, OverflowError]
+
     def test_ffi_callback_decorator(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
@@ -122,6 +134,37 @@
         assert deco(lambda x: x + "")(10) == -66
         assert deco(lambda x: x + 42)(10) == 52
 
+    def test_ffi_callback_onerror(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def oops(*args):
+            seen.append(args)
+
+        @ffi.callback("int(int)", onerror=oops)
+        def fn1(x):
+            return x + ""
+        assert fn1(10) == 0
+
+        @ffi.callback("int(int)", onerror=oops, error=-66)
+        def fn2(x):
+            return x + ""
+        assert fn2(10) == -66
+
+        assert len(seen) == 2
+        exc, val, tb = seen[0]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn1"
+        exc, val, tb = seen[1]
+        assert exc is TypeError
+        assert isinstance(val, TypeError)
+        assert tb.tb_frame.f_code.co_name == "fn2"
+        del seen[:]
+        #
+        raises(TypeError, ffi.callback, "int(int)",
+               lambda x: x, onerror=42)   # <- not callable
+
     def test_ffi_getctype(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
@@ -228,3 +271,99 @@
             import gc
             gc.collect()
         assert seen == [1]
+
+    def test_ffi_new_allocator_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        alloc1 = ffi.new_allocator()
+        alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
+        for retry in range(100):
+            p1 = alloc1("int[10]")
+            p2 = alloc2("int[10]")
+            combination = 0
+            for i in range(10):
+                assert p1[i] == 0
+                combination |= p2[i]
+                p1[i] = -42
+                p2[i] = -43
+            if combination != 0:
+                break
+            del p1, p2
+            import gc; gc.collect()
+        else:
+            raise AssertionError("cannot seem to get an int[10] not "
+                                 "completely cleared")
+
+    def test_ffi_new_allocator_2(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", b"X" * size)
+        def myfree(raw):
+            seen.append(raw)
+        alloc1 = ffi.new_allocator(myalloc, myfree)
+        alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+                                   should_clear_after_alloc=False)
+        p1 = alloc1("int[10]")
+        p2 = alloc2("int[]", 10)
+        assert seen == [40, 40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert ffi.typeof(p2) == ffi.typeof("int[]")
+        assert ffi.sizeof(p2) == 40
+        assert p1[5] == 0
+        assert p2[6] == ord('X') * 0x01010101
+        raw1 = ffi.cast("char *", p1)
+        raw2 = ffi.cast("char *", p2)
+        del p1, p2
+        retries = 0
+        while len(seen) != 4:
+            retries += 1
+            assert retries <= 5
+            import gc; gc.collect()
+        assert seen == [40, 40, raw1, raw2]
+        assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+        assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+    def test_ffi_new_allocator_3(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        seen = []
+        def myalloc(size):
+            seen.append(size)
+            return ffi.new("char[]", b"X" * size)
+        alloc1 = ffi.new_allocator(myalloc)    # no 'free'
+        p1 = alloc1("int[10]")
+        assert seen == [40]
+        assert ffi.typeof(p1) == ffi.typeof("int[10]")
+        assert ffi.sizeof(p1) == 40
+        assert p1[5] == 0
+
+    def test_ffi_new_allocator_4(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(TypeError, ffi.new_allocator, free=lambda x: None)
+        #
+        def myalloc2(size):
+            raise LookupError
+        alloc2 = ffi.new_allocator(myalloc2)
+        raises(LookupError, alloc2, "int[5]")
+        #
+        def myalloc3(size):
+            return 42
+        alloc3 = ffi.new_allocator(myalloc3)
+        e = raises(TypeError, alloc3, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata object (got int)"
+        #
+        def myalloc4(size):
+            return ffi.cast("int", 42)
+        alloc4 = ffi.new_allocator(myalloc4)
+        e = raises(TypeError, alloc4, "int[5]")
+        assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+        #
+        def myalloc5(size):
+            return ffi.NULL
+        alloc5 = ffi.new_allocator(myalloc5)
+        raises(MemoryError, alloc5, "int[5]")
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -16,8 +16,8 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 0, 4):
-        py.test.skip("system cffi module needs to be at least 1.0.4")
+    if cffi.__version_info__ < (1, 2, 0):
+        py.test.skip("system cffi module needs to be at least 1.2.0")
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -500,28 +500,33 @@
             "int foo(int x) { return x + 32; }")
         assert lib.foo(10) == 42
 
-    def test_bad_size_of_global_1(self):
-        ffi, lib = self.prepare(
-            "short glob;",
-            "test_bad_size_of_global_1",
-            "long glob;")
-        raises(ffi.error, getattr, lib, "glob")
-
-    def test_bad_size_of_global_2(self):
-        ffi, lib = self.prepare(
-            "int glob[10];",
-            "test_bad_size_of_global_2",
-            "int glob[9];")
-        e = raises(ffi.error, getattr, lib, "glob")
-        assert str(e.value) == ("global variable 'glob' should be 40 bytes "
-                                "according to the cdef, but is actually 36")
-
-    def test_unspecified_size_of_global(self):
+    def test_unspecified_size_of_global_1(self):
         ffi, lib = self.prepare(
             "int glob[];",
-            "test_unspecified_size_of_global",
+            "test_unspecified_size_of_global_1",
             "int glob[10];")
-        lib.glob    # does not crash
+        assert ffi.typeof(lib.glob) == ffi.typeof("int *")
+
+    def test_unspecified_size_of_global_2(self):
+        ffi, lib = self.prepare(
+            "int glob[][5];",
+            "test_unspecified_size_of_global_2",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+    def test_unspecified_size_of_global_3(self):
+        ffi, lib = self.prepare(
+            "int glob[][...];",
+            "test_unspecified_size_of_global_3",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
+
+    def test_unspecified_size_of_global_4(self):
+        ffi, lib = self.prepare(
+            "int glob[...][...];",
+            "test_unspecified_size_of_global_4",
+            "int glob[10][5];")
+        assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
     def test_include_1(self):
         ffi1, lib1 = self.prepare(
@@ -869,11 +874,22 @@
             """)
         assert lib.almost_forty_two == 42.25
 
+    def test_constant_of_unknown_size(self):
+        ffi, lib = self.prepare(
+            "typedef ... opaque_t;"
+            "const opaque_t CONSTANT;",
+            'test_constant_of_unknown_size',
+            "typedef int opaque_t;"
+            "const int CONSTANT = 42;")
+        e = raises(ffi.error, getattr, lib, 'CONSTANT')
+        assert str(e.value) == ("constant 'CONSTANT' is of "
+                                "type 'opaque_t', whose size is not known")
+
     def test_variable_of_unknown_size(self):
         ffi, lib = self.prepare("""
             typedef ... opaque_t;
             opaque_t globvar;
-        """, 'test_constant_of_unknown_size', """
+        """, 'test_variable_of_unknown_size', """
             typedef char opaque_t[6];
             opaque_t globvar = "hello";
         """)
@@ -1012,3 +1028,51 @@
         assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
         assert lib.__name__ == repr(lib)
+
+    def test_macro_var_callback(self):
+        ffi, lib = self.prepare(
+            "int my_value; int *(*get_my_value)(void);",
+            'test_macro_var_callback',
+            "int *(*get_my_value)(void);\n"
+            "#define my_value (*get_my_value())")
+        #
+        values = ffi.new("int[50]")
+        def it():
+            for i in range(50):
+                yield i
+        it = it()
+        #
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            return values + it.next()
+        lib.get_my_value = get_my_value
+        #
+        values[0] = 41
+        assert lib.my_value == 41            # [0]
+        p = ffi.addressof(lib, 'my_value')   # [1]
+        assert p == values + 1
+        assert p[-1] == 41
+        assert p[+1] == 0
+        lib.my_value = 42                    # [2]
+        assert values[2] == 42
+        assert p[-1] == 41
+        assert p[+1] == 42
+        #
+        # if get_my_value raises or returns nonsense, the exception is printed
+        # to stderr like with any callback, but then the C expression 'my_value'
+        # expand to '*NULL'.  We assume here that '&my_value' will return NULL
+        # without segfaulting, and check for NULL when accessing the variable.
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            raise LookupError
+        lib.get_my_value = get_my_value
+        raises(ffi.error, getattr, lib, 'my_value')
+        raises(ffi.error, setattr, lib, 'my_value', 50)
+        raises(ffi.error, ffi.addressof, lib, 'my_value')
+        @ffi.callback("int *(*)(void)")
+        def get_my_value():
+            return "hello"
+        lib.get_my_value = get_my_value
+        raises(ffi.error, getattr, lib, 'my_value')
+        e = raises(ffi.error, setattr, lib, 'my_value', 50)
+        assert str(e.value) == "global variable 'my_value' is at address NULL"
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -9,6 +9,7 @@
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
+from pypy.module._cffi_backend import allocator
 
 
 class W_FunctionWrapper(W_Root):
@@ -74,7 +75,8 @@
             # then internally allocate the struct and pass a pointer to it as
             # a first argument.
             if locs[0] == 'R':
-                w_result_cdata = ctype.fargs[0].newp(space.w_None)
+                w_result_cdata = ctype.fargs[0].newp(space.w_None,
+                                                    allocator.nonzero_allocator)
                 args_w = [w_result_cdata] + args_w
                 prepare_args(space, rawfunctype, args_w, 1)
                 #
@@ -116,7 +118,7 @@
             # the equivalent of ffi.new()
             if space.is_w(w_arg, space.w_None):
                 continue
-            w_arg = farg.newp(w_arg)
+            w_arg = farg.newp(w_arg, allocator.default_allocator)
         args_w[i] = w_arg
 
 
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -23,11 +23,16 @@
 # running make inside the src dir
 DYNAMIC_VMPROF = False
 
+if sys.platform.startswith('linux'):
+    libs = ['dl']
+else:
+    libs = []
+
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['vmprof.h', 'trampoline.h'],
     separate_module_files = [SRC.join('trampoline.vmprof.s')],
-    libraries = ['dl'],
+    libraries = libs,
     
     post_include_bits=["""
         int pypy_vmprof_init(void);
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
--- a/pypy/module/_vmprof/src/config.h
+++ b/pypy/module/_vmprof/src/config.h
@@ -1,2 +1,6 @@
 #define HAVE_SYS_UCONTEXT_H
+#if defined(__FreeBSD__) || defined(__APPLE__)
+#define PC_FROM_UCONTEXT uc_mcontext.mc_rip
+#else
 #define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
+#endif
diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
--- a/pypy/module/_vmprof/src/getpc.h
+++ b/pypy/module/_vmprof/src/getpc.h
@@ -132,7 +132,7 @@
   }
 };
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   // See comment above struct CallUnrollInfo.  Only try instruction
   // flow matching if both eip and esp looks reasonable.
   const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
@@ -168,7 +168,7 @@
 typedef int ucontext_t;
 #endif
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
   return NULL;
 }
@@ -178,7 +178,7 @@
 // the right value for your system, and add it to the list in
 // configure.ac (or set it manually in your config.h).
 #else
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
 }
 
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -33,6 +33,9 @@
 //#include <libunwind.h>
 
 #include "vmprof.h"
+#if defined(__FreeBSD__) || defined(__APPLE__)
+#define sighandler_t sig_t
+#endif
 
 #define _unused(x) ((void)x)
 
@@ -259,13 +262,31 @@
 	int marker = MARKER_TRAILER;
 	write(profile_file, &marker, 1);
 
+#ifdef __linux__
     // copy /proc/PID/maps to the end of the profile file
     sprintf(buf, "/proc/%d/maps", getpid());
-    src = fopen(buf, "r");    
+    src = fopen(buf, "r");
+    if (!src) {
+        vmprof_error = "error opening proc maps";
+        return -1;
+    }
     while ((size = fread(buf, 1, BUFSIZ, src))) {
         write(profile_file, buf, size);
     }
     fclose(src);
+#else
+    // freebsd and mac
+    sprintf(buf, "procstat -v %d", getpid());
+    src = popen(buf, "r");
+    if (!src) {
+        vmprof_error = "error calling procstat";
+        return -1;
+    }
+    while ((size = fread(buf, 1, BUFSIZ, src))) {
+        write(profile_file, buf, size);
+    }
+    pclose(src);
+#endif
     close(profile_file);
 	return 0;
 }
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -72,6 +72,14 @@
              b          = "insubpackage = 1",
              )
     setuppkg("pkg.pkg2", a='', b='')
+    setuppkg("pkg.withall",
+             __init__  = "__all__ = ['foobar']",
+             foobar    = "found = 123")
+    setuppkg("pkg.withoutall",
+             __init__  = "",
+             foobar    = "found = 123")
+    setuppkg("pkg.bogusall",
+             __init__  = "__all__ = 42")
     setuppkg("pkg_r", inpkg = "import x.y")
     setuppkg("pkg_r.x", y='')
     setuppkg("x")
@@ -697,6 +705,32 @@
         import imp
         raises(ValueError, imp.load_module, "", "", "", [1, 2, 3, 4])
 
+    def test_import_star_finds_submodules_with___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec("from pkg.withall import *", d)
+            assert d["foobar"].found == 123
+
+    def test_import_star_does_not_find_submodules_without___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec("from pkg.withoutall import *", d)
+            assert "foobar" not in d
+        import pkg.withoutall.foobar     # <- import it here only
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec("from pkg.withoutall import *", d)
+            assert d["foobar"].found == 123
+
+    def test_import_star_with_bogus___all__(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            try:
+                exec("from pkg.bogusall import *", {})
+            except TypeError:
+                pass    # 'int' object does not support indexing
+            else:
+                raise AssertionError("should have failed")
+
     def test_source_encoding(self):
         import imp
         import encoded
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -196,7 +196,12 @@
                     "'%T' object is not iterable", self)
 
     def descr_str(self, space):
-        return space.wrap(self.get_dtype(space).itemtype.str_format(self, add_quotes=False))
+        tp = self.get_dtype(space).itemtype
+        return space.wrap(tp.str_format(self, add_quotes=False))
+
+    def descr_repr(self, space):
+        tp = self.get_dtype(space).itemtype
+        return space.wrap(tp.str_format(self, add_quotes=True))
 
     def descr_format(self, space, w_spec):
         return space.format(self.item(space), w_spec)
@@ -607,16 +612,25 @@
         return W_StringBox(arr, 0, arr.dtype)
 
 class W_UnicodeBox(W_CharacterBox):
+    def __init__(self, value):
+        self._value = value
+
+    def convert_to(self, space, dtype):
+        if dtype.is_unicode():
+            return self
+        elif dtype.is_object():
+            return W_ObjectBox(space.wrap(self._value))
+        else:
+            raise oefmt(space.w_NotImplementedError,
+                        "Conversion from unicode not implemented yet")
+
+    def get_dtype(self, space):
+        from pypy.module.micronumpy.descriptor import new_unicode_dtype
+        return new_unicode_dtype(space, len(self._value))
+
     def descr__new__unicode_box(space, w_subtype, w_arg):
-        raise oefmt(space.w_NotImplementedError, "Unicode is not supported yet")
-        from pypy.module.micronumpy.descriptor import new_unicode_dtype
-        arg = space.unicode_w(space.unicode_from_object(w_arg))

From noreply at buildbot.pypy.org  Fri Jul 17 15:45:27 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 17 Jul 2015 15:45:27 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: copy all the stuff needed to
 produce the pdf, expand the slides
Message-ID: <20150717134527.067861C101F@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: extradoc
Changeset: r5542:0e6f6ebd4d39
Date: 2015-07-17 15:46 +0200
http://bitbucket.org/pypy/extradoc/changeset/0e6f6ebd4d39/

Log:	copy all the stuff needed to produce the pdf, expand the slides

diff --git a/talk/ep2015/performance/Makefile b/talk/ep2015/performance/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/ep2015/performance/Makefile
@@ -0,0 +1,18 @@
+# you can find rst2beamer.py here:
+# http://codespeak.net/svn/user/antocuni/bin/rst2beamer.py
+
+# WARNING: to work, it needs this patch for docutils
+# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414
+
+talk.pdf: talk.rst author.latex stylesheet.latex
+	python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
+	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
+	pdflatex talk.latex  || exit
+
+view: talk.pdf
+	evince talk.pdf &
+
+xpdf: talk.pdf
+	xpdf talk.pdf &
diff --git a/talk/ep2015/performance/author.latex b/talk/ep2015/performance/author.latex
new file mode 100644
--- /dev/null
+++ b/talk/ep2015/performance/author.latex
@@ -0,0 +1,8 @@
+\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0}
+
+\title[Python and PyPy performance]{Python and PyPy performance\\(not) for dummies}
+\author[antocuni,fijal]
+{Antonio Cuni and Maciej Fijalkowski}
+
+\institute{EuroPython 2015}
+\date{July 21, 2015}
diff --git a/talk/ep2015/performance/beamerdefs.txt b/talk/ep2015/performance/beamerdefs.txt
new file mode 100644
--- /dev/null
+++ b/talk/ep2015/performance/beamerdefs.txt
@@ -0,0 +1,108 @@
+.. colors
+.. ===========================
+
+.. role:: green
+.. role:: red
+
+
+.. general useful commands
+.. ===========================
+
+.. |pause| raw:: latex
+
+   \pause
+
+.. |small| raw:: latex
+
+   {\small
+
+.. |end_small| raw:: latex
+
+   }
+
+.. |scriptsize| raw:: latex
+
+   {\scriptsize
+
+.. |end_scriptsize| raw:: latex
+
+   }
+
+.. |strike<| raw:: latex
+
+   \sout{
+
+.. closed bracket
+.. ===========================
+
+.. |>| raw:: latex
+
+   }
+
+
+.. example block
+.. ===========================
+
+.. |example<| raw:: latex
+
+   \begin{exampleblock}{
+
+
+.. |end_example| raw:: latex
+
+   \end{exampleblock}
+
+
+
+.. alert block
+.. ===========================
+
+.. |alert<| raw:: latex
+
+   \begin{alertblock}{
+
+
+.. |end_alert| raw:: latex
+
+   \end{alertblock}
+
+
+
+.. columns
+.. ===========================
+
+.. |column1| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.45\textwidth}
+
+.. |column2| raw:: latex
+
+      \end{column}
+      \begin{column}{0.45\textwidth}
+
+
+.. |end_columns| raw:: latex
+
+      \end{column}
+   \end{columns}
+
+
+
+.. |snake| image:: ../../img/py-web-new.png
+           :scale: 15%
+           
+
+
+.. nested blocks
+.. ===========================
+
+.. |nested| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.85\textwidth}
+
+.. |end_nested| raw:: latex
+
+      \end{column}
+   \end{columns}
diff --git a/talk/ep2015/performance/jit-overview1.pdf b/talk/ep2015/performance/jit-overview1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..6b511590edd06e9deb157eb02cecb7cf64c61926
GIT binary patch

[cut]

diff --git a/talk/ep2015/performance/jit-overview2.pdf b/talk/ep2015/performance/jit-overview2.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..c07d0e3948403a7c42951a65ba023311d4739ac8
GIT binary patch

[cut]

diff --git a/talk/ep2015/performance/jit-overview3.pdf b/talk/ep2015/performance/jit-overview3.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..277b2067bc2daf79f7696edaabd22614182e6bfa
GIT binary patch

[cut]

diff --git a/talk/ep2015/performance/speed.png b/talk/ep2015/performance/speed.png
new file mode 100644
index 0000000000000000000000000000000000000000..e980452b50ec2cd6bcc1787cb5596bf106ddcd99
GIT binary patch

[cut]

diff --git a/talk/ep2015/performance/stylesheet.latex b/talk/ep2015/performance/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/ep2015/performance/stylesheet.latex
@@ -0,0 +1,10 @@
+\usetheme{Boadilla}
+\setbeamercovered{transparent}
+\setbeamertemplate{navigation symbols}{}
+
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/ep2015/performance/talk.rst b/talk/ep2015/performance/talk.rst
--- a/talk/ep2015/performance/talk.rst
+++ b/talk/ep2015/performance/talk.rst
@@ -1,7 +1,237 @@
+.. include:: beamerdefs.txt
 
+==========================
 Python & PyPy performance
+==========================
+
+About us
+---------
+
+- PyPy core devs
+
+- ``pdb++``, ``fancycompleter``, ...
+
+- Consultant
+
+- http://baroquesoftware.com/
+
+
+About you
+-------------
+
+- Target audience
+
+- Your Python program is slow
+
+- You want to make it fast(er)
+
+
+Optimization for dummies
 -------------------------
 
+* Obligatory citation
+
+  - *premature optimization is the root of all evil* (D. Knuth)
+
+* Pareto principle, or 80-20 rule
+
+  - 80% of the time will be spent in 20% of the program
+
+* Two golden rules:
+
+  1. Identify the slow spots
+
+  2. Optimize them
+
+
+This talk
+----------------------------
+
+* Two parts
+
+  1. PyPy as a tool to make Python faster
+
+  2. How to identify the slow spots
+
+
+
+Tools
+------
+
+- Endless list of tools/techniques to increment speed
+
+- C extension
+
+- Cython
+
+- numba
+
+- "performance tricks"
+
+- **PyPy**
+
+  * We'll concentrate on it
+
+  * WARNING: we wrote it, we are biased :)
+
+
+
+What is PyPy
+---------------
+
+- Alternative, fast Python implementation
+
+- Performance: JIT compiler, advanced GC
+
+- PyPy 2.6.0 (Python version 2.7.9)
+
+- Py3k as usual in progress (3.2.5 out, 3.3 in development)
+
+- http://pypy.org
+
+- EP Talks:
+  
+  * The GIL is dead: PyPy-STM
+    
+    (July 23, 16:45 by Armin Rigo)
+
+  * PyPy ecosystem: CFFI, numpy, scipy, etc 
+    
+    (July 24, 15:15 by Romain Guillebert)
+
+
+Speed: 7x faster than CPython
+-------------------------------
+
+.. image:: speed.png
+   :scale: 47%
+
+
+The JIT
+--------
+
+.. image:: jit-overview1.pdf
+   :scale: 50%
+
+
+The JIT
+--------
+
+.. image:: jit-overview2.pdf
+   :scale: 50%
+
+
+The JIT
+--------
+
+.. image:: jit-overview3.pdf
+   :scale: 50%
+
+
+JIT overview
+-------------
+
+- Tracing JIT
+
+  * detect and compile "hot" loops
+
+  * (although not only loops)
+
+- **Specialization**
+
+- Precompute as much as possible
+
+- Constant propagation
+
+- Aggressive inlining
+
+
+Specialization (1)
+-------------------
+
+- ``obj.foo()``
+
+- which code is executed? (SIMPLIFIED)
+
+  * lookup ``foo`` in obj.__dict__
+
+  * lookup ``foo`` in obj.__class__
+
+  * lookup ``foo`` in obj.__bases__[0], etc.
+
+  * finally, execute ``foo``
+
+- without JIT, you need to do these steps again and again
+
+- Precompute the lookup?
+
+
+Specialization (2)
+--------------------
+
+- pretend and assume that ``obj.__class__`` IS constant
+
+  * "promotion"
+
+- guard
+
+  * check our assumption: if it's false, bail out
+
+- now we can directly jump to ``foo`` code
+
+  * ...unless ``foo`` is in ``obj.__dict__``: GUARD!
+
+  * ...unless ``foo.__class__.__dict__`` changed: GUARD!
+
+- Too many guard failures?
+
+  * Compile some more assembler!
+
+- guards are cheap
+
+  * out-of-line guards even more
+
+
+Specialization (3)
+---------------------
+
+- who decides what to promote/specialize for?
+
+  * we, the PyPy devs :)
+
+  * heuristics
+
+- instance attributes are never promoted
+
+- class attributes are promoted by default (with some exceptions)
+
+- module attributes (i.e., globals) as well
+
+- bytecode constants
+
+
+Specialization trade-offs
+--------------------------
+
+- Too much specialization
+
+  * guards fails often
+
+  * explosion of assembler
+
+- Not enough specialization
+
+  * inefficient code
+
+
+Part 2
+-------
+
+* Measure performance
+
+* Identify problems
+
+
 What is performance?
 --------------------
 

From noreply at buildbot.pypy.org  Fri Jul 17 16:37:42 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 17 Jul 2015 16:37:42 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: added clone to a descriptor. I used
 invent_fail_descr before that, which did the same thing,
 but in a huge switch statement
Message-ID: <20150717143742.A77851C139F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78578:a05aff84b29f
Date: 2015-07-17 15:28 +0200
http://bitbucket.org/pypy/pypy/changeset/a05aff84b29f/

Log:	added clone to a descriptor. I used invent_fail_descr before that,
	which did the same thing, but in a huge switch statement

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -674,6 +674,11 @@
                 assert 0, box.type
             self.status = ty | (r_uint(i) << self.ST_SHIFT)
 
+    def clone(self):
+        cloned = self.__class__()
+        cloned.copy_all_attributes_from(self)
+        return cloned
+
 class ResumeGuardNonnullDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_NONNULL
 
@@ -815,6 +820,13 @@
         hidden_all_virtuals = obj.hide(metainterp_sd.cpu)
         metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals)
 
+    def clone(self):
+        cloned = self.__class__()
+        cloned.metainterp_sd = self.metainterp_sd
+        cloned.jitdriver_sd = self.jitdriver_sd
+        cloned.copy_all_attributes_from(self)
+        return cloned
+
 def invent_fail_descr_for_op(opnum, optimizer):
     if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
         resumedescr = ResumeGuardForcedDescr()
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -863,7 +863,14 @@
         return version
 
     def copy_operations(self):
-        return [ op.clone() for op in self.operations ]
+        operations = []
+        for op in self.operations:
+            cloned = op.clone()
+            operations.append(cloned)
+            if cloned.getdescr():
+                descr = cloned.getdescr().clone()
+                cloned.setdescr(descr)
+        return operations
 
     def get_display_text(self):    # for graphpage.py
         return self.name + '\n' + repr(self.inputargs)

From noreply at buildbot.pypy.org  Fri Jul 17 16:37:43 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 17 Jul 2015 16:37:43 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: minor simplifications,
 fixed some tests cases (more to go),
 the additional abc opt removes some opts,
 the tests should not check for equivalence of all instructions,
 but check for a sequence
Message-ID: <20150717143743.ECBFA1C13C2@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78579:aa9a9d9cc94b
Date: 2015-07-17 16:37 +0200
http://bitbucket.org/pypy/pypy/changeset/aa9a9d9cc94b/

Log:	minor simplifications, fixed some tests cases (more to go), the
	additional abc opt removes some opts, the tests should not check for
	equivalence of all instructions, but check for a sequence

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -199,10 +199,12 @@
     if loop.versions is not None:
         token = jitcell_token
         for version in loop.versions:
-            for faildescr in version.faildescrs:
+            for i, faildescr in enumerate(version.faildescrs):
                 vl = create_empty_loop(metainterp)
                 vl.inputargs = version.inputargs
-                vl.operations = version.copy_operations()
+                vl.operations = version.operations
+                if i > 0:
+                    vl.operations = vl.copy_operations()
                 vl.original_jitcell_token = jitcell_token
                 send_bridge_to_backend(jitdriver_sd, metainterp_sd,
                                        faildescr, version.inputargs,
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -746,9 +746,8 @@
 
 class LoopVersion(object):
 
-    def __init__(self, operations, aligned=False):
+    def __init__(self, operations):
         self.operations = operations
-        self.aligned = aligned
         self.faildescrs = []
         #
         idx = index_of_first(rop.LABEL, operations)
@@ -796,9 +795,6 @@
         op.setfailargs(self.inputargs)
         op.rd_snapshot = None
 
-    def copy_operations(self):
-        return [op.clone() for op in self.operations]
-
     def update_token(self, jitcell_token):
         label = self.operations[self.label_pos]
         jump = self.operations[-1]
@@ -858,18 +854,20 @@
         return index_of_first(opnum, self.operations)
 
     def snapshot(self):
-        version = LoopVersion(self.copy_operations(), [])
+        version = LoopVersion(self.copy_operations())
         self.versions.append(version)
         return version
 
     def copy_operations(self):
+        from rpython.jit.metainterp.compile import ResumeGuardDescr
         operations = []
         for op in self.operations:
             cloned = op.clone()
             operations.append(cloned)
-            if cloned.getdescr():
-                descr = cloned.getdescr().clone()
-                cloned.setdescr(descr)
+            descr = cloned.getdescr()
+            if cloned.is_guard() and descr:
+                assert isinstance(descr, ResumeGuardDescr)
+                cloned.setdescr(descr.clone())
         return operations
 
     def get_display_text(self):    # for graphpage.py
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -68,6 +68,12 @@
             unroll_factor = opt.get_unroll_count(ARCH_VEC_REG_SIZE)
             print ""
             print "unroll factor: ", unroll_factor, opt.smallest_type_bytes
+        if opt.loop.find_first_index(rop.GUARD_EARLY_EXIT) == -1:
+            idx = loop.find_first_index(rop.LABEL)
+            guard = ResOperation(rop.GUARD_EARLY_EXIT, [], None)
+            guard.setfailargs([])
+            guard.setdescr(compile.ResumeAtLoopHeaderDescr())
+            loop.operations.insert(idx+1, guard)
         opt.analyse_index_calculations()
         if opt.dependency_graph is not None:
             self._write_dot_and_convert_to_svg(opt.dependency_graph, "ee" + self.test_name)
@@ -216,7 +222,6 @@
     def test_vectorize_empty_with_early_exit(self):
         ops = """
         []
-        guard_early_exit() []
         jump()
         """
         try:
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -94,20 +94,6 @@
     return a.left.getindex() < b.left.getindex()
 packsort = listsort.make_timsort_class(lt=cmp_pack_lt)
 
-def copy_fail_descr(op, optimizer):
-    olddescr = op.getdescr()
-    exits_early = olddescr.guard_opnum == rop.GUARD_EARLY_EXIT
-    if exits_early:
-        if isinstance(olddescr, CompileLoopVersionDescr):
-            descr = CompileLoopVersionDescr()
-        else:
-            descr = ResumeAtLoopHeaderDescr()
-    else:
-        descr = invent_fail_descr_for_op(op.getopnum(), optimizer)
-    if olddescr:
-        descr.copy_all_attributes_from(olddescr)
-    return descr
-
 class VectorizingOptimizer(Optimizer):
     """ Try to unroll the loop and find instructions to group """
 
@@ -195,15 +181,9 @@
         self.emit_unrolled_operation(label_op)
 
         renamer = Renamer()
-        pure = True
         operations = []
-        ee_pos = -1
         for i in range(1,op_count-1):
             op = loop.operations[i].clone()
-            opnum = op.getopnum()
-            if opnum == rop.GUARD_EARLY_EXIT:
-                ee_pos = i
-
             if op.is_guard():
                 assert isinstance(op, GuardResOp)
                 failargs = renamer.rename_failargs(op, clone=True)
@@ -249,7 +229,9 @@
                 if copied_op.is_guard():
                     assert isinstance(copied_op, GuardResOp)
                     target_guard = copied_op
-                    copied_op.setdescr(copy_fail_descr(copied_op, self))
+                    descr = copied_op.getdescr()
+                    assert isinstance(descr, ResumeGuardDescr)
+                    copied_op.setdescr(descr.clone())
                     descr = target_guard.getdescr()
                     # copy failargs/snapshot
                     copied_op.rd_snapshot = \

From noreply at buildbot.pypy.org  Fri Jul 17 16:53:26 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Jul 2015 16:53:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix for 32 bit
Message-ID: <20150717145326.CCA761C101F@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78580:16f93a8bbc64
Date: 2015-07-17 17:53 +0300
http://bitbucket.org/pypy/pypy/changeset/16f93a8bbc64/

Log:	fix for 32 bit

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -10,7 +10,6 @@
 from rpython.rlib.objectmodel import (
         specialize, compute_hash, we_are_translated, enforceargs)
 from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
-from rpython.rlib.rstring import StringBuilder
 from pypy.module.micronumpy import types, boxes, support, constants as NPY
 from .base import W_NDimArray
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
@@ -951,9 +950,9 @@
                 raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple.")
             else:
                 raise
-        if dim > 2 ** 32 -1:
+        if dim > int(0x7fffffff):
             raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
-                  "dimension does not fit into a C int.")
+                      "dimension does not fit into a C int.")
         elif dim < 0:
             raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
                   "dimension smaller than zero.")
@@ -978,9 +977,10 @@
         assert isinstance(subdtype, W_Dtype)
         size = support.product(shape)
         size *= subdtype.elsize
-        if size >= 2 ** 31:
+        if size > int(0x7fffffff):
             raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
                   "dtype size in bytes must fit into a C int.")
+        
         return _set_metadata_and_copy(space, w_metadata,
                W_Dtype(types.VoidType(space), space.gettypefor(boxes.W_VoidBox),
                        shape=shape, subdtype=subdtype, elsize=size))

From noreply at buildbot.pypy.org  Fri Jul 17 17:03:04 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 17 Jul 2015 17:03:04 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: split Chunk into IntegerChunk and
	SliceChunk
Message-ID: <20150717150304.AA9201C1FF4@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78581:e7efb3c05cee
Date: 2015-07-17 15:20 +0100
http://bitbucket.org/pypy/pypy/changeset/e7efb3c05cee/

Log:	split Chunk into IntegerChunk and SliceChunk

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -10,7 +10,7 @@
     ArrayArgumentException, W_NumpyObject
 from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (
-    Chunk, new_view, NewAxisChunk, EllipsisChunk,
+    IntegerChunk, SliceChunk, NewAxisChunk, EllipsisChunk, new_view,
     calc_strides, calc_new_strides, shape_agreement,
     calculate_broadcast_strides, calc_backstrides, calc_start, is_c_contiguous,
     is_f_contiguous)
@@ -219,18 +219,19 @@
             raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
                 "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
                 "boolean arrays are valid indices")
-        if (space.isinstance_w(w_idx, space.w_int) or
-                space.isinstance_w(w_idx, space.w_slice)):
+        if space.isinstance_w(w_idx, space.w_slice):
             if len(self.get_shape()) == 0:
                 raise oefmt(space.w_ValueError, "cannot slice a 0-d array")
-            return [Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]
+            return [SliceChunk(w_idx, space, self.get_shape()[0])]
+        elif space.isinstance_w(w_idx, space.w_int):
+            return [IntegerChunk(w_idx, space, self.get_shape()[0])]
         elif isinstance(w_idx, W_NDimArray) and w_idx.is_scalar():
             w_idx = w_idx.get_scalar_value().item(space)
             if not space.isinstance_w(w_idx, space.w_int) and \
                     not space.isinstance_w(w_idx, space.w_bool):
                 raise OperationError(space.w_IndexError, space.wrap(
                     "arrays used as indices must be of integer (or boolean) type"))
-            return [Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]
+            return [IntegerChunk(w_idx, space, self.get_shape()[0])]
         elif space.is_w(w_idx, space.w_None):
             return [NewAxisChunk()]
         result = []
@@ -247,9 +248,11 @@
                 has_ellipsis = True
             elif space.is_w(w_item, space.w_None):
                 result.append(NewAxisChunk())
+            elif space.isinstance_w(w_item, space.w_slice):
+                result.append(SliceChunk(w_item, space, self.get_shape()[i]))
+                i += 1
             else:
-                result.append(Chunk(*space.decode_index4(w_item,
-                                                         self.get_shape()[i])))
+                result.append(IntegerChunk(w_item, space, self.get_shape()[i]))
                 i += 1
         return result
 
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -23,6 +23,19 @@
         return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step,
                                           self.lgt)
 
+class IntegerChunk(Chunk):
+    def __init__(self, w_idx, space, base_length):
+        self.w_idx = w_idx
+        args = space.decode_index4(w_idx, base_length)
+        Chunk.__init__(self, *args)
+
+
+class SliceChunk(Chunk):
+    def __init__(self, w_slice, space, base_length):
+        self.w_slice = w_slice
+        args = space.decode_index4(w_slice, base_length)
+        Chunk.__init__(self, *args)
+
 
 class NewAxisChunk(Chunk):
     start = 0

From noreply at buildbot.pypy.org  Fri Jul 17 17:03:06 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 17 Jul 2015 17:03:06 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: Delay computation of slice length
 (required to implement ellipses)
Message-ID: <20150717150306.28C221C1FF4@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78582:79db5bdf0df2
Date: 2015-07-17 16:03 +0100
http://bitbucket.org/pypy/pypy/changeset/79db5bdf0df2/

Log:	Delay computation of slice length (required to implement ellipses)

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -222,16 +222,16 @@
         if space.isinstance_w(w_idx, space.w_slice):
             if len(self.get_shape()) == 0:
                 raise oefmt(space.w_ValueError, "cannot slice a 0-d array")
-            return [SliceChunk(w_idx, space, self.get_shape()[0])]
+            return [SliceChunk(w_idx)]
         elif space.isinstance_w(w_idx, space.w_int):
-            return [IntegerChunk(w_idx, space, self.get_shape()[0])]
+            return [IntegerChunk(w_idx)]
         elif isinstance(w_idx, W_NDimArray) and w_idx.is_scalar():
             w_idx = w_idx.get_scalar_value().item(space)
             if not space.isinstance_w(w_idx, space.w_int) and \
                     not space.isinstance_w(w_idx, space.w_bool):
                 raise OperationError(space.w_IndexError, space.wrap(
                     "arrays used as indices must be of integer (or boolean) type"))
-            return [IntegerChunk(w_idx, space, self.get_shape()[0])]
+            return [IntegerChunk(w_idx)]
         elif space.is_w(w_idx, space.w_None):
             return [NewAxisChunk()]
         result = []
@@ -249,10 +249,10 @@
             elif space.is_w(w_item, space.w_None):
                 result.append(NewAxisChunk())
             elif space.isinstance_w(w_item, space.w_slice):
-                result.append(SliceChunk(w_item, space, self.get_shape()[i]))
+                result.append(SliceChunk(w_item))
                 i += 1
             else:
-                result.append(IntegerChunk(w_item, space, self.get_shape()[i]))
+                result.append(IntegerChunk(w_item))
                 i += 1
         return result
 
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -11,7 +11,7 @@
 
 
 class Chunk(BaseChunk):
-    axis_step = 1
+    input_dim = 1
 
     def __init__(self, start, stop, step, lgt):
         self.start = start
@@ -19,22 +19,45 @@
         self.step = step
         self.lgt = lgt
 
+    @property
+    def out_dim(self):
+        if self.step == 0:
+            return 0
+        else:
+            return 1
+
+    def compute(self, space, base_length, base_stride):
+        stride = base_stride * self.step
+        backstride = base_stride * max(0, self.lgt - 1) * self.step
+        return self.start, self.lgt, stride, backstride
+
     def __repr__(self):
         return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step,
                                           self.lgt)
 
-class IntegerChunk(Chunk):
-    def __init__(self, w_idx, space, base_length):
+class IntegerChunk(BaseChunk):
+    input_dim = 1
+    out_dim = 0
+    def __init__(self, w_idx):
         self.w_idx = w_idx
-        args = space.decode_index4(w_idx, base_length)
-        Chunk.__init__(self, *args)
 
+    def compute(self, space, base_length, base_stride):
+        start, _, _, _ = space.decode_index4(self.w_idx, base_length)
+        return start, 0, 0, 0
 
-class SliceChunk(Chunk):
-    def __init__(self, w_slice, space, base_length):
+
+class SliceChunk(BaseChunk):
+    input_dim = 1
+    out_dim = 1
+
+    def __init__(self, w_slice):
         self.w_slice = w_slice
-        args = space.decode_index4(w_slice, base_length)
-        Chunk.__init__(self, *args)
+
+    def compute(self, space, base_length, base_stride):
+        start, stop, step, length = space.decode_index4(self.w_slice, base_length)
+        stride = base_stride * step
+        backstride = base_stride * max(0, length - 1) * step
+        return start, length, stride, backstride
 
 
 class NewAxisChunk(Chunk):
@@ -42,11 +65,15 @@
     stop = 1
     step = 1
     lgt = 1
-    axis_step = 0
+    input_dim = 0
+    out_dim = 1
 
     def __init__(self):
         pass
 
+    def compute(self, space, base_length, base_stride):
+        return 0, 1, 0, 0
+
 class EllipsisChunk(BaseChunk):
     def __init__(self):
         pass
@@ -54,10 +81,9 @@
 
 def new_view(space, w_arr, chunks):
     arr = w_arr.implementation
-    shape = _extend_shape(arr.shape, chunks)
-    r = calculate_slice_strides(arr.shape, arr.start, arr.get_strides(),
+    r = calculate_slice_strides(space, arr.shape, arr.start, arr.get_strides(),
                                 arr.get_backstrides(), chunks)
-    _, start, strides, backstrides = r
+    shape, start, strides, backstrides = r
     return W_NDimArray.new_slice(space, start, strides[:], backstrides[:],
                                  shape[:], arr, w_arr)
 
@@ -66,7 +92,7 @@
     shape = []
     i = -1
     for i, c in enumerate_chunks(chunks):
-        if c.step != 0:
+        if c.out_dim > 0:
             shape.append(c.lgt)
     s = i + 1
     assert s >= 0
@@ -93,35 +119,40 @@
     result = []
     i = -1
     for chunk in chunks:
-        i += chunk.axis_step
+        i += chunk.input_dim
         result.append((i, chunk))
     return result
 
 
- at jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks:
+ at jit.look_inside_iff(lambda space, shape, start, strides, backstrides, chunks:
                      jit.isconstant(len(chunks)))
-def calculate_slice_strides(shape, start, strides, backstrides, chunks):
+def calculate_slice_strides(space, shape, start, strides, backstrides, chunks):
     size = 0
     for chunk in chunks:
-        if chunk.step != 0:
-            size += 1
+        size += chunk.out_dim
     rstrides = [0] * size
     rbackstrides = [0] * size
     rstart = start
     rshape = [0] * size
-    i = -1
-    j = 0
-    for i, chunk in enumerate_chunks(chunks):
+    i = -1  # index of the current dimension in the input array
+    j = 0  # index of the current dimension in the result view
+    for chunk in chunks:
+        i += chunk.input_dim
+        if isinstance(chunk, NewAxisChunk):
+            rshape[j] = 1
+            j += 1
+            continue
         try:
             s_i = strides[i]
         except IndexError:
             continue
-        if chunk.step != 0:
-            rstrides[j] = s_i * chunk.step
-            rbackstrides[j] = s_i * max(0, chunk.lgt - 1) * chunk.step
-            rshape[j] = chunk.lgt
-            j += 1
-        rstart += s_i * chunk.start
+        start, length, stride, backstride = chunk.compute(space, shape[i], strides[i])
+        if chunk.out_dim == 1:
+            rstrides[j] = stride
+            rbackstrides[j] = backstride
+            rshape[j] = length
+            j += chunk.out_dim
+        rstart += s_i * start
     # add a reminder
     s = i + 1
     assert s >= 0

From noreply at buildbot.pypy.org  Fri Jul 17 17:12:46 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 17 Jul 2015 17:12:46 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: hg merge default
Message-ID: <20150717151246.60B511C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78583:d579a53d3316
Date: 2015-07-17 16:12 +0100
http://bitbucket.org/pypy/pypy/changeset/d579a53d3316/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -37,3 +37,9 @@
 .. branch: unicode-dtype
 
 Partial implementation of unicode dtype and unicode scalars.
+
+.. branch: dtypes-compatability
+
+Improve compatibility with numpy dtypes; handle offsets to create unions,
+fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
+allow subclassing dtype
diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
--- a/pypy/module/_vmprof/src/getpc.h
+++ b/pypy/module/_vmprof/src/getpc.h
@@ -132,7 +132,7 @@
   }
 };
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   // See comment above struct CallUnrollInfo.  Only try instruction
   // flow matching if both eip and esp looks reasonable.
   const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
@@ -168,7 +168,7 @@
 typedef int ucontext_t;
 #endif
 
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
   return NULL;
 }
@@ -178,7 +178,7 @@
 // the right value for your system, and add it to the list in
 // configure.ac (or set it manually in your config.h).
 #else
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
   return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
 }
 
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -262,13 +262,31 @@
 	int marker = MARKER_TRAILER;
 	write(profile_file, &marker, 1);
 
+#ifdef __linux__
     // copy /proc/PID/maps to the end of the profile file
     sprintf(buf, "/proc/%d/maps", getpid());
-    src = fopen(buf, "r");    
+    src = fopen(buf, "r");
+    if (!src) {
+        vmprof_error = "error opening proc maps";
+        return -1;
+    }
     while ((size = fread(buf, 1, BUFSIZ, src))) {
         write(profile_file, buf, size);
     }
     fclose(src);
+#else
+    // freebsd and mac
+    sprintf(buf, "procstat -v %d", getpid());
+    src = popen(buf, "r");
+    if (!src) {
+        vmprof_error = "error calling procstat";
+        return -1;
+    }
+    while ((size = fread(buf, 1, BUFSIZ, src))) {
+        write(profile_file, buf, size);
+    }
+    pclose(src);
+#endif
     close(profile_file);
 	return 0;
 }
diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -8,6 +8,7 @@
     w__commastring = None
     w_array_repr = None
     w_array_str = None
+    w__usefields = None
 
     def __init__(self, space):
         pass
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -563,7 +563,7 @@
         elif space.isinstance_w(w_item, space.w_int):
             indx = space.int_w(w_item)
             try:
-                item = self.dtype.names[indx]
+                item = self.dtype.names[indx][0]
             except IndexError:
                 if indx < 0:
                     indx += len(self.dtype.names)
@@ -596,7 +596,7 @@
         try:
             ofs, dtype = self.dtype.fields[item]
         except KeyError:
-            raise oefmt(space.w_IndexError, "222only integers, slices (`:`), "
+            raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
                 "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
                 "boolean arrays are valid indices")
         dtype.store(self.arr, self.ofs, ofs,
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -47,6 +47,9 @@
     def lookup(self, name):
         return self.getdictvalue(self, name)
 
+    def getname(self, space):
+        return self.name
+
 class FakeSpace(ObjSpace):
     w_ValueError = W_TypeObject("ValueError")
     w_TypeError = W_TypeObject("TypeError")
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -624,15 +624,17 @@
         self.impl = impl
         self.readonly = readonly
 
-    def getitem(self, item):
-        return raw_storage_getitem(lltype.Char, self.impl.storage, item)
+    def getitem(self, index):
+        return raw_storage_getitem(lltype.Char, self.impl.storage,
+                 index + self.impl.start)
 
-    def setitem(self, item, v):
-        raw_storage_setitem(self.impl.storage, item,
+    def setitem(self, index, v):
+        raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
     def getlength(self):
-        return self.impl.size
+        return self.impl.size - self.impl.start
 
     def get_raw_address(self):
-        return self.impl.storage
+        from rpython.rtyper.lltypesystem import rffi
+        return rffi.ptradd(self.impl.storage, self.impl.start)
diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -92,6 +92,21 @@
 ARRAY_ELEMENTSTRIDES = 0x0080 # strides  are units of the dtype element size
 ARRAY_NOTSWAPPED  = 0x0200 #native byte order
 
+#dtype flags
+ITEM_REFCOUNT   = 0x01
+ITEM_HASOBJECT  = 0x01
+LIST_PICKLE     = 0x02
+ITEM_IS_POINTER = 0x04
+NEEDS_INIT      = 0x08
+NEEDS_PYAPI     = 0x10
+USE_GETITEM     = 0x20
+USE_SETITEM     = 0x40
+ALIGNED_STRUCT  = 0x80
+FROM_FIELDS     = NEEDS_INIT | LIST_PICKLE | ITEM_REFCOUNT | NEEDS_PYAPI
+OBJECT_DTYPE_FLAGS = (LIST_PICKLE | USE_GETITEM | ITEM_IS_POINTER |
+                        ITEM_REFCOUNT | NEEDS_INIT | NEEDS_PYAPI)
+
+
 LITTLE = '<'
 BIG = '>'
 NATIVE = '='
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -54,11 +54,11 @@
 class W_Dtype(W_Root):
     _immutable_fields_ = [
         "itemtype?", "w_box_type", "byteorder?", "names?", "fields?",
-        "elsize?", "alignment?", "shape?", "subdtype?", "base?"]
+        "elsize?", "alignment?", "shape?", "subdtype?", "base?", "flags?"]
 
     @enforceargs(byteorder=SomeChar())
     def __init__(self, itemtype, w_box_type, byteorder=NPY.NATIVE, names=[],
-                 fields={}, elsize=None, shape=[], subdtype=None):
+                 fields={}, elsize=-1, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.w_box_type = w_box_type
         if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType):
@@ -66,16 +66,21 @@
         self.byteorder = byteorder
         self.names = names
         self.fields = fields
-        if elsize is None:
+        if elsize < 0:
             elsize = itemtype.get_element_size()
         self.elsize = elsize
-        self.alignment = itemtype.alignment
         self.shape = shape
         self.subdtype = subdtype
+        self.flags = 0
+        self.metadata = None
+        if isinstance(itemtype, types.ObjectType):
+            self.flags = NPY.OBJECT_DTYPE_FLAGS
         if not subdtype:
             self.base = self
+            self.alignment = itemtype.get_element_size()
         else:
             self.base = subdtype.base
+            self.alignment = subdtype.itemtype.get_element_size()
 
     @property
     def num(self):
@@ -172,51 +177,149 @@
         return dtype
 
     def get_name(self):
-        name = self.w_box_type.name
-        if name.startswith('numpy.'):
-            name = name[6:]
+        name = self.w_box_type.getname(self.itemtype.space)
         if name.endswith('_'):
             name = name[:-1]
         return name
 
-    def descr_get_name(self, space):
-        name = self.get_name()
+    def descr_get_name(self, space, quote=False):
+        if quote:
+            name = "'" + self.get_name() + "'"
+        else:
+            name = self.get_name()
         if self.is_flexible() and self.elsize != 0:
             return space.wrap(name + str(self.elsize * 8))
         return space.wrap(name)
 
-    def descr_get_str(self, space):
+    def descr_get_str(self, space, ignore='|', simple=True):
+        if not simple and self.fields and len(self.fields) > 0:
+            return self.descr_get_descr(space)
+        total = 0
+        for s in self.shape:
+            total += s
+        if not simple and total > 0:
+            return space.newtuple(
+                [space.wrap(self.subdtype.get_str(ignore='')), 
+                 space.newtuple([space.wrap(s) for s in self.shape]),
+                ])
+        return space.wrap(self.get_str(ignore=ignore))
+
+    def get_str(self, ignore='|'):
         basic = self.kind
         endian = self.byteorder
         size = self.elsize
         if endian == NPY.NATIVE:
             endian = NPY.NATBYTE
+        elif endian == NPY.IGNORE:
+            endian = ignore
         if self.num == NPY.UNICODE:
             size >>= 2
-        return space.wrap("%s%s%s" % (endian, basic, size))
+        return "%s%s%s" % (endian, basic, size)
 
-    def descr_get_descr(self, space):
+    def descr_get_descr(self, space, style='descr', force_dict=False):
+        simple = False
+        if style == 'descr':
+            simple = True
         if not self.is_record():
             return space.newlist([space.newtuple([space.wrap(""),
-                                                  self.descr_get_str(space)])])
+                                                  self.descr_get_str(space, simple=simple)])])
+        elif (self.alignment > 1 and not style.startswith('descr')) or force_dict:
+            # we need to force a sorting order for the keys,
+            # so return a string instead of a dict. Also, numpy formats
+            # the lists without spaces between elements, so we cannot simply
+            # do str(names)
+            names = ["'names':["]
+            formats = ["'formats':["]
+            offsets = ["'offsets':["]
+            titles = ["'titles':["]
+            use_titles = False
+            show_offsets = False
+            offsets_n = []
+            total = 0
+            for name, title in self.names:
+                offset, subdtype = self.fields[name]
+                if subdtype.is_record():
+                    substr = [space.str_w(space.str(subdtype.descr_get_descr(
+                                                space, style='descr_subdtype'))), ","]
+                elif subdtype.subdtype is not None:
+                    substr = ["(", space.str_w(space.str(
+                        subdtype.subdtype.descr_get_descr(space, style='descr_subdtype'))),
+                        ', ',
+                        space.str_w(space.repr(space.newtuple([space.wrap(s) for s in subdtype.shape]))),
+                        "),"]
+                else:
+                    substr = ["'", subdtype.get_str(ignore=''), "',"]
+                formats += substr
+                offsets += [str(offset),  ',']
+                names += ["'", name, "',"]
+                titles += ["'", str(title), "',"]
+                if title is not None:
+                    use_titles = True
+                if total != offset:
+                    show_offsets = True
+                total += subdtype.elsize
+                # make sure offsets_n is sorted
+                i = 0
+                for i in range(len(offsets_n)):
+                    if offset < offsets_n[i]:
+                        break
+                offsets_n.insert(i, offset)
+            total = 0
+            for i in range(len(offsets_n)):
+                if offsets_n[i] != self.alignment * i:
+                    show_offsets = True
+            if use_titles and not show_offsets: 
+                return self.descr_get_descr(space, style='descr')
+            # replace the last , with a ]
+            formats[-1] = formats[-1][:-1] + ']'
+            offsets[-1] = offsets[-1][:-1] + ']'
+            names[-1] = names[-1][:-1] + ']'
+            titles[-1] = titles[-1][:-1] + ']'
+            if self.alignment < 2 or style.endswith('subdtype'):
+                suffix = "}"
+            elif style == 'str':
+                suffix = ", 'aligned':True}"
+            elif style == 'substr':
+                suffix = '}'
+            else:
+                suffix = "}, align=True"
+            s_as_list = ['{'] + names + [', '] + formats + [', '] + offsets + [', ']
+            if use_titles:
+                s_as_list += titles + [', ']
+                    
+            s_as_list += ["'itemsize':", str(self.elsize), suffix]
+            return space.wrap(''.join(s_as_list))
         else:
             descr = []
-            for name in self.names:
-                subdtype = self.fields[name][1]
-                subdescr = [space.wrap(name)]
+            total = 0
+            for name, title in self.names:
+                offset, subdtype = self.fields[name]
+                show_offsets = False
+                if total != offset and len(subdtype.shape) < 1:
+                    # whoops, need to use other format
+                    return self.descr_get_descr(space, style=style + '_subdtype', force_dict=True)
+                total += subdtype.elsize
+                ignore = '|'
+                if title:
+                    subdescr = [space.newtuple([space.wrap(title), space.wrap(name)])]
+                    ignore = ''
+                else:
+                    subdescr = [space.wrap(name)]
                 if subdtype.is_record():
-                    subdescr.append(subdtype.descr_get_descr(space))
+                    subdescr.append(subdtype.descr_get_descr(space, style))
                 elif subdtype.subdtype is not None:
-                    subdescr.append(subdtype.subdtype.descr_get_str(space))
+                    subdescr.append(subdtype.subdtype.descr_get_str(space, simple=False))
                 else:
-                    subdescr.append(subdtype.descr_get_str(space))
+                    subdescr.append(subdtype.descr_get_str(space, ignore=ignore, simple=False))
                 if subdtype.shape != []:
                     subdescr.append(subdtype.descr_get_shape(space))
                 descr.append(space.newtuple(subdescr[:]))
+            if self.alignment >= 0 and not style.endswith('subdtype'):
+                return space.wrap(space.str_w(space.repr(space.newlist(descr))) + ', align=True')                 
             return space.newlist(descr)
 
     def descr_get_hasobject(self, space):
-        return space.w_False
+        return space.wrap(self.is_object())
 
     def descr_get_isbuiltin(self, space):
         if self.fields is None:
@@ -238,19 +341,28 @@
     def descr_get_shape(self, space):
         return space.newtuple([space.wrap(dim) for dim in self.shape])
 
+    def descr_get_flags(self, space):
+        return space.wrap(self.flags)
+
     def descr_get_fields(self, space):
         if not self.fields:
             return space.w_None
         w_fields = space.newdict()
-        for name, (offset, subdtype) in self.fields.iteritems():
-            space.setitem(w_fields, space.wrap(name),
+        for name, title in self.names:
+            offset, subdtype = self.fields[name]
+            if title is not None:
+                w_nt = space.newtuple([space.wrap(name), space.wrap(title)]) 
+                space.setitem(w_fields, w_nt,
+                          space.newtuple([subdtype, space.wrap(offset)]))
+            else:
+                space.setitem(w_fields, space.wrap(name),
                           space.newtuple([subdtype, space.wrap(offset)]))
         return w_fields
 
     def descr_get_names(self, space):
         if not self.fields:
             return space.w_None
-        return space.newtuple([space.wrap(name) for name in self.names])
+        return space.newtuple([space.wrap(name[0]) for name in self.names])
 
     def descr_set_names(self, space, w_names):
         if not self.fields:
@@ -262,23 +374,43 @@
                         "with a sequence of length %d",
                         len(self.names))
         names = []
-        for w_name in space.fixedview(w_names):
+        names_w = space.fixedview(w_names)
+        for i in range(len(names_w)):
+            w_name = names_w[i]
+            title = self.names[i][1]
             if not space.isinstance_w(w_name, space.w_str):
                 raise oefmt(space.w_ValueError,
                             "item #%d of names is of type %T and not string",
                             len(names), w_name)
-            names.append(space.str_w(w_name))
+            names.append((space.str_w(w_name), title))
         fields = {}
         for i in range(len(self.names)):
-            if names[i] in fields:
+            if names[i][0] in fields:
                 raise oefmt(space.w_ValueError, "Duplicate field names given.")
-            fields[names[i]] = self.fields[self.names[i]]
+            fields[names[i][0]] = self.fields[self.names[i][0]]
+            if self.names[i][1] is not None:
+                fields[self.names[i][1]] = self.fields[self.names[i][0]]
         self.fields = fields
         self.names = names
 
     def descr_del_names(self, space):
-        raise OperationError(space.w_AttributeError, space.wrap(
-            "Cannot delete dtype names attribute"))
+        raise oefmt(space.w_AttributeError, 
+            "Cannot delete dtype names attribute")
+
+    def descr_get_metadata(self, space):
+        if self.metadata is None:
+            return space.w_None
+        return self.metadata
+
+    def descr_set_metadata(self, space, w_metadata):
+        if w_metadata is None:
+            return
+        if not space.isinstance_w(w_metadata, space.w_dict):
+            raise oefmt(space.w_TypeError, "argument 4 must be dict, not str")
+        self.metadata = w_metadata
+
+    def descr_del_metadata(self, space):
+        self.metadata = None
 
     def eq(self, space, w_other):
         w_other = space.call_function(space.gettypefor(W_Dtype), w_other)
@@ -331,7 +463,8 @@
                 y = intmask((1000003 * y) ^ self.alignment)
             return intmask((1000003 * x) ^ y)
         if self.fields:
-            for name, (offset, subdtype) in self.fields.iteritems():
+            for name in self.fields.keys():
+                offset, subdtype = self.fields[name]
                 assert isinstance(subdtype, W_Dtype)
                 y = intmask(1000003 * (0x345678 ^ compute_hash(name)))
                 y = intmask(1000003 * (y ^ compute_hash(offset)))
@@ -349,7 +482,7 @@
 
     def descr_str(self, space):
         if self.fields:
-            return space.str(self.descr_get_descr(space))
+            return space.str(self.descr_get_descr(space, style='str'))
         elif self.subdtype is not None:
             return space.str(space.newtuple([
                 self.subdtype.descr_get_str(space),
@@ -362,7 +495,7 @@
 
     def descr_repr(self, space):
         if self.fields:
-            r = self.descr_get_descr(space)
+            r = self.descr_get_descr(space, style='repr')
         elif self.subdtype is not None:
             r = space.newtuple([self.subdtype.descr_get_str(space),
                                 self.descr_get_shape(space)])
@@ -375,9 +508,11 @@
                 size = self.elsize
                 if self.num == NPY.UNICODE:
                     size >>= 2
-                r = space.wrap(byteorder + self.char + str(size))
+                r = space.wrap("'" + byteorder + self.char + str(size) + "'")
             else:
-                r = self.descr_get_name(space)
+                r = self.descr_get_name(space, quote=True)
+        if space.isinstance_w(r, space.w_str):
+            return space.wrap("dtype(%s)" % space.str_w(r))
         return space.wrap("dtype(%s)" % space.str_w(space.repr(r)))
 
     def descr_getitem(self, space, w_item):
@@ -389,7 +524,7 @@
         elif space.isinstance_w(w_item, space.w_int):
             indx = space.int_w(w_item)
             try:
-                item = self.names[indx]
+                item,title = self.names[indx]
             except IndexError:
                 raise oefmt(space.w_IndexError,
                     "Field index %d out of range.", indx)
@@ -436,14 +571,17 @@
         values = self.descr_get_fields(space)
         if self.is_flexible():
             w_size = space.wrap(self.elsize)
-            alignment = space.wrap(self.alignment)
+            if self.alignment > 2:
+                w_alignment = space.wrap(self.alignment)
+            else:
+                w_alignment = space.wrap(1)
         else:
             w_size = space.wrap(-1)
-            alignment = space.wrap(-1)
-        flags = space.wrap(0)
+            w_alignment = space.wrap(-1)
+        w_flags = space.wrap(self.flags)
 
         data = space.newtuple([version, space.wrap(endian), subdescr,
-                               names, values, w_size, alignment, flags])
+                               names, values, w_size, w_alignment, w_flags])
         return space.newtuple([w_class, builder_args, data])
 
     def descr_setstate(self, space, w_data):
@@ -465,6 +603,9 @@
         w_fields = space.getitem(w_data, space.wrap(4))
         size = space.int_w(space.getitem(w_data, space.wrap(5)))
         alignment = space.int_w(space.getitem(w_data, space.wrap(6)))
+        if alignment < 2:
+            alignment = -1
+        flags = space.int_w(space.getitem(w_data, space.wrap(7)))
 
         if (w_names == space.w_None) != (w_fields == space.w_None):
             raise oefmt(space.w_ValueError, "inconsistent fields and names in Numpy dtype unpickling")
@@ -492,20 +633,21 @@
             self.names = []
             self.fields = {}
             for w_name in space.fixedview(w_names):
+                # XXX what happens if there is a title in the pickled dtype?
                 name = space.str_w(w_name)
                 value = space.getitem(w_fields, w_name)
 
                 dtype = space.getitem(value, space.wrap(0))
+                offset = space.int_w(space.getitem(value, space.wrap(1)))
+                self.names.append((name, None))
                 assert isinstance(dtype, W_Dtype)
-                offset = space.int_w(space.getitem(value, space.wrap(1)))
-
-                self.names.append(name)
                 self.fields[name] = offset, dtype
             self.itemtype = types.RecordType(space)
 
         if self.is_flexible():
             self.elsize = size
             self.alignment = alignment
+        self.flags = flags
 
     @unwrap_spec(new_order=str)
     def descr_newbyteorder(self, space, new_order=NPY.SWAP):
@@ -526,17 +668,24 @@
 
 
 @specialize.arg(2)
-def dtype_from_list(space, w_lst, simple, align=False):
+def dtype_from_list(space, w_lst, simple, alignment, offsets=None, itemsize=0):
     lst_w = space.listview(w_lst)
     fields = {}
-    offset = 0
-    names = []
-    maxalign = 0
+    use_supplied_offsets = True
+    if offsets is None:
+        use_supplied_offsets = False
+        offsets = [0] * len(lst_w)
+    maxalign = alignment 
+    fldnames = [''] * len(lst_w)
+    subdtypes = [None] * len(lst_w)
+    titles = [None] * len(lst_w)
+    total = 0
     for i in range(len(lst_w)):
         w_elem = lst_w[i]
         if simple:
-            subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_elem)
-            fldname = 'f%d' % i
+            subdtype = make_new_dtype(space, space.gettypefor(W_Dtype), w_elem,
+                                    maxalign)
+            fldnames[i] = 'f%d' % i
         else:
             w_shape = space.newtuple([])
             if space.len_w(w_elem) == 3:
@@ -545,52 +694,213 @@
                     w_shape = space.newtuple([w_shape])
             else:
                 w_fldname, w_flddesc = space.fixedview(w_elem, 2)
-            subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc, w_shape=w_shape)
-            fldname = space.str_w(w_fldname)
-            if fldname == '':
-                fldname = 'f%d' % i
-            if fldname in fields:
+            subdtype = make_new_dtype(space, space.gettypefor(W_Dtype),
+                                    w_flddesc, maxalign, w_shape=w_shape)
+            if space.isinstance_w(w_fldname, space.w_tuple):
+                fldlist = space.listview(w_fldname)
+                fldnames[i] = space.str_w(fldlist[0])
+                if space.is_w(fldlist[1], space.w_None):
+                    titles[i] = None
+                else:
+                    titles[i] = space.str_w(fldlist[1])
+                if len(fldlist) != 2:
+                    raise oefmt(space.w_TypeError, "data type not understood")
+            elif space.isinstance_w(w_fldname, space.w_str): 
+                fldnames[i] = space.str_w(w_fldname)
+            else:
+                raise oefmt(space.w_TypeError, "data type not understood")
+            if fldnames[i] == '':
+                fldnames[i] = 'f%d' % i
+        assert isinstance(subdtype, W_Dtype)
+        if alignment >= 0:
+            maxalign = max(subdtype.alignment, maxalign)
+            delta = subdtype.alignment
+            # Set offset to the next power-of-two above delta
+            delta = (delta + maxalign -1) & (-maxalign)
+            if not use_supplied_offsets:
+                if delta > offsets[i]:
+                    for j in range(i):
+                        offsets[j+1] = delta + offsets[j]
+                if  i + 1 < len(offsets) and offsets[i + 1] == 0:
+                    offsets[i + 1] = offsets[i] + max(delta, subdtype.elsize)
+        elif not use_supplied_offsets:
+            if  i + 1 < len(offsets) and offsets[i + 1] == 0:
+                offsets[i+1] = offsets[i] + subdtype.elsize
+        subdtypes[i] = subdtype
+        if use_supplied_offsets:
+            sz = subdtype.elsize
+        else:
+            sz = max(maxalign, subdtype.elsize)
+        if offsets[i] + sz > total:
+            total = offsets[i] + sz
+    # padding?
+    if alignment >= 0 and total % maxalign:
+        total = total // maxalign * maxalign + maxalign
+    names = []
+    for i in range(len(subdtypes)):
+        subdtype = subdtypes[i]
+        assert isinstance(subdtype, W_Dtype)
+        if alignment >=0 and subdtype.is_record():
+            subdtype.alignment = maxalign
+        if fldnames[i] in fields:
+            raise oefmt(space.w_ValueError, "two fields with the same name")
+        if maxalign > 1 and offsets[i] % subdtype.alignment:
+            raise oefmt(space.w_ValueError, "offset %d for NumPy dtype with "
+                    "fields is not divisible by the field alignment %d "
+                    "with align=True", offsets[i], maxalign)
+        fields[fldnames[i]] = offsets[i], subdtype
+        if titles[i] is not None:
+            if titles[i] in fields:
                 raise oefmt(space.w_ValueError, "two fields with the same name")
-        assert isinstance(subdtype, W_Dtype)
-        fields[fldname] = (offset, subdtype)
-        offset += subdtype.elsize
-        maxalign = max(subdtype.elsize, maxalign)
-        names.append(fldname)
+            fields[titles[i]] = offsets[i], subdtype
+        names.append((fldnames[i], titles[i]))
+    if itemsize > 1:
+        if total > itemsize:
+            raise oefmt(space.w_ValueError,
+                     "NumPy dtype descriptor requires %d bytes, cannot"
+                     " override to smaller itemsize of %d", total, itemsize)
+        if alignment >= 0 and itemsize % maxalign:
+            raise oefmt(space.w_ValueError,
+                    "NumPy dtype descriptor requires alignment of %d bytes, "
+                    "which is not divisible into the specified itemsize %d",
+                    maxalign, itemsize) 
+        total = itemsize
+    retval = W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
+                   names=names, fields=fields, elsize=total)
+    if alignment >=0:
+        retval.alignment = maxalign
+    else:
+        retval.alignment = -1
+    retval.flags |= NPY.NEEDS_PYAPI
+    return retval
+
+def _get_val_or_none(space, w_dict, key):
+    w_key = space.wrap(key)
+    try:
+        w_val = space.getitem(w_dict, w_key)
+    except OperationError as e:
+        if e.match(space, space.w_KeyError):
+            return None
+        else:
+            raise
+    return w_val
+
+def _get_list_or_none(space, w_dict, key):
+    w_val = _get_val_or_none(space, w_dict, key)
+    if w_val is None:
+        return None
+    if space.isinstance_w(w_val, space.w_set):
+        raise oefmt(space.w_TypeError, "'set' object does not support indexing")
+    return space.listview(w_val)
+
+def _usefields(space, w_dict, align):
+    # Only for testing, a shortened version of the real _usefields
+    allfields = []
+    for fname in w_dict.iterkeys().iterator:
+        obj = _get_list_or_none(space, w_dict, fname)
+        num = space.int_w(obj[1])
+        if align:
+            alignment = 0
+        else:
+            alignment = -1
+        format = dtype_from_spec(space, obj[0], alignment=alignment)
+        if len(obj) > 2:
+            title = space.wrap(obj[2])
+        else:
+            title = space.w_None
+        allfields.append((space.wrap(fname), format, num, title))
+    allfields.sort(key=lambda x: x[2])
+    names   = [space.newtuple([x[0], x[3]]) for x in allfields]
+    formats = [x[1] for x in allfields]
+    offsets = [x[2] for x in allfields]
+    aslist = []
     if align:
-        # Set offset to the next power-of-two above offset
-        offset = (offset + maxalign -1) & (-maxalign)
-    return W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox),
-                   names=names, fields=fields, elsize=offset)
+        alignment = 0
+    else:
+        alignment = -1
+    for i in range(len(names)):
+        aslist.append(space.newtuple([space.wrap(names[i]), space.wrap(formats[i])]))
+    return dtype_from_list(space, space.newlist(aslist), False, alignment, offsets=offsets)
+    
+def dtype_from_dict(space, w_dict, alignment):
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    assert isinstance(w_dict, W_DictMultiObject)
+    names_w = _get_list_or_none(space, w_dict, 'names')
+    formats_w = _get_list_or_none(space, w_dict, 'formats') 
+    offsets_w = _get_list_or_none(space, w_dict, 'offsets')
+    titles_w = _get_list_or_none(space, w_dict, 'titles')
+    metadata_w = _get_val_or_none(space, w_dict, 'metadata')
+    aligned_w = _get_val_or_none(space, w_dict, 'align')
+    itemsize_w = _get_val_or_none(space, w_dict, 'itemsize')
+    if names_w is None or formats_w is None:
+        if we_are_translated():
+            return get_appbridge_cache(space).call_method(space,
+                'numpy.core._internal', '_usefields', Arguments(space, 
+                                [w_dict, space.wrap(alignment >= 0)]))
+        else:
+            return _usefields(space, w_dict, alignment >= 0)
+    n = len(names_w)
+    if (n != len(formats_w) or 
+        (offsets_w is not None and n != len(offsets_w)) or
+        (titles_w is not None and n != len(titles_w))):
+        raise oefmt(space.w_ValueError, "'names', 'formats', 'offsets', and "
+            "'titles' dicct entries must have the same length")
+    if aligned_w is not None:
+        if space.isinstance_w(aligned_w, space.w_bool) and space.is_true(aligned_w):
+            if alignment < 0:
+                alignment = 0 
+        else:
+            raise oefmt(space.w_ValueError,
+                    "NumPy dtype descriptor includes 'aligned' entry, "
+                    "but its value is neither True nor False");
+    if offsets_w is None:
+        offsets = None
+    else:
+        offsets = [space.int_w(i) for i in offsets_w]
+    if titles_w is not None:
+        _names_w = []
+        for i in range(min(len(names_w), len(titles_w))):
+            _names_w.append(space.newtuple([names_w[i], titles_w[i]]))
+        names_w = _names_w
+    aslist = []
+    if itemsize_w is None:
+        itemsize = 0
+    else:
+        itemsize = space.int_w(itemsize_w)
+    for i in range(min(len(names_w), len(formats_w))):
+        aslist.append(space.newtuple([names_w[i], formats_w[i]]))
+    retval = dtype_from_list(space, space.newlist(aslist), False, alignment,
+                             offsets=offsets, itemsize=itemsize)
+    if metadata_w is not None:
+        retval.descr_set_metadata(space, metadata_w)
+    retval.flags |= NPY.NEEDS_PYAPI
+    return retval 
 
-
-def dtype_from_dict(space, w_dict):
-    raise OperationError(space.w_NotImplementedError, space.wrap(
-        "dtype from dict"))
-
-
-def dtype_from_spec(space, w_spec):
+def dtype_from_spec(space, w_spec, alignment):
 
     if we_are_translated():
         w_lst = get_appbridge_cache(space).call_method(space,
             'numpy.core._internal', '_commastring', Arguments(space, [w_spec]))
     else:
-        # testing, handle manually
-        if space.eq_w(w_spec, space.wrap('u4,u4,u4')):
-            w_lst = space.newlist([space.wrap('u4')]*3)
-        if space.eq_w(w_spec, space.wrap('u4,u4,u4')):
-            w_lst = space.newlist([space.wrap('u4')]*3)
-        else:
-            raise oefmt(space.w_RuntimeError,
-                    "cannot parse w_spec")
+        # handle only simple cases for testing
+        if space.isinstance_w(w_spec, space.w_str):
+            spec = [s.strip() for s in space.str_w(w_spec).split(',')]
+            w_lst = space.newlist([space.wrap(s) for s in spec]) 
+        elif space.isinstance_w(w_spec, space.w_list):
+            w_lst = w_spec
     if not space.isinstance_w(w_lst, space.w_list) or space.len_w(w_lst) < 1:
         raise oefmt(space.w_RuntimeError,
                     "_commastring is not returning a list with len >= 1")
     if space.len_w(w_lst) == 1:
         return descr__new__(space, space.gettypefor(W_Dtype),
-                            space.getitem(w_lst, space.wrap(0)))
+                            space.getitem(w_lst, space.wrap(0)), align=alignment>0)
     else:
-        return dtype_from_list(space, w_lst, True)
-
+        try:
+            return dtype_from_list(space, w_lst, True, alignment)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return dtype_from_list(space, w_lst, False, alignment)
+            raise
 
 def _check_for_commastring(s):
     if s[0] in string.digits or s[0] in '<>=|' and s[1] in string.digits:
@@ -608,30 +918,83 @@
             sqbracket -= 1
     return False
 
- at unwrap_spec(align=bool)
-def descr__new__(space, w_subtype, w_dtype, align=False, w_copy=None, w_shape=None):
-    # align and w_copy are necessary for pickling
+def _set_metadata_and_copy(space, w_metadata, dtype, copy=False):
     cache = get_dtype_cache(space)
+    assert isinstance(dtype, W_Dtype)
+    if copy or (dtype in cache.builtin_dtypes and w_metadata is not None):
+        dtype = W_Dtype(dtype.itemtype, dtype.w_box_type, dtype.byteorder)
+    if w_metadata is not None:
+        dtype.descr_set_metadata(space, w_metadata)
+    return dtype
 
-    if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or
-                                space.len_w(w_shape) > 0):
-        subdtype = descr__new__(space, w_subtype, w_dtype, align, w_copy)
+def _get_shape(space, w_shape):
+    if w_shape is None:
+        return None
+    if space.isinstance_w(w_shape, space.w_int):
+        dim = space.int_w(w_shape)
+        if dim == 1:
+            return None
+        return [dim]
+    shape_w = space.fixedview(w_shape)
+    if len(shape_w) < 1:
+        return None
+    elif len(shape_w) == 1 and space.isinstance_w(shape_w[0], space.w_tuple):
+        # (base_dtype, new_dtype) dtype spectification
+        return None
+    shape = []
+    for w_dim in shape_w:
+        try:
+            dim = space.int_w(w_dim)
+        except OperationError as e:
+            if e.match(space, space.w_OverflowError):
+                raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple.")
+            else:
+                raise
+        if dim > int(0x7fffffff):
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                      "dimension does not fit into a C int.")
+        elif dim < 0:
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                  "dimension smaller than zero.")
+        shape.append(dim)
+    return shape
+
+ at unwrap_spec(align=bool, copy=bool)
+def descr__new__(space, w_subtype, w_dtype, align=False, copy=False,
+                 w_shape=None, w_metadata=None):
+    if align:
+        alignment = 0
+    else:
+        alignment = -1
+    return make_new_dtype(space, w_subtype, w_dtype, alignment, copy=copy,
+                          w_shape=w_shape, w_metadata=w_metadata)
+
+def make_new_dtype(space, w_subtype, w_dtype, alignment, copy=False, w_shape=None, w_metadata=None):
+    cache = get_dtype_cache(space)
+    shape = _get_shape(space, w_shape)
+    if shape is not None:
+        subdtype = make_new_dtype(space, w_subtype, w_dtype, alignment, copy, w_metadata=w_metadata)
         assert isinstance(subdtype, W_Dtype)
-        size = 1
-        if space.isinstance_w(w_shape, space.w_int):
-            w_shape = space.newtuple([w_shape])
-        shape = []
-        for w_dim in space.fixedview(w_shape):
-            dim = space.int_w(w_dim)
-            shape.append(dim)
-            size *= dim
-        if size == 1:
-            return subdtype
+        size = support.product(shape)
         size *= subdtype.elsize
-        return W_Dtype(types.VoidType(space),
-                       space.gettypefor(boxes.W_VoidBox),
-                       shape=shape, subdtype=subdtype, elsize=size)
-
+        if size > int(0x7fffffff):
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                  "dtype size in bytes must fit into a C int.")
+        
+        return _set_metadata_and_copy(space, w_metadata,
+               W_Dtype(types.VoidType(space), space.gettypefor(boxes.W_VoidBox),
+                       shape=shape, subdtype=subdtype, elsize=size))
+    elif w_shape is not None and not space.isinstance_w(w_shape, space.w_int):
+        spec = space.listview(w_shape)
+        if len(spec) > 0:
+            # this is (base_dtype, new_dtype) so just make it a union by setting both
+            # parts' offset to 0
+            try:
+                dtype1 = make_new_dtype(space, w_subtype, w_shape, alignment)
+            except:
+                raise
+            raise oefmt(space.w_NotImplementedError, 
+                "(base_dtype, new_dtype) dtype spectification discouraged, not implemented")
     if space.is_none(w_dtype):
         return cache.w_float64dtype
     if space.isinstance_w(w_dtype, w_subtype):
@@ -641,7 +1004,8 @@
     if space.isinstance_w(w_dtype, space.w_str):
         name = space.str_w(w_dtype)
         if _check_for_commastring(name):
-            return dtype_from_spec(space, w_dtype)
+            return _set_metadata_and_copy(space, w_metadata,
+                                dtype_from_spec(space, w_dtype, alignment))
         cname = name[1:] if name[0] == NPY.OPPBYTE else name
         try:
             dtype = cache.dtypes_by_name[cname]
@@ -655,26 +1019,34 @@
             return variable_dtype(space, name)
         raise oefmt(space.w_TypeError, 'data type "%s" not understood', name)
     elif space.isinstance_w(w_dtype, space.w_list):
-        return dtype_from_list(space, w_dtype, False, align=align)
+        return _set_metadata_and_copy( space, w_metadata,
+                        dtype_from_list(space, w_dtype, False, alignment), copy)
     elif space.isinstance_w(w_dtype, space.w_tuple):
         w_dtype0 = space.getitem(w_dtype, space.wrap(0))
         w_dtype1 = space.getitem(w_dtype, space.wrap(1))
-        subdtype = descr__new__(space, w_subtype, w_dtype0, align, w_copy)
+        subdtype = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy)
         assert isinstance(subdtype, W_Dtype)
         if subdtype.elsize == 0:
             name = "%s%d" % (subdtype.kind, space.int_w(w_dtype1))
-            return descr__new__(space, w_subtype, space.wrap(name), align, w_copy)
-        return descr__new__(space, w_subtype, w_dtype0, align, w_copy, w_shape=w_dtype1)
+            retval = make_new_dtype(space, w_subtype, space.wrap(name), alignment, copy)
+        else:
+            retval = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy, w_shape=w_dtype1)
+        return _set_metadata_and_copy(space, w_metadata, retval, copy)
     elif space.isinstance_w(w_dtype, space.w_dict):
-        return dtype_from_dict(space, w_dtype)
+        return _set_metadata_and_copy(space, w_metadata,
+                dtype_from_dict(space, w_dtype, alignment), copy)
     for dtype in cache.builtin_dtypes:
         if dtype.num in cache.alternate_constructors and \
                 w_dtype in cache.alternate_constructors[dtype.num]:
-            return dtype
+            return _set_metadata_and_copy(space, w_metadata, dtype, copy)
         if w_dtype is dtype.w_box_type:
-            return dtype
+            return _set_metadata_and_copy(space, w_metadata, dtype, copy)
+        if space.isinstance_w(w_dtype, space.w_type) and \
+           space.is_true(space.issubtype(w_dtype, dtype.w_box_type)):
+            return _set_metadata_and_copy( space, w_metadata,
+                            W_Dtype(dtype.itemtype, w_dtype, elsize=0), copy)
     if space.isinstance_w(w_dtype, space.w_type):
-        return cache.w_objectdtype
+        return _set_metadata_and_copy(space, w_metadata, cache.w_objectdtype, copy)
     raise oefmt(space.w_TypeError, "data type not understood")
 
 
@@ -702,6 +1074,11 @@
     names = GetSetProperty(W_Dtype.descr_get_names,
                            W_Dtype.descr_set_names,
                            W_Dtype.descr_del_names),
+    metadata = GetSetProperty(W_Dtype.descr_get_metadata,
+                           #W_Dtype.descr_set_metadata,
+                           #W_Dtype.descr_del_metadata,
+                            ),
+    flags = GetSetProperty(W_Dtype.descr_get_flags),
 
     __eq__ = interp2app(W_Dtype.descr_eq),
     __ne__ = interp2app(W_Dtype.descr_ne),
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -258,7 +258,7 @@
         if field not in dtype.fields:
             raise oefmt(space.w_ValueError, "field named %s not found", field)
         arr = self.implementation
-        ofs, subdtype = arr.dtype.fields[field]
+        ofs, subdtype = arr.dtype.fields[field][:2]
         # ofs only changes start
         # create a view of the original array by extending
         # the shape, strides, backstrides of the array
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -92,6 +92,7 @@
         assert d == np.dtype('i8')
         assert d.shape == ()
         d = np.dtype((np.int64, 1,))
+        assert d.shape == ()
         assert d == np.dtype('i8')
         assert d.shape == ()
         d = np.dtype((np.int64, 4))
@@ -111,6 +112,7 @@
         assert "int8" == dtype("int8")
         raises(TypeError, lambda: dtype("int8") == 3)
         assert dtype(bool) == bool
+        assert dtype('f8') != dtype(('f8', (1,)))
 
     def test_dtype_cmp(self):
         from numpy import dtype
@@ -342,10 +344,10 @@
         raises(TypeError, type, "Foo", (dtype,), {})
 
     def test_can_subclass(self):
-        import numpy
-        class xyz(numpy.void):
+        import numpy as np
+        class xyz(np.void):
             pass
-        assert True
+        assert np.dtype(xyz).name == 'xyz'
 
     def test_index(self):
         import numpy as np
@@ -413,7 +415,7 @@
         assert loads(dumps(a.dtype)) == a.dtype
         assert np.dtype('bool').__reduce__() == (dtype, ('b1', 0, 1), (3, '|', None, None, None, -1, -1, 0))
         assert np.dtype('|V16').__reduce__() == (dtype, ('V16', 0, 1), (3, '|', None, None, None, 16, 1, 0))
-        assert np.dtype(('<f8', 2)).__reduce__() == (dtype, ('V16', 0, 1), (3, '|', (dtype('float64'), (2,)), None, None, 16, 1, 0))
+        assert np.dtype(('<f8', 2)).__reduce__() == (dtype, ('V16', 0, 1), (3, '|', (dtype('float64'), (2,)), None, None, 16, 8, 0))
 
     def test_newbyteorder(self):
         import numpy as np
@@ -480,13 +482,21 @@
         class O(object):
             pass
         for o in [object, O]:
-            print np.dtype(o).byteorder
             if self.ptr_size == 4:
                 assert np.dtype(o).str == '|O4'
             elif self.ptr_size == 8:
                 assert np.dtype(o).str == '|O8'
             else:
                 assert False,'self._ptr_size unknown'
+        # Issue gh-2798
+        if '__pypy__' in sys.builtin_module_names:
+            a = np.array(['a'], dtype="O")
+            raises(NotImplementedError, a.astype, ("O", [("name", "O")]))
+            skip("(base_dtype, new_dtype) dtype specification discouraged")
+        a = np.array(['a'], dtype="O").astype(("O", [("name", "O")]))
+        assert a[0] == 'a'
+        assert a == 'a'
+        assert a['name'].dtype == a.dtype
 
 class AppTestTypes(BaseAppTestDtypes):
     def test_abstract_types(self):
@@ -686,16 +696,8 @@
                                       numpy.integer, numpy.number,
                                       numpy.generic, object]
         import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            # These tests pass "by chance" on numpy, things that are larger than
-            # platform long (i.e. a python int), don't get put in a normal box,
-            # instead they become an object array containing a long, we don't have
-            # yet, so these can't pass.
-            assert numpy.uint64(9223372036854775808) == 9223372036854775808
-            assert numpy.uint64(18446744073709551615) == 18446744073709551615
-        else:
-            raises(OverflowError, numpy.int64, 9223372036854775808)
-            raises(OverflowError, numpy.int64, 18446744073709551615)
+        raises(OverflowError, numpy.int64, 9223372036854775808)
+        raises(OverflowError, numpy.int64, 18446744073709551615)
         raises(OverflowError, numpy.uint64, 18446744073709551616)
         assert numpy.uint64((2<<63) - 1) == (2<<63) - 1
 
@@ -1136,10 +1138,7 @@
         import sys
         d = {'names': ['r','g','b','a'],
              'formats': [np.uint8, np.uint8, np.uint8, np.uint8]}
-        if '__pypy__' not in sys.builtin_module_names:
-            dt = np.dtype(d)
-        else:
-            raises(NotImplementedError, np.dtype, d)
+        dt = np.dtype(d)
 
     def test_create_subarrays(self):
         from numpy import dtype
@@ -1272,7 +1271,7 @@
                      ('x', 'y', 'z', 'value'),
                      {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0),
                       'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12),
-                      }, 20, 1, 0))
+                      }, 20, 1, 16))
 
         new_d = loads(dumps(d))
 
@@ -1291,6 +1290,182 @@
 
         assert new_d.itemsize == d.itemsize == 76
 
+    def test_shape_invalid(self):
+        import numpy as np
+        # Check that the shape is valid.
+        max_int = 2 ** (8 * 4 - 1)
+        max_intp = 2 ** (8 * np.dtype('intp').itemsize - 1) - 1
+        # Too large values (the datatype is part of this)
+        raises(ValueError, np.dtype, [('a', 'f4', max_int // 4 + 1)])
+        raises(ValueError, np.dtype, [('a', 'f4', max_int + 1)])
+        raises(ValueError, np.dtype, [('a', 'f4', (max_int, 2))])
+        # Takes a different code path (fails earlier:
+        raises(ValueError, np.dtype, [('a', 'f4', max_intp + 1)])
+        # Negative values
+        raises(ValueError, np.dtype, [('a', 'f4', -1)])
+        raises(ValueError, np.dtype, [('a', 'f4', (-1, -1))])
+
+    def test_aligned_size(self):
+        import numpy as np
+        # Check that structured dtypes get padded to an aligned size
+        dt = np.dtype('i4, i1', align=True)
+        assert dt.itemsize == 8
+        dt = np.dtype([('f0', 'i4'), ('f1', 'i1')], align=True)
+        assert dt.itemsize == 8
+        dt = np.dtype({'names':['f0', 'f1'],
+                       'formats':['i4', 'u1'],
+                       'offsets':[0, 4]}, align=True)
+        assert dt.itemsize == 8
+        dt = np.dtype({'f0': ('i4', 0), 'f1':('u1', 4)}, align=True)
+        assert dt.itemsize == 8
+        assert dt.alignment == 4
+        assert str(dt) == "{'names':['f0','f1'], 'formats':['<i4','u1'], 'offsets':[0,4], 'itemsize':8, 'aligned':True}"
+        dt = np.dtype([('f1', 'u1'), ('f0', 'i4')], align=True)
+        assert str(dt) == "{'names':['f1','f0'], 'formats':['u1','<i4'], 'offsets':[0,4], 'itemsize':8, 'aligned':True}"
+        # Nesting should preserve that alignment
+        dt1 = np.dtype([('f0', 'i4'),
+                       ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
+                       ('f2', 'i1')], align=True)
+        assert dt1.alignment == 4
+        assert dt1['f1'].itemsize == 12
+        assert dt1.itemsize == 20
+        dt2 = np.dtype({'names':['f0', 'f1', 'f2'],
+                       'formats':['i4',
+                                  [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')],
+                                  'i1'],
+                       'offsets':[0, 4, 16]}, align=True)
+        assert dt2.itemsize == 20
+        dt3 = np.dtype({'f0': ('i4', 0),
+                       'f1': ([('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')], 4),
+                       'f2': ('i1', 16)}, align=True)
+        assert dt3.itemsize == 20
+        assert dt1 == dt2
+        answer = "{'names':['f0','f1','f2'], " + \
+                    "'formats':['<i4',{'names':['f1','f2','f3'], " + \
+                                      "'formats':['i1','<i4','i1'], " + \
+                                      "'offsets':[0,4,8], 'itemsize':12}," + \
+                                 "'i1'], " + \
+                    "'offsets':[0,4,16], 'itemsize':20, 'aligned':True}"
+        assert str(dt3) == answer
+        assert dt2 == dt3
+        # Nesting should preserve packing
+        dt1 = np.dtype([('f0', 'i4'),
+                       ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
+                       ('f2', 'i1')], align=False)
+        assert dt1.itemsize == 11
+        dt2 = np.dtype({'names':['f0', 'f1', 'f2'],
+                       'formats':['i4',
+                                  [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')],
+                                  'i1'],
+                       'offsets':[0, 4, 10]}, align=False)
+        assert dt2.itemsize == 11
+        dt3 = np.dtype({'f0': ('i4', 0),
+                       'f1': ([('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')], 4),
+                       'f2': ('i1', 10)}, align=False)
+        assert dt3.itemsize == 11
+        assert dt1 == dt2
+        assert dt2 == dt3
+
+    def test_bad_param(self):
+        import numpy as np
+        # Can't give a size that's too small
+        raises(ValueError, np.dtype,
+                        {'names':['f0', 'f1'],
+                         'formats':['i4', 'i1'],
+                         'offsets':[0, 4],
+                         'itemsize':4})
+        # If alignment is enabled, the alignment (4) must divide the itemsize
+        raises(ValueError, np.dtype,
+                        {'names':['f0', 'f1'],
+                         'formats':['i4', 'i1'],
+                         'offsets':[0, 4],
+                         'itemsize':9}, align=True)
+        # If alignment is enabled, the individual fields must be aligned
+        raises(ValueError, np.dtype,
+                        {'names':['f0', 'f1'],
+                         'formats':['i1', 'f4'],
+                         'offsets':[0, 2]}, align=True)
+        dt = np.dtype(np.double)
+        attr = ["subdtype", "descr", "str", "name", "base", "shape",
+                "isbuiltin", "isnative", "isalignedstruct", "fields",
+                "metadata", "hasobject"]
+        for s in attr:
+            raises(AttributeError, delattr, dt, s)
+
+        raises(TypeError, np.dtype,
+            dict(names=set(['A', 'B']), formats=['f8', 'i4']))
+        raises(TypeError, np.dtype,
+            dict(names=['A', 'B'], formats=set(['f8', 'i4'])))
+
+    def test_complex_dtype_repr(self):
+        import numpy as np
+        dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
+                                ('rtile', '>f4', (64, 36))], (3,)),
+                       ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
+                                   ('bright', '>f4', (8, 36))])])
+        assert repr(dt) == (
+                     "dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)), "
+                     "('rtile', '>f4', (64, 36))], (3,)), "
+                     "('bottom', [('bleft', ('>f4', (8, 64)), (1,)), "
+                     "('bright', '>f4', (8, 36))])])")
+
+        # If the sticky aligned flag is set to True, it makes the
+        # str() function use a dict representation with an 'aligned' flag
+        dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
+                                ('rtile', '>f4', (64, 36))],
+                                (3,)),
+                       ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
+                                   ('bright', '>f4', (8, 36))])],
+                       align=True)
+        assert str(dt) == (
+                    "{'names':['top','bottom'], "
+                     "'formats':[([('tiles', ('>f4', (64, 64)), (1,)), "
+                                  "('rtile', '>f4', (64, 36))], (3,)),"
+                                 "[('bleft', ('>f4', (8, 64)), (1,)), "
+                                  "('bright', '>f4', (8, 36))]], "
+                     "'offsets':[0,76800], "
+                     "'itemsize':80000, "
+                     "'aligned':True}")
+
+        dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'],
+                        'offsets': [0, 1, 2],
+                        'titles': ['Red pixel', 'Green pixel', 'Blue pixel']},
+                        align=True)
+        assert repr(dt) == (
+                    "dtype([(('Red pixel', 'r'), 'u1'), "
+                    "(('Green pixel', 'g'), 'u1'), "
+                    "(('Blue pixel', 'b'), 'u1')], align=True)")
+
+        dt = np.dtype({'names': ['rgba', 'r', 'g', 'b'],
+                       'formats': ['<u4', 'u1', 'u1', 'u1'],
+                       'offsets': [0, 0, 1, 2],
+                       'titles': ['Color', 'Red pixel',
+                                  'Green pixel', 'Blue pixel']}, align=True)
+        assert repr(dt) == (
+                    "dtype({'names':['rgba','r','g','b'],"
+                    " 'formats':['<u4','u1','u1','u1'],"
+                    " 'offsets':[0,0,1,2],"
+                    " 'titles':['Color','Red pixel',"
+                              "'Green pixel','Blue pixel'],"
+                    " 'itemsize':4}, align=True)")
+
+        dt = np.dtype({'names': ['r', 'b'], 'formats': ['u1', 'u1'],
+                        'offsets': [0, 2],
+                        'titles': ['Red pixel', 'Blue pixel'],
+                        'itemsize': 4})
+        assert repr(dt) == (
+                    "dtype({'names':['r','b'], "
+                    "'formats':['u1','u1'], "
+                    "'offsets':[0,2], "
+                    "'titles':['Red pixel','Blue pixel'], "
+                    "'itemsize':4})")
+        if 'datetime64' not in dir(np):
+            skip('datetime dtype not available')
+        dt = np.dtype([('a', '<M8[D]'), ('b', '<m8[us]')])
+        assert repr(dt) == (
+                    "dtype([('a', '<M8[D]'), ('b', '<m8[us]')])")
+
+
 class AppTestNotDirect(BaseNumpyAppTest):
     def setup_class(cls):
         BaseNumpyAppTest.setup_class.im_func(cls)
@@ -1332,4 +1507,45 @@
         assert a[0] == 1
         assert (a + a)[1] == 4
 
+class AppTestMonsterType(BaseNumpyAppTest):
+    """Test deeply nested subtypes."""
+    def test1(self):
+        import numpy as np
+        simple1 = np.dtype({'names': ['r', 'b'], 'formats': ['u1', 'u1'],
+            'titles': ['Red pixel', 'Blue pixel']})
+        a = np.dtype([('yo', int), ('ye', simple1),
+            ('yi', np.dtype((int, (3, 2))))])
+        b = np.dtype([('yo', int), ('ye', simple1),
+            ('yi', np.dtype((int, (3, 2))))])
+        assert a == b
 
+        c = np.dtype([('yo', int), ('ye', simple1),
+            ('yi', np.dtype((a, (3, 2))))])
+        d = np.dtype([('yo', int), ('ye', simple1),
+            ('yi', np.dtype((a, (3, 2))))])
+        assert c == d
+
+
+class AppTestMetadata(BaseNumpyAppTest):
+    def test_no_metadata(self):
+        import numpy as np
+        d = np.dtype(int)
+        assert d.metadata is None
+
+    def test_metadata_takes_dict(self):
+        import numpy as np
+        d = np.dtype(int, metadata={'datum': 1})
+        assert d.metadata == {'datum': 1}
+
+    def test_metadata_rejects_nondict(self):
+        import numpy as np
+        raises(TypeError, np.dtype, int, metadata='datum')
+        raises(TypeError, np.dtype, int, metadata=1)
+        raises(TypeError, np.dtype, int, metadata=None)
+
+    def test_nested_metadata(self):
+        import numpy as np
+        d = np.dtype([('a', np.dtype(int, metadata={'datum': 1}))])
+        assert d['a'].metadata == {'datum': 1}
+
+
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -764,6 +764,8 @@
         assert (a[1:] == b).all()
         assert (a[1:,newaxis] == d).all()
         assert (a[newaxis,1:] == c).all()
+        assert a.strides == (8,)
+        assert a[:, newaxis].strides == (8, 0)
 
     def test_newaxis_assign(self):
         from numpy import array, newaxis
@@ -2345,6 +2347,7 @@
         assert a[1] == 0xff
         assert len(a.data) == 16
         assert type(a.data) is buffer
+        assert a[1:].data._pypy_raw_address() - a.data._pypy_raw_address() == a.strides[0]
 
     def test_explicit_dtype_conversion(self):
         from numpy import array
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -2396,7 +2396,7 @@
         dt = item.arr.dtype
         ret_unwrapped = []
         for name in dt.names:
-            ofs, dtype = dt.fields[name]
+            ofs, dtype = dt.fields[name[0]]
             # XXX: code duplication with W_VoidBox.descr_getitem()
             if isinstance(dtype.itemtype, VoidType):
                 read_val = dtype.itemtype.readarray(item.arr, ofs, 0, dtype)
@@ -2431,14 +2431,14 @@
                 return w_item
             else:
                 # match up the field names
-                items_w = [None] * len(dtype.fields)
-                for i in range(len(dtype.fields)):
+                items_w = [None] * len(dtype.names)
+                for i in range(len(dtype.names)):
                     name = dtype.names[i]
                     if name in w_item.dtype.names:
-                        items_w[i] = w_item.descr_getitem(space, space.wrap(name))
+                        items_w[i] = w_item.descr_getitem(space, space.wrap(name[0]))
         elif w_item is not None:
             if space.isinstance_w(w_item, space.w_tuple):
-                if len(dtype.fields) != space.len_w(w_item):
+                if len(dtype.names) != space.len_w(w_item):
                     raise OperationError(space.w_ValueError, space.wrap(
                         "size of tuple must match number of fields."))
                 items_w = space.fixedview(w_item)
@@ -2449,12 +2449,12 @@
                             "expected a readable buffer object")
             else:
                 # XXX support initializing from readable buffers
-                items_w = [w_item] * len(dtype.fields)
+                items_w = [w_item] * len(dtype.names)
         else:
             items_w = [None] * len(dtype.fields)
         arr = VoidBoxStorage(dtype.elsize, dtype)
-        for i in range(len(dtype.fields)):
-            ofs, subdtype = dtype.fields[dtype.names[i]]
+        for i in range(len(dtype.names)):
+            ofs, subdtype = dtype.fields[dtype.names[i][0]]
             try:
                 w_box = subdtype.coerce(space, items_w[i])
             except IndexError:
@@ -2492,7 +2492,7 @@
         items = []
         dtype = box.dtype
         for name in dtype.names:
-            ofs, subdtype = dtype.fields[name]
+            ofs, subdtype = dtype.fields[name[0]]
             subbox = subdtype.read(box.arr, box.ofs, ofs)
             items.append(subdtype.itemtype.to_builtin_type(space, subbox))
         return space.newtuple(items)
@@ -2503,7 +2503,7 @@
         pieces = ["("]
         first = True
         for name in box.dtype.names:
-            ofs, subdtype = box.dtype.fields[name]
+            ofs, subdtype = box.dtype.fields[name[0]]
             if first:
                 first = False
             else:
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -72,6 +72,10 @@
     def get_module(self):
         return w_some_obj()
 
+
+    def getname(self, space):
+        return self.name
+
 def w_some_obj():
     if NonConstant(False):
         return W_Root()
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1031,6 +1031,48 @@
         A.__dict__['x'] = 5
         assert A.x == 5
 
+    def test_we_already_got_one_1(self):
+        # Issue #2079: highly obscure: CPython complains if we say
+        # ``__slots__="__dict__"`` and there is already a __dict__...
+        # but from the "best base" only.  If the __dict__ comes from
+        # another base, it doesn't complain.  Shrug and copy the logic.
+        class A(object):
+            __slots__ = ()
+        class B(object):
+            pass
+        class C(A, B):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(A, B):     # "best base" is A
+            __slots__ = ("__weakref__",)
+        try:
+            class E(B, A):   # "best base" is B
+                __slots__ = ("__dict__",)
+        except TypeError, e:
+            assert 'we already got one' in str(e)
+        else:
+            raise AssertionError("TypeError not raised")
+        try:
+            class F(B, A):   # "best base" is B
+                __slots__ = ("__weakref__",)
+        except TypeError, e:
+            assert 'we already got one' in str(e)
+        else:
+            raise AssertionError("TypeError not raised")
+
+    def test_we_already_got_one_2(self):
+        class A(object):
+            __slots__ = ()
+        class B:
+            pass
+        class C(A, B):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(A, B):     # "best base" is A
+            __slots__ = ("__weakref__",)
+        class C(B, A):     # "best base" is A
+            __slots__ = ("__dict__",)
+        class D(B, A):     # "best base" is A
+            __slots__ = ("__weakref__",)
+
 
 class AppTestWithMethodCacheCounter:
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1022,7 +1022,7 @@
     w_self.nslots = w_bestbase.nslots
     return hasoldstylebase
 
-def create_all_slots(w_self, hasoldstylebase):
+def create_all_slots(w_self, hasoldstylebase, w_bestbase):
     space = w_self.space
     dict_w = w_self.dict_w
     if '__slots__' not in dict_w:
@@ -1040,12 +1040,12 @@
         for w_slot_name in slot_names_w:
             slot_name = space.str_w(w_slot_name)
             if slot_name == '__dict__':
-                if wantdict or w_self.hasdict:
+                if wantdict or w_bestbase.hasdict:
                     raise oefmt(space.w_TypeError,
                                 "__dict__ slot disallowed: we already got one")
                 wantdict = True
             elif slot_name == '__weakref__':
-                if wantweakref or w_self.weakrefable:
+                if wantweakref or w_bestbase.weakrefable:
                     raise oefmt(space.w_TypeError,
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
@@ -1106,7 +1106,7 @@
         w_self.flag_abstract |= w_base.flag_abstract
 
     hasoldstylebase = copy_flags_from_bases(w_self, w_bestbase)
-    create_all_slots(w_self, hasoldstylebase)
+    create_all_slots(w_self, hasoldstylebase, w_bestbase)
 
     ensure_common_attributes(w_self)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8637,5 +8637,27 @@
         """
         self.optimize_loop(ops, expected, preamble)
 
+    def test_getfield_proven_constant(self):
+        py.test.skip("not working")
+        ops = """
+        [p0]
+        i1 = getfield_gc(p0, descr=valuedescr)
+        guard_value(i1, 13) []
+        escape(i1)
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        escape(13)
+        jump(p0)
+        """
+        expected_short = """
+        [p0]
+        i1 = getfield_gc(p0, descr=valuedescr)
+        guard_value(i1, 13) []
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected, expected_short=expected_short)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Fri Jul 17 17:19:48 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 17 Jul 2015 17:19:48 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: trace versioning now compiles each
 version just once, and attaches patches all further guards to the bridge
Message-ID: <20150717151948.1B3C71C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78584:0d1dc4ba30d3
Date: 2015-07-17 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/0d1dc4ba30d3/

Log:	trace versioning now compiles each version just once, and attaches
	patches all further guards to the bridge

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -69,6 +69,9 @@
         self.rtyper = cpu.rtyper
         self._debug = False
 
+    def attach_bridge(self, failargs, token):
+        raise NotImplementedError
+
     def setup_once(self):
         # the address of the function called by 'new'
         gc_ll_descr = self.cpu.gc_ll_descr
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -97,6 +97,9 @@
                                             inputargs, operations,
                                             looptoken, log=log)
 
+    def attach_bridge(self, faildescr, token):
+        self.assembler.attach_bridge(faildescr, token)
+
     def _setup_frame_realloc(self, translate_support_code):
         FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed],
                                              llmemory.GCREF))
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -589,6 +589,10 @@
                                                        rawstart, fullsize)
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
+    def attach_bridge(self, failargs, token):
+        rawstart = self.materialize_loop(token)
+        self.patch_jump_for_descr(faildescr, rawstart)
+
     def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
         # at the end of self.mc.
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -199,17 +199,19 @@
     if loop.versions is not None:
         token = jitcell_token
         for version in loop.versions:
-            for i, faildescr in enumerate(version.faildescrs):
-                vl = create_empty_loop(metainterp)
-                vl.inputargs = version.inputargs
-                vl.operations = version.operations
-                if i > 0:
-                    vl.operations = vl.copy_operations()
-                vl.original_jitcell_token = jitcell_token
-                send_bridge_to_backend(jitdriver_sd, metainterp_sd,
-                                       faildescr, version.inputargs,
-                                       version.operations, jitcell_token)
-                record_loop_or_bridge(metainterp_sd, vl)
+            if len(version.faildescrs) == 0:
+                continue
+            faildescr = version.faildescrs[0]
+            vl = create_empty_loop(metainterp)
+            vl.inputargs = version.inputargs
+            vl.operations = version.operations
+            vl.original_jitcell_token = jitcell_token
+            send_bridge_to_backend(jitdriver_sd, metainterp_sd,
+                                   faildescr, version.inputargs,
+                                   version.operations, jitcell_token)
+            record_loop_or_bridge(metainterp_sd, vl)
+            for faildescr in version.faildescrs[1:]:
+                cpu.attach_bridge(faildescr, jitcell_token)
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,

From noreply at buildbot.pypy.org  Fri Jul 17 17:30:15 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 17 Jul 2015 17:30:15 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: rpython hint,
	attach_bridge is now called stitch_bridge
Message-ID: <20150717153015.F2DAA1C1035@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78585:6b68821bc6e3
Date: 2015-07-17 17:30 +0200
http://bitbucket.org/pypy/pypy/changeset/6b68821bc6e3/

Log:	rpython hint, attach_bridge is now called stitch_bridge

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -69,7 +69,7 @@
         self.rtyper = cpu.rtyper
         self._debug = False
 
-    def attach_bridge(self, failargs, token):
+    def stitch_bridge(self, failargs, token):
         raise NotImplementedError
 
     def setup_once(self):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -97,8 +97,8 @@
                                             inputargs, operations,
                                             looptoken, log=log)
 
-    def attach_bridge(self, faildescr, token):
-        self.assembler.attach_bridge(faildescr, token)
+    def stitch_bridge(self, faildescr, token):
+        self.assembler.stitch_bridge(faildescr, token)
 
     def _setup_frame_realloc(self, translate_support_code):
         FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed],
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -589,7 +589,7 @@
                                                        rawstart, fullsize)
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
-    def attach_bridge(self, failargs, token):
+    def attach_bridge(self, faildescr, token):
         rawstart = self.materialize_loop(token)
         self.patch_jump_for_descr(faildescr, rawstart)
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -211,7 +211,7 @@
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
             for faildescr in version.faildescrs[1:]:
-                cpu.attach_bridge(faildescr, jitcell_token)
+                cpu.stitch_bridge(faildescr, jitcell_token)
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,

From noreply at buildbot.pypy.org  Fri Jul 17 18:11:30 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 18:11:30 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Fix test because repr of a class defined
 inside a function changed.
Message-ID: <20150717161130.87CFB1C1035@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78586:7305d753afde
Date: 2015-07-17 12:37 +0200
http://bitbucket.org/pypy/pypy/changeset/7305d753afde/

Log:	Fix test because repr of a class defined inside a function changed.

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -678,7 +678,7 @@
         globals()['__name__'] = 'a'
         class A(object):
             pass
-        assert repr(A) == "<class 'a.A'>"
+        assert repr(A) == "<class 'a.test_repr.<locals>.A'>"
         A.__module__ = 123
         assert repr(A) == "<class 'A'>"
         assert repr(type(type)) == "<class 'type'>" 

From noreply at buildbot.pypy.org  Fri Jul 17 18:11:31 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 18:11:31 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Implement improved function call
	exception messages.
Message-ID: <20150717161131.B72771C1035@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78587:cb29adb88801
Date: 2015-07-17 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/cb29adb88801/

Log:	Implement improved function call exception messages.

diff --git a/lib-python/3/test/test_extcall.py b/lib-python/3/test/test_extcall.py
--- a/lib-python/3/test/test_extcall.py
+++ b/lib-python/3/test/test_extcall.py
@@ -89,19 +89,19 @@
 
     >>> class Nothing: pass
     ...
-    >>> g(*Nothing())
+    >>> g(*Nothing())                     #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: g() argument after * must be a sequence, not Nothing
+    TypeError: ...argument after * must be a sequence, not Nothing
 
     >>> class Nothing:
     ...     def __len__(self): return 5
     ...
 
-    >>> g(*Nothing())
+    >>> g(*Nothing())                     #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: g() argument after * must be a sequence, not Nothing
+    TypeError: ...argument after * must be a sequence, not Nothing
 
     >>> class Nothing():
     ...     def __len__(self): return 5
@@ -153,52 +153,50 @@
       ...
     TypeError: g() got multiple values for argument 'x'
 
-    >>> f(**{1:2})
+    >>> f(**{1:2})                             #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: f() keywords must be strings
+    TypeError: ...keywords must be strings
 
     >>> h(**{'e': 2})
     Traceback (most recent call last):
       ...
     TypeError: h() got an unexpected keyword argument 'e'
 
-    >>> h(*h)
+    >>> h(*h)                                  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after * must be a sequence, not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> dir(*h)
+    >>> dir(*h)                                #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() argument after * must be a sequence, not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> None(*h)
+    >>> None(*h)                               #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: NoneType object argument after * must be a sequence, \
-not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> h(**h)
+    >>> h(**h)                                 #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> dir(**h)
+    >>> dir(**h)                               #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> None(**h)
+    >>> None(**h)                              #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: NoneType object argument after ** must be a mapping, \
-not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> dir(b=1, **{'b': 1})
+    >>> dir(b=1, **{'b': 1})                   #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() got multiple values for keyword argument 'b'
+    TypeError: ...got multiple values for keyword argument 'b'
 
 Another helper function
 
@@ -239,10 +237,10 @@
     ...     False
     True
 
-    >>> id(1, **{'foo': 1})
+    >>> id(1, **{'foo': 1})                 #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: id() takes no keyword arguments
+    TypeError: id() ... keyword argument...
 
 A corner case of keyword dictionary items being deleted during
 the function call setup. See <http://bugs.python.org/issue2016>.
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -4,6 +4,7 @@
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import enforceargs
+from rpython.rlib.rstring import StringBuilder
 
 from pypy.interpreter.error import OperationError, oefmt
 
@@ -210,7 +211,13 @@
             loc = co_argcount + co_kwonlyargcount
             scope_w[loc] = self.space.newtuple(starargs_w)
         elif avail > co_argcount:
-            raise ArgErrCount(avail, num_kwds, signature, defaults_w, w_kw_defs, 0)
+            kwonly_given = 0
+            for i in range(co_argcount, co_argcount + co_kwonlyargcount):
+                if scope_w[i] is None:
+                    kwonly_given += 1
+            raise ArgErrTooMany(signature.num_argnames(),
+                                0 if defaults_w is None else len(defaults_w),
+                                avail, kwonly_given)
 
         # if a **kwargs argument is needed, create the dict
         w_kwds = None
@@ -243,16 +250,13 @@
                             self.space, keywords, keywords_w, w_kwds,
                             kwds_mapping, self.keyword_names_w, self._jit_few_keywords)
                 else:
-                    if co_argcount == 0:
-                        raise ArgErrCount(avail, num_kwds, signature, defaults_w,
-                                          w_kw_defs, 0)
-                    
                     raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
                                             kwds_mapping, self.keyword_names_w)
 
         # check for missing arguments and fill them from the kwds,
         # or with defaults, if available
-        missing = 0
+        missing_positional = []
+        missing_kwonly = []
         if input_argcount < co_argcount + co_kwonlyargcount:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
             j = 0
@@ -273,25 +277,26 @@
                 if defnum >= 0:
                     scope_w[i] = defaults_w[defnum]
                 else:
-                    missing += 1
+                    missing_positional.append(signature.argnames[i])
 
             # finally, fill kwonly arguments with w_kw_defs (if needed)
             for i in range(co_argcount, co_argcount + co_kwonlyargcount):
                 if scope_w[i] is not None:
                     continue
-                elif w_kw_defs is None:
-                    missing += 1
+                name = signature.kwonlyargnames[i - co_argcount]
+                if w_kw_defs is None:
+                    missing_kwonly.append(name)
                     continue
-                name = signature.kwonlyargnames[i - co_argcount]
                 w_def = self.space.finditem_str(w_kw_defs, name)
                 if w_def is not None:
                     scope_w[i] = w_def
                 else:
-                    missing += 1
+                    missing_kwonly.append(name)
 
-        if missing:
-            raise ArgErrCount(avail, num_kwds, signature,
-                              defaults_w, w_kw_defs, missing)
+        if missing_positional:
+            raise ArgErrMissing(missing_positional, True)
+        if missing_kwonly:
+            raise ArgErrMissing(missing_kwonly, False)
 
 
     def parse_into_scope(self, w_firstarg,
@@ -461,61 +466,68 @@
     def getmsg(self):
         raise NotImplementedError
 
-class ArgErrCount(ArgErr):
 
-    def __init__(self, got_nargs, nkwds, signature,
-                 defaults_w, w_kw_defs, missing_args):
-        self.signature = signature
-        self.num_defaults = 0 if defaults_w is None else len(defaults_w)
-        self.missing_args = missing_args
-        self.num_args = got_nargs
-        self.num_kwds = nkwds
+class ArgErrMissing(ArgErr):
+    def __init__(self, missing, positional):
+        self.missing = missing
+        self.positional = positional  # keyword-only otherwise
 
     def getmsg(self):
-        n = self.signature.num_argnames()
-        if n == 0:
-            msg = "takes no arguments (%d given)" % (
-                self.num_args + self.num_kwds)
+        arguments_str = StringBuilder()
+        for i, arg in enumerate(self.missing):
+            if i == 0:
+                pass
+            elif i == len(self.missing) - 1:
+                if len(self.missing) == 2:
+                    arguments_str.append(" and ")
+                else:
+                    arguments_str.append(", and ")
+            else:
+                arguments_str.append(", ")
+            arguments_str.append("'%s'" % arg)
+        msg = "missing %s required %s argument%s: %s" % (
+            len(self.missing),
+            "positional" if self.positional else "keyword-only",
+            "s" if len(self.missing) != 1 else "",
+            arguments_str.build())
+        return msg
+
+
+class ArgErrTooMany(ArgErr):
+    def __init__(self, num_args, num_defaults, given, kwonly_given):
+        self.num_args = num_args
+        self.num_defaults = num_defaults
+        self.given = given
+        self.kwonly_given = kwonly_given
+
+    def getmsg(self):
+        num_args = self.num_args
+        num_defaults = self.num_defaults
+        if num_defaults:
+            takes_str = "from %d to %d positional arguments" % (
+                num_args - num_defaults, num_args)
         else:
-            defcount = self.num_defaults
-            has_kwarg = self.signature.has_kwarg()
-            num_args = self.num_args
-            num_kwds = self.num_kwds
-            if defcount == 0 and not self.signature.has_vararg():
-                msg1 = "exactly"
-                if not has_kwarg:
-                    num_args += num_kwds
-                    num_kwds = 0
-            elif not self.missing_args:
-                msg1 = "at most"
-            else:
-                msg1 = "at least"
-                has_kwarg = False
-                n -= defcount
-            if n == 1:
-                plural = ""
-            else:
-                plural = "s"
-            if has_kwarg or num_kwds > 0:
-                msg2 = " non-keyword"
-            else:
-                msg2 = ""
-            msg = "takes %s %d%s argument%s (%d given)" % (
-                msg1,
-                n,
-                msg2,
-                plural,
-                num_args)
+            takes_str = "%d positional argument%s" % (
+                num_args, "s" if num_args != 1 else "")
+        if self.kwonly_given:
+            given_str = ("%s positional argument%s "
+                         "(and %s keyword-only argument%s) were") % (
+                self.given, "s" if self.given != 1 else "",
+                self.kwonly_given, "s" if self.kwonly_given != 1 else "")
+        else:
+            given_str = "%s %s" % (
+                self.given, "were" if self.given != 1 else "was")
+        msg = "takes %s but %s given" % (takes_str, given_str)
         return msg
 
+
 class ArgErrMultipleValues(ArgErr):
 
     def __init__(self, argname):
         self.argname = argname
 
     def getmsg(self):
-        msg = "got multiple values for keyword argument '%s'" % (
-            self.argname)
+        msg = "got multiple values for argument '%s'" % self.argname
         return msg
 
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import py
 from pypy.interpreter.argument import (Arguments, ArgErr, ArgErrUnknownKwds,
-        ArgErrMultipleValues, ArgErrCount)
+        ArgErrMultipleValues, ArgErrMissing, ArgErrTooMany)
 from pypy.interpreter.signature import Signature
 from pypy.interpreter.error import OperationError
 
@@ -575,51 +575,54 @@
 
 class TestErrorHandling(object):
     def test_missing_args(self):
-        # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
-        # defaults_w, missing_args
-        sig = Signature([], None, None)
-        err = ArgErrCount(1, 0, sig, None, None, 0)
+        err = ArgErrMissing(['a'], True)
         s = err.getmsg()
-        assert s == "takes no arguments (1 given)"
+        assert s == "missing 1 required positional argument: 'a'"
 
-        sig = Signature(['a'], None, None)
-        err = ArgErrCount(0, 0, sig, [], None, 1)
+        err = ArgErrMissing(['a', 'b'], True)
         s = err.getmsg()
-        assert s == "takes exactly 1 argument (0 given)"
+        assert s == "missing 2 required positional arguments: 'a' and 'b'"
 
-        sig = Signature(['a', 'b'], None, None)
-        err = ArgErrCount(3, 0, sig, [], None, 0)
+        err = ArgErrMissing(['a', 'b', 'c'], True)
         s = err.getmsg()
-        assert s == "takes exactly 2 arguments (3 given)"
-        err = ArgErrCount(3, 0, sig, ['a'], None, 0)
+        assert s == "missing 3 required positional arguments: 'a', 'b', and 'c'"
+
+        err = ArgErrMissing(['a'], False)
         s = err.getmsg()
-        assert s == "takes at most 2 arguments (3 given)"
+        assert s == "missing 1 required keyword-only argument: 'a'"
 
-        sig = Signature(['a', 'b'], '*', None)
-        err = ArgErrCount(1, 0, sig, [], None, 1)
+    def test_too_many(self):
+        err = ArgErrTooMany(0, 0, 1, 0)
         s = err.getmsg()
-        assert s == "takes at least 2 arguments (1 given)"
-        err = ArgErrCount(0, 1, sig, ['a'], None, 1)
+        assert s == "takes 0 positional arguments but 1 was given"
+
+        err = ArgErrTooMany(0, 0, 2, 0)
         s = err.getmsg()
-        assert s == "takes at least 1 non-keyword argument (0 given)"
+        assert s == "takes 0 positional arguments but 2 were given"
 
-        sig = Signature(['a'], None, '**')
-        err = ArgErrCount(2, 1, sig, [], None, 0)
+        err = ArgErrTooMany(1, 0, 2, 0)
         s = err.getmsg()
-        assert s == "takes exactly 1 non-keyword argument (2 given)"
-        err = ArgErrCount(0, 1, sig, [], None, 1)
+        assert s == "takes 1 positional argument but 2 were given"
+
+        err = ArgErrTooMany(2, 0, 3, 0)
         s = err.getmsg()
-        assert s == "takes exactly 1 non-keyword argument (0 given)"
+        assert s == "takes 2 positional arguments but 3 were given"
 
-        sig = Signature(['a'], '*', '**')
-        err = ArgErrCount(0, 1, sig, [], None, 1)
+        err = ArgErrTooMany(2, 1, 3, 0)
         s = err.getmsg()
-        assert s == "takes at least 1 non-keyword argument (0 given)"
+        assert s == "takes from 1 to 2 positional arguments but 3 were given"
 
-        sig = Signature(['a'], None, '**')
-        err = ArgErrCount(2, 1, sig, ['a'], None, 0)
+        err = ArgErrTooMany(0, 0, 1, 1)
         s = err.getmsg()
-        assert s == "takes at most 1 non-keyword argument (2 given)"
+        assert s == "takes 0 positional arguments but 1 positional argument (and 1 keyword-only argument) were given"
+
+        err = ArgErrTooMany(0, 0, 2, 1)
+        s = err.getmsg()
+        assert s == "takes 0 positional arguments but 2 positional arguments (and 1 keyword-only argument) were given"
+
+        err = ArgErrTooMany(0, 0, 1, 2)
+        s = err.getmsg()
+        assert s == "takes 0 positional arguments but 1 positional argument (and 2 keyword-only arguments) were given"
 
     def test_bad_type_for_star(self):
         space = self.space
@@ -665,28 +668,36 @@
     def test_multiple_values(self):
         err = ArgErrMultipleValues('bla')
         s = err.getmsg()
-        assert s == "got multiple values for keyword argument 'bla'"
+        assert s == "got multiple values for argument 'bla'"
 
 class AppTestArgument:
     def test_error_message(self):
         exc = raises(TypeError, (lambda a, b=2: 0), b=3)
-        assert str(exc.value) == "<lambda>() takes at least 1 non-keyword argument (0 given)"
+        assert str(exc.value) == "<lambda>() missing 1 required positional argument: 'a'"
         exc = raises(TypeError, (lambda: 0), b=3)
-        assert str(exc.value) == "<lambda>() takes no arguments (1 given)"
+        assert str(exc.value) == "<lambda>() got an unexpected keyword argument 'b'"
         exc = raises(TypeError, (lambda a, b: 0), 1, 2, 3, a=1)
-        assert str(exc.value) == "<lambda>() takes exactly 2 arguments (4 given)"
+        assert str(exc.value) == "<lambda>() takes 2 positional arguments but 3 were given"
         exc = raises(TypeError, (lambda a, b=1: 0), 1, 2, 3, a=1)
-        assert str(exc.value) == "<lambda>() takes at most 2 non-keyword arguments (3 given)"
+        assert str(exc.value) == "<lambda>() takes from 1 to 2 positional arguments but 3 were given"
+        exc = raises(TypeError, (lambda a, **kw: 0), 1, 2, 3)
+        assert str(exc.value) == "<lambda>() takes 1 positional argument but 3 were given"
         exc = raises(TypeError, (lambda a, b=1, **kw: 0), 1, 2, 3)
-        assert str(exc.value) == "<lambda>() takes at most 2 non-keyword arguments (3 given)"
+        assert str(exc.value) == "<lambda>() takes from 1 to 2 positional arguments but 3 were given"
         exc = raises(TypeError, (lambda a, b, c=3, **kw: 0), 1)
-        assert str(exc.value) == "<lambda>() takes at least 2 arguments (1 given)"
+        assert str(exc.value) == "<lambda>() missing 1 required positional argument: 'b'"
         exc = raises(TypeError, (lambda a, b, **kw: 0), 1)
-        assert str(exc.value) == "<lambda>() takes exactly 2 non-keyword arguments (1 given)"
+        assert str(exc.value) == "<lambda>() missing 1 required positional argument: 'b'"
         exc = raises(TypeError, (lambda a, b, c=3, **kw: 0), a=1)
-        assert str(exc.value) == "<lambda>() takes at least 2 non-keyword arguments (0 given)"
+        assert str(exc.value) == "<lambda>() missing 1 required positional argument: 'b'"
         exc = raises(TypeError, (lambda a, b, **kw: 0), a=1)
-        assert str(exc.value) == "<lambda>() takes exactly 2 non-keyword arguments (0 given)"
+        assert str(exc.value) == "<lambda>() missing 1 required positional argument: 'b'"
+        exc = raises(TypeError, '(lambda *, a: 0)()')
+        assert str(exc.value) == "<lambda>() missing 1 required keyword-only argument: 'a'"
+        exc = raises(TypeError, '(lambda *, a=1, b: 0)(a=1)')
+        assert str(exc.value) == "<lambda>() missing 1 required keyword-only argument: 'b'"
+        exc = raises(TypeError, '(lambda *, kw: 0)(1, kw=3)')
+        assert str(exc.value) == "<lambda>() takes 0 positional arguments but 1 positional argument (and 1 keyword-only argument) were given"
 
     def test_unicode_keywords(self):
         """

From noreply at buildbot.pypy.org  Fri Jul 17 21:02:17 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 17 Jul 2015 21:02:17 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: implement ellipsis indexing
Message-ID: <20150717190217.D221A1C0EB1@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78588:5663f55d1d87
Date: 2015-07-17 20:02 +0100
http://bitbucket.org/pypy/pypy/changeset/5663f55d1d87/

Log:	implement ellipsis indexing

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -222,18 +222,18 @@
         if space.isinstance_w(w_idx, space.w_slice):
             if len(self.get_shape()) == 0:
                 raise oefmt(space.w_ValueError, "cannot slice a 0-d array")
-            return [SliceChunk(w_idx)]
+            return [SliceChunk(w_idx), EllipsisChunk()]
         elif space.isinstance_w(w_idx, space.w_int):
-            return [IntegerChunk(w_idx)]
+            return [IntegerChunk(w_idx), EllipsisChunk()]
         elif isinstance(w_idx, W_NDimArray) and w_idx.is_scalar():
             w_idx = w_idx.get_scalar_value().item(space)
             if not space.isinstance_w(w_idx, space.w_int) and \
                     not space.isinstance_w(w_idx, space.w_bool):
                 raise OperationError(space.w_IndexError, space.wrap(
                     "arrays used as indices must be of integer (or boolean) type"))
-            return [IntegerChunk(w_idx)]
+            return [IntegerChunk(w_idx), EllipsisChunk()]
         elif space.is_w(w_idx, space.w_None):
-            return [NewAxisChunk()]
+            return [NewAxisChunk(), EllipsisChunk()]
         result = []
         i = 0
         has_ellipsis = False
@@ -254,6 +254,8 @@
             else:
                 result.append(IntegerChunk(w_item))
                 i += 1
+        if not has_ellipsis:
+            result.append(EllipsisChunk())
         return result
 
     def descr_getitem(self, space, orig_arr, w_index):
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -59,12 +59,7 @@
         backstride = base_stride * max(0, length - 1) * step
         return start, length, stride, backstride
 
-
 class NewAxisChunk(Chunk):
-    start = 0
-    stop = 1
-    step = 1
-    lgt = 1
     input_dim = 0
     out_dim = 1
 
@@ -75,9 +70,15 @@
         return 0, 1, 0, 0
 
 class EllipsisChunk(BaseChunk):
+    input_dim = 0
+    out_dim = 0
     def __init__(self):
         pass
 
+    def compute(self, space, base_length, base_stride):
+        backstride = base_stride * max(0, base_length - 1)
+        return 0, base_length, base_stride, backstride
+
 
 def new_view(space, w_arr, chunks):
     arr = w_arr.implementation
@@ -127,38 +128,48 @@
 @jit.look_inside_iff(lambda space, shape, start, strides, backstrides, chunks:
                      jit.isconstant(len(chunks)))
 def calculate_slice_strides(space, shape, start, strides, backstrides, chunks):
+    """
+    Note: `chunks` must contain exactly one EllipsisChunk object.
+    """
     size = 0
+    used_dims = 0
     for chunk in chunks:
+        used_dims += chunk.input_dim
         size += chunk.out_dim
-    rstrides = [0] * size
-    rbackstrides = [0] * size
+    if used_dims > len(shape):
+        raise oefmt(space.w_IndexError, "too many indices for array")
+    else:
+        extra_dims = len(shape) - used_dims
+    rstrides = [0] * (size + extra_dims)
+    rbackstrides = [0] * (size + extra_dims)
     rstart = start
-    rshape = [0] * size
-    i = -1  # index of the current dimension in the input array
+    rshape = [0] * (size + extra_dims)
+    rstart = start
+    i = 0  # index of the current dimension in the input array
     j = 0  # index of the current dimension in the result view
     for chunk in chunks:
-        i += chunk.input_dim
         if isinstance(chunk, NewAxisChunk):
             rshape[j] = 1
             j += 1
             continue
-        try:
-            s_i = strides[i]
-        except IndexError:
+        elif isinstance(chunk, EllipsisChunk):
+            for k in range(extra_dims):
+                start, length, stride, backstride = chunk.compute(
+                        space, shape[i], strides[i])
+                rshape[j] = length
+                rstrides[j] = stride
+                rbackstrides[j] = backstride
+                j += 1
+                i += 1
             continue
         start, length, stride, backstride = chunk.compute(space, shape[i], strides[i])
         if chunk.out_dim == 1:
+            rshape[j] = length
             rstrides[j] = stride
             rbackstrides[j] = backstride
-            rshape[j] = length
             j += chunk.out_dim
-        rstart += s_i * start
-    # add a reminder
-    s = i + 1
-    assert s >= 0
-    rstrides += strides[s:]
-    rbackstrides += backstrides[s:]
-    rshape += shape[s:]
+        rstart += strides[i] * start
+        i += chunk.input_dim
     return rshape, rstart, rstrides, rbackstrides
 
 
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -4,7 +4,7 @@
 
 from pypy.conftest import option
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
-from pypy.module.micronumpy.strides import Chunk, new_view
+from pypy.module.micronumpy.strides import Chunk, new_view, EllipsisChunk
 from pypy.module.micronumpy.ndarray import W_NDimArray
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
@@ -22,6 +22,8 @@
 
 
 def create_slice(space, a, chunks):
+    if not any(isinstance(c, EllipsisChunk) for c in chunks):
+        chunks.append(EllipsisChunk())
     return new_view(space, W_NDimArray(a), chunks).implementation
 
 
@@ -2490,6 +2492,9 @@
         a = np.arange(6).reshape(2, 3)
         if '__pypy__' in sys.builtin_module_names:
             raises(ValueError, "a[..., ...]")
+        b = a [..., 0]
+        assert (b == [0, 3]).all()
+        assert b.base is a
 
     def test_empty_indexing(self):
         import numpy as np

From noreply at buildbot.pypy.org  Fri Jul 17 21:09:24 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 21:09:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Fix test.
Message-ID: <20150717190924.D34021C034E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78589:5d8ec598c9cd
Date: 2015-07-17 18:39 +0200
http://bitbucket.org/pypy/pypy/changeset/5d8ec598c9cd/

Log:	Fix test.

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -763,7 +763,7 @@
         class C(metaclass=T):
             pass
         assert d
-        assert sorted(d[0].keys()) == ['__dict__', '__doc__', '__module__', '__weakref__']
+        assert sorted(d[0].keys()) == ['__dict__', '__doc__', '__module__', '__qualname__', '__weakref__']
         d = []
         class T(type):
             def mro(cls):
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -179,7 +179,7 @@
             try:
                 source = runner.statement
                 source = str(source).strip()
-            except py.error.ENOENT:
+            except (py.error.ENOENT, SyntaxError):
                 source = None
             from pypy import conftest
             if source and py.test.config._assertstate.mode != "off":

From noreply at buildbot.pypy.org  Fri Jul 17 21:09:26 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 21:09:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Fix test to check for DeprecationWarning
 instead of PendingDeprecationWarning.
Message-ID: <20150717190926.00A9F1C034E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78590:2293d8db2159
Date: 2015-07-17 20:48 +0200
http://bitbucket.org/pypy/pypy/changeset/2293d8db2159/

Log:	Fix test to check for DeprecationWarning instead of
	PendingDeprecationWarning.

diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -152,18 +152,18 @@
         msg = "object.__format__ with a non-empty format string is deprecated",
         with warnings.catch_warnings(record=True) as log:
             # This is ok because warnings.catch_warnings resets the filters
-            warnings.simplefilter("always", PendingDeprecationWarning)
+            warnings.simplefilter("always", DeprecationWarning)
             assert self.s("{0:^10}").format(E("data")) == " E(data)  "
             assert log[0].message.args == msg
-            assert type(log[0].message) is PendingDeprecationWarning
+            assert type(log[0].message) is DeprecationWarning
 
             assert self.s("{0:^10s}").format(E("data")) == " E(data)  "
             assert log[1].message.args == msg
-            assert type(log[1].message) is PendingDeprecationWarning
+            assert type(log[1].message) is DeprecationWarning
 
             assert self.s("{0:>15s}").format(G("data")) == " string is data"
             assert log[2].message.args == msg
-            assert type(log[2].message) is PendingDeprecationWarning
+            assert type(log[2].message) is DeprecationWarning
         assert len(log) == 3
 
     def test_bogus_cases(self):

From noreply at buildbot.pypy.org  Fri Jul 17 21:09:27 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 21:09:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Back out changeset 78b221682191.
Message-ID: <20150717190927.24BD51C034E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78591:b8d4cf2a1100
Date: 2015-07-17 20:52 +0200
http://bitbucket.org/pypy/pypy/changeset/b8d4cf2a1100/

Log:	Back out changeset 78b221682191.

diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -193,8 +193,6 @@
                         # clarity
                         raise etype, evalue, etb
             else:
-                if value is None:
-                    return value
                 #print spec, "->", value
                 if hasattr(value, 'func_code'):  # semi-evil
                     return space.wrap(gateway.interp2app(value))

From noreply at buildbot.pypy.org  Fri Jul 17 21:09:28 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 21:09:28 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Remove outdated comment.
Message-ID: <20150717190928.433D91C034E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78592:9f276b11158d
Date: 2015-07-17 20:54 +0200
http://bitbucket.org/pypy/pypy/changeset/9f276b11158d/

Log:	Remove outdated comment.

diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -88,8 +88,6 @@
     return space.wrap("short")
 
 def get_thread_info(space):
-    # TODO: implement this instead of returning None (which means unknown) for
-    # every field
     if not space.config.objspace.usemodules.thread:
         return None
     from rpython.rlib import rthread

From noreply at buildbot.pypy.org  Fri Jul 17 21:09:29 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 21:09:29 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Properly implement sys.thread_info.
Message-ID: <20150717190929.625BE1C034E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78593:75dcac9caa42
Date: 2015-07-17 21:05 +0200
http://bitbucket.org/pypy/pypy/changeset/75dcac9caa42/

Log:	Properly implement sys.thread_info.

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -78,7 +78,6 @@
         'int_info'              : 'system.get_int_info(space)',
         'hash_info'             : 'system.get_hash_info(space)',
         'float_repr_style'      : 'system.get_float_repr_style(space)',
-        'thread_info'           : 'system.get_thread_info(space)',
         }
 
     if sys.platform == 'win32':
@@ -114,6 +113,11 @@
                 w_handle = vm.get_dllhandle(space)
                 space.setitem(self.w_dict, space.wrap("dllhandle"), w_handle)
 
+        from pypy.module.sys import system
+        thread_info = system.get_thread_info(space)
+        if thread_info is not None:
+            space.setitem(self.w_dict, space.wrap('thread_info'), thread_info)
+
     def setup_after_space_initialization(self):
         space = self.space
 

From noreply at buildbot.pypy.org  Fri Jul 17 21:09:30 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 21:09:30 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Add _struct module to spaceconfig.
Message-ID: <20150717190930.7E8F31C034E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78594:0eb1db3dec84
Date: 2015-07-17 21:09 +0200
http://bitbucket.org/pypy/pypy/changeset/0eb1db3dec84/

Log:	Add _struct module to spaceconfig.

diff --git a/pypy/module/__builtin__/test/test_filter_pickle.py b/pypy/module/__builtin__/test/test_filter_pickle.py
--- a/pypy/module/__builtin__/test/test_filter_pickle.py
+++ b/pypy/module/__builtin__/test/test_filter_pickle.py
@@ -1,4 +1,5 @@
 class AppTestFilterPickle:
+    spaceconfig = dict(usemodules=['struct'])
 
     def test_filter_unpickle(self):
         """Test just the unpickling."""
diff --git a/pypy/module/__builtin__/test/test_map_pickle.py b/pypy/module/__builtin__/test/test_map_pickle.py
--- a/pypy/module/__builtin__/test/test_map_pickle.py
+++ b/pypy/module/__builtin__/test/test_map_pickle.py
@@ -1,4 +1,5 @@
 class AppTestMapPickle:
+    spaceconfig = dict(usemodules=['struct'])
 
     def test_map_pickle(self):
         """Pickle a map with one sequence."""
diff --git a/pypy/module/__builtin__/test/test_zip_pickle.py b/pypy/module/__builtin__/test/test_zip_pickle.py
--- a/pypy/module/__builtin__/test/test_zip_pickle.py
+++ b/pypy/module/__builtin__/test/test_zip_pickle.py
@@ -1,4 +1,5 @@
 class AppTestZipPickle:
+    spaceconfig = dict(usemodules=['struct'])
 
     def test_zip_pickle(self):
         import pickle

From noreply at buildbot.pypy.org  Fri Jul 17 21:35:44 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 21:35:44 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Add frozenset to fake objspace's
	BUILTIN_TYPES.
Message-ID: <20150717193544.170661C1035@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78595:06468b42f6a5
Date: 2015-07-17 21:20 +0200
http://bitbucket.org/pypy/pypy/changeset/06468b42f6a5/

Log:	Add frozenset to fake objspace's BUILTIN_TYPES.

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -114,7 +114,7 @@
 
 BUILTIN_TYPES = ['int', 'str', 'float', 'tuple', 'list', 'dict', 'bytes',
                  'unicode', 'complex', 'slice', 'bool', 'text', 'object',
-                 'set', 'bytearray', 'memoryview']
+                 'set', 'frozenset', 'bytearray', 'memoryview']
 
 class FakeObjSpace(ObjSpace):
     def __init__(self, config=None):

From noreply at buildbot.pypy.org  Fri Jul 17 21:35:45 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 17 Jul 2015 21:35:45 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Add _posixsubprocess to various test
	classes' spaceconfig.
Message-ID: <20150717193545.4B9531C1035@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78596:23225d87521e
Date: 2015-07-17 21:35 +0200
http://bitbucket.org/pypy/pypy/changeset/23225d87521e/

Log:	Add _posixsubprocess to various test classes' spaceconfig.

diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -9,7 +9,8 @@
 
 class AppTestBufferTooShort:
     spaceconfig = {'usemodules': ['_multiprocessing', 'thread', 'signal',
-                                  'select', 'struct', 'binascii']}
+                                  'select', 'struct', 'binascii',
+                                  '_posixsubprocess']}
     if sys.platform == 'win32':
         spaceconfig['usemodules'].append('_rawffi')
     else:
@@ -105,7 +106,7 @@
     spaceconfig = {
         "usemodules": [
             '_multiprocessing', 'thread', 'signal', 'struct', 'array',
-            '_socket', 'binascii', 'select' ]
+            '_socket', 'binascii', 'select', '_posixsubprocess']
     }
     if sys.platform == 'win32':
         spaceconfig['usemodules'].append('_rawffi')
diff --git a/pypy/module/_multiprocessing/test/test_memory.py b/pypy/module/_multiprocessing/test/test_memory.py
--- a/pypy/module/_multiprocessing/test/test_memory.py
+++ b/pypy/module/_multiprocessing/test/test_memory.py
@@ -2,7 +2,7 @@
     spaceconfig = dict(usemodules=('_multiprocessing', 'mmap',
                                    '_rawffi',
                                    'signal', 'select', 'fcntl',
-                                   'binascii'))
+                                   'binascii', '_posixsubprocess'))
 
     def test_address_of(self):
         import _multiprocessing
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -5,7 +5,7 @@
 class AppTestSemaphore:
     spaceconfig = dict(usemodules=('_multiprocessing', 'thread',
                                    'signal', 'select', 'fcntl',
-                                   'binascii', 'struct'))
+                                   'binascii', 'struct', '_posixsubprocess'))
 
     def setup_class(cls):
         cls.w_SEMAPHORE = cls.space.wrap(SEMAPHORE)

From noreply at buildbot.pypy.org  Sat Jul 18 18:11:33 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Jul 2015 18:11:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: We get a guard_nonnull here. It may be
 a bit hard to remove, but
Message-ID: <20150718161133.D2B231C034E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78597:2bc3e9cb805a
Date: 2015-07-18 18:11 +0200
http://bitbucket.org/pypy/pypy/changeset/2bc3e9cb805a/

Log:	We get a guard_nonnull here. It may be a bit hard to remove, but
	ideally it should be...

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -252,6 +252,8 @@
         guard_no_exception(descr=...)
         p53 = call(ConstClass(fast_str_decode_ascii), p80, descr=<Callr . r EF=4>)
         guard_no_exception(descr=...)
+        guard_nonnull(p53, descr=...)
         --TICK--
         jump(..., descr=...)
         """)
+        # XXX remove the guard_nonnull above?

From noreply at buildbot.pypy.org  Sat Jul 18 22:38:42 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 18 Jul 2015 22:38:42 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-buffered: a branch to implement buffered
 keyword for nditer
Message-ID: <20150718203842.7CEDE1C06D7@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-buffered
Changeset: r78598:9b6522fc060c
Date: 2015-07-18 23:03 +0300
http://bitbucket.org/pypy/pypy/changeset/9b6522fc060c/

Log:	a branch to implement buffered keyword for nditer


From noreply at buildbot.pypy.org  Sat Jul 18 22:38:43 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 18 Jul 2015 22:38:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: cleanup and fix failing tests
Message-ID: <20150718203843.AEC5B1C071E@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78599:5d329a647a3f
Date: 2015-07-18 23:06 +0300
http://bitbucket.org/pypy/pypy/changeset/5d329a647a3f/

Log:	cleanup and fix failing tests

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -950,7 +950,7 @@
                 raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple.")
             else:
                 raise
-        if dim > int(0x7fffffff):
+        if dim > 0x7fffffff:
             raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
                       "dimension does not fit into a C int.")
         elif dim < 0:
@@ -977,7 +977,7 @@
         assert isinstance(subdtype, W_Dtype)
         size = support.product(shape)
         size *= subdtype.elsize
-        if size > int(0x7fffffff):
+        if size > 0x7fffffff:
             raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
                   "dtype size in bytes must fit into a C int.")
         
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1067,6 +1067,12 @@
 
 class AppTestRecordDtypes(BaseNumpyAppTest):
     spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
+    def setup_class(cls):
+        BaseNumpyAppTest.setup_class.im_func(cls)
+        if option.runappdirect:
+            cls.w_test_for_core_internal = cls.space.wrap(True)
+        else:
+            cls.w_test_for_core_internal = cls.space.wrap(False)
 
     def test_create(self):
         from numpy import dtype, void
@@ -1307,6 +1313,11 @@
 
     def test_aligned_size(self):
         import numpy as np
+        if self.test_for_core_internal:
+            try:
+                from numpy.core import _internal
+            except ImportError:
+                skip ('no numpy.core._internal available')
         # Check that structured dtypes get padded to an aligned size
         dt = np.dtype('i4, i1', align=True)
         assert dt.itemsize == 8
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -764,8 +764,9 @@
         assert (a[1:] == b).all()
         assert (a[1:,newaxis] == d).all()
         assert (a[newaxis,1:] == c).all()
-        assert a.strides == (8,)
-        assert a[:, newaxis].strides == (8, 0)
+        n = a.dtype.itemsize
+        assert a.strides == (n,)
+        assert a[:, newaxis].strides == (n, 0)
 
     def test_newaxis_assign(self):
         from numpy import array, newaxis

From noreply at buildbot.pypy.org  Sat Jul 18 22:38:44 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 18 Jul 2015 22:38:44 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-buffered: revive test,
 implementation of "buffered" keyword
Message-ID: <20150718203844.D38A91C06D7@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-buffered
Changeset: r78600:abf5ad87eb45
Date: 2015-07-18 23:18 +0300
http://bitbucket.org/pypy/pypy/changeset/abf5ad87eb45/

Log:	revive test, implementation of "buffered" keyword

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -108,9 +108,6 @@
         if item == 'external_loop':
             nditer.external_loop = True
         elif item == 'buffered':
-            raise oefmt(space.w_NotImplementedError,
-                'nditer buffered not implemented yet')
-            # For numpy compatability
             nditer.buffered = True
         elif item == 'c_index':
             nditer.tracked_index = 'C'
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -152,9 +152,6 @@
         from numpy import arange, nditer, array
         a = arange(6).reshape(2,3)
         import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['buffered'])
-            skip('nditer buffered not implmented')
         r = []
         for x in nditer(a, flags=['external_loop', 'buffered'], order='F'):
             r.append(x)
@@ -188,9 +185,6 @@
         from numpy import arange, nditer
         import sys
         a = arange(6.)
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['buffered'], op_dtypes=['float32'])
-            skip('nditer casting not implemented yet')
         exc = raises(TypeError, nditer, a, flags=['buffered'], op_dtypes=['float32'])
         assert str(exc.value).startswith("Iterator operand 0 dtype could not be cast")
         r = []
@@ -232,9 +226,6 @@
             return it.operands[1]
         assert (square1([1, 2, 3]) == [1, 4, 9]).all()
 
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, [1, 2], flags=['buffered'])
-            skip('nditer buffered not implmented')
         def square2(a, out=None):
             it = nditer([a, out], flags=['external_loop', 'buffered'],
                         op_flags=[['readonly'],

From noreply at buildbot.pypy.org  Sat Jul 18 22:48:31 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 18 Jul 2015 22:48:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add another test for properties (fails
	on PyPy)
Message-ID: <20150718204831.943841C06D7@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78601:f7292d95a6a9
Date: 2015-07-18 21:48 +0100
http://bitbucket.org/pypy/pypy/changeset/f7292d95a6a9/

Log:	Add another test for properties (fails on PyPy)

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4433,6 +4433,35 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f2, [int])
 
+    @py.test.mark.xfail("'__pypy__' in sys.modules")
+    def test_property_union_2(self):
+        class Base(object):
+            pass
+
+        class A(Base):
+            def __init__(self):
+                pass
+
+            @property
+            def x(self):
+                return 42
+
+        class B(Base):
+            def __init__(self, x):
+                self.x = x
+
+        def f(n):
+            if n < 0:
+                obj = A()
+            else:
+                obj = B(n)
+            return obj.x
+        a = self.RPythonAnnotator()
+        # Ideally, this should translate to something sensible,
+        # but for now, UnionError is better than silently mistranslating.
+        with py.test.raises(annmodel.UnionError):
+            a.build_types(f, [int])
+
 
 def g(n):
     return [0, 1, 2, n]

From noreply at buildbot.pypy.org  Sun Jul 19 16:56:03 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sun, 19 Jul 2015 16:56:03 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Add _struct module to spaceconfig.
Message-ID: <20150719145603.5BD031C056E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78602:afbe7361ae01
Date: 2015-07-18 01:50 +0200
http://bitbucket.org/pypy/pypy/changeset/afbe7361ae01/

Log:	Add _struct module to spaceconfig.

diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -409,7 +409,8 @@
 
 
 class AppTestSSLError:
-    spaceconfig = dict(usemodules=('_ssl', '_socket', 'binascii', 'thread'))
+    spaceconfig = dict(usemodules=('_ssl', '_socket', 'binascii', 'thread',
+                                   'struct'))
 
     def setup_class(cls):
         tmpfile = udir / "tmpfile.pem"

From noreply at buildbot.pypy.org  Sun Jul 19 16:56:04 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sun, 19 Jul 2015 16:56:04 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix this test to work on Python 2.7.6.
Message-ID: <20150719145604.A419A1C056E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r78603:3861ef3a5b57
Date: 2015-07-19 16:55 +0200
http://bitbucket.org/pypy/pypy/changeset/3861ef3a5b57/

Log:	Fix this test to work on Python 2.7.6.

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1253,6 +1253,9 @@
     def teardown_class(cls):
         _teardown(cls.space, cls.w_saved_modules)
 
+    def w_exec_(self, cmd, ns):
+        exec(cmd, ns)
+
     def test_meta_path(self):
         tried_imports = []
         class Importer(object):
@@ -1449,7 +1452,7 @@
         import sys, types
         sys.meta_path.append(ImportHook())
         try:
-            exec("from meta_path_2_pseudo_module import *", {})
+            self.exec_("from meta_path_2_pseudo_module import *", {})
         finally:
             sys.meta_path.pop()
 

From noreply at buildbot.pypy.org  Mon Jul 20 07:43:46 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 20 Jul 2015 07:43:46 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-buffered: improve test by comparing to
 upstream, move things around until part of test passes
Message-ID: <20150720054346.349861C1048@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-buffered
Changeset: r78604:f5d927f06929
Date: 2015-07-20 08:44 +0300
http://bitbucket.org/pypy/pypy/changeset/f5d927f06929/

Log:	improve test by comparing to upstream, move things around until part
	of test passes

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -18,7 +18,7 @@
 from pypy.module.micronumpy.converters import multi_axis_converter, \
     order_converter, shape_converter, searchside_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
-from pypy.module.micronumpy.strides import get_shape_from_iterable, \
+from pypy.module.micronumpy.strideops import get_shape_from_iterable, \
     shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
 from pypy.module.micronumpy.casting import can_cast_array
 
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -7,8 +7,7 @@
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array, W_NumpyObject
 from pypy.module.micronumpy.descriptor import decode_w_dtype
 from pypy.module.micronumpy.iterators import ArrayIter
-from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
-                                            shape_agreement, shape_agreement_multiple)
+from pypy.module.micronumpy import strideops
 from pypy.module.micronumpy.casting import find_binop_result_dtype
 
 
@@ -213,6 +212,15 @@
 def get_iter(space, order, arr, shape, dtype, op_flags, base):
     imp = arr.implementation
     backward = is_backward(imp, order)
+    if len(shape) == 1:
+        min_dim = 0
+        min_stride = 0xefffffff
+        for i in range(len(imp.shape)):
+            if imp.strides[i] < min_stride:
+                min_dim = i
+                min_stride = imp.strides[i]
+        return ConcreteIter(imp, imp.get_size(), shape, [imp.strides[i]],
+                           [imp.backstrides[i]], op_flags, base)
     if arr.is_scalar():
         return ConcreteIter(imp, 1, [], [], [], op_flags, base)
     if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \
@@ -220,19 +228,20 @@
         # flip the strides. Is this always true for multidimension?
         strides = imp.strides[:]
         backstrides = imp.backstrides[:]
-        shape = imp.shape[:]
         strides.reverse()
         backstrides.reverse()
-        shape.reverse()
+        _shape = imp.shape[:]
+        _shape.reverse()
     else:
         strides = imp.strides
         backstrides = imp.backstrides
-    r = calculate_broadcast_strides(strides, backstrides, imp.shape,
+        _shape = imp.shape
+    r = strideops.calculate_broadcast_strides(strides, backstrides, _shape,
                                     shape, backward)
     if len(shape) != len(r[0]):
         # shape can be shorter when using an external loop, just return a view
-        return ConcreteIter(imp, imp.get_size(), imp.shape, r[0], r[1], op_flags, base)
-    return ConcreteIter(imp, imp.get_size(), shape, r[0], r[1], op_flags, base)
+        return ConcreteIter(imp, imp.get_size(), _shape, r[0], r[1], op_flags, base)
+    return ConcreteIter(imp, imp.get_size(), _shape, r[0], r[1], op_flags, base)
 
 def calculate_ndim(op_in, oa_ndim):
     if oa_ndim >=0:
@@ -413,15 +422,15 @@
         outargs = [i for i in range(len(self.seq))
                    if self.seq[i] is None or self.op_flags[i].rw == 'w']
         if len(outargs) > 0:
-            out_shape = shape_agreement_multiple(space, [self.seq[i] for i in outargs])
+            out_shape = strideops.shape_agreement_multiple(space, [self.seq[i] for i in outargs])
         else:
             out_shape = None
         if space.isinstance_w(w_itershape, space.w_tuple) or \
            space.isinstance_w(w_itershape, space.w_list):
             self.shape = [space.int_w(i) for i in space.listview(w_itershape)]
         else:
-            self.shape = shape_agreement_multiple(space, self.seq,
-                                                           shape=out_shape)
+            self.shape = strideops.shape_agreement_multiple(space, self.seq,
+                                                     shape=out_shape)
         if len(outargs) > 0:
             # Make None operands writeonly and flagged for allocation
             if len(self.dtypes) > 0:
@@ -443,7 +452,7 @@
                 else:
                     if not self.op_flags[i].broadcast:
                         # Raises if ooutput cannot be broadcast
-                        shape_agreement(space, self.shape, self.seq[i], False)
+                        strideops.shape_agreement(space, self.shape, self.seq[i], False)
 
         if self.tracked_index != "":
             if self.order == "K":
@@ -453,7 +462,6 @@
             else:
                 backward = self.order != self.tracked_index
             self.index_iter = IndexIterator(self.shape, backward=backward)
-
         # handle w_op_dtypes part 2: copy where needed if possible
         if len(self.dtypes) > 0:
             for i in range(len(self.seq)):
@@ -553,7 +561,7 @@
             res.append(self.getitem(it, st))
             self.iters[i] = (it, it.next(st))
         if len(res) < 2:
-            return res[0]
+            return res[0] 
         return space.newtuple(res)
 
     def iternext(self):
@@ -648,7 +656,9 @@
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
 
     def descr_get_shape(self, space):
-        raise oefmt(space.w_NotImplementedError, "not implemented yet")
+        if self.done:
+            raise oefmt(space.w_ValueError, "Iterator is past the end")
+        return space.newtuple([space.wrap(i) for i in self.shape])
 
     def descr_get_value(self, space):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strideops.py
rename from pypy/module/micronumpy/strides.py
rename to pypy/module/micronumpy/strideops.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strideops.py
@@ -3,7 +3,6 @@
 from pypy.module.micronumpy import support, constants as NPY
 from pypy.module.micronumpy.base import W_NDimArray
 
-
 # structures to describe slicing
 
 class BaseChunk(object):
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -11,7 +11,6 @@
            space.isinstance_w(w_obj, space.w_buffer) or
            isinstance(w_obj, W_NDimArray))
 
-
 def index_w(space, w_obj):
     try:
         return space.int_w(space.index(w_obj))
@@ -23,7 +22,6 @@
                 "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
                 "boolean arrays are valid indices")
 
-
 @jit.unroll_safe
 def product(s):
     i = 1
@@ -31,7 +29,6 @@
         i = ovfcheck(i * x)
     return i
 
-
 def check_and_adjust_index(space, index, size, axis):
     if index < -size or index >= size:
         if axis >= 0:
diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -38,3 +38,12 @@
     a = zeros(*args, **kwargs)
     a.fill(1)
     return a
+
+def shape(a):
+    try:
+        result = a.shape
+    except AttributeError:
+        result = asarray(a).shape
+    return result
+
+
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -149,16 +149,26 @@
         # assert str(exc.value).startswith("Iterator flag EXTERNAL_LOOP cannot")
 
     def test_buffered(self):
-        from numpy import arange, nditer, array
-        a = arange(6).reshape(2,3)
-        import sys
+        from numpy import arange, nditer, array, shape
+        a = arange(6).reshape(2, 3)
         r = []
-        for x in nditer(a, flags=['external_loop', 'buffered'], order='F'):
-            r.append(x)
-        array_r = array(r)
-        assert len(array_r.shape) == 2
-        assert array_r.shape == (1, 6)
-        assert (array_r == [0, 3, 1, 4, 2, 5]).all()
+        x = nditer(a, flags=['buffered'], order='F')
+        assert x.shape == a.shape # maybe a numpy bug, but be compatible
+        for i in x:
+            r.append(i)
+            assert x.shape == a.shape
+        assert r[0].shape == ()
+        raises(ValueError, shape, x)
+        assert len(r) == 6 # buffered flattens the iterator
+        r = []
+        assert a.shape == (2, 3)
+        x = nditer(a, flags=['external_loop', 'buffered'], order='F')
+        assert x.shape == a.shape # maybe a numpy bug, but be compatible
+        for i in x:
+            r.append(i)
+        assert len(r) == 1 # external_loop coalesces the flattened iterator
+        assert r[0].shape == (6,) 
+        assert (r[0] == [0, 3, 1, 4, 2, 5]).all() #respect the order
 
     def test_op_dtype(self):
         from numpy import arange, nditer, sqrt, array
@@ -186,7 +196,8 @@
         import sys
         a = arange(6.)
         exc = raises(TypeError, nditer, a, flags=['buffered'], op_dtypes=['float32'])
-        assert str(exc.value).startswith("Iterator operand 0 dtype could not be cast")
+        errstr = str(exc.value)
+        assert errstr.startswith("Iterator operand 0 dtype could not be cast")
         r = []
         for x in nditer(a, flags=['buffered'], op_dtypes=['float32'],
                                 casting='same_kind'):

From noreply at buildbot.pypy.org  Mon Jul 20 07:56:16 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 20 Jul 2015 07:56:16 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-buffered: fix non-buffered related test
	failures
Message-ID: <20150720055616.D87DC1C1169@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-buffered
Changeset: r78605:899d1db5c5a4
Date: 2015-07-20 08:56 +0300
http://bitbucket.org/pypy/pypy/changeset/899d1db5c5a4/

Log:	fix non-buffered related test failures

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -219,8 +219,8 @@
             if imp.strides[i] < min_stride:
                 min_dim = i
                 min_stride = imp.strides[i]
-        return ConcreteIter(imp, imp.get_size(), shape, [imp.strides[i]],
-                           [imp.backstrides[i]], op_flags, base)
+        return ConcreteIter(imp, imp.get_size(), shape, [imp.strides[min_dim]],
+                           [imp.backstrides[min_dim]], op_flags, base)
     if arr.is_scalar():
         return ConcreteIter(imp, 1, [], [], [], op_flags, base)
     if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \
@@ -238,9 +238,9 @@
         _shape = imp.shape
     r = strideops.calculate_broadcast_strides(strides, backstrides, _shape,
                                     shape, backward)
-    if len(shape) != len(r[0]):
+    if len(_shape) != len(r[0]):
         # shape can be shorter when using an external loop, just return a view
-        return ConcreteIter(imp, imp.get_size(), _shape, r[0], r[1], op_flags, base)
+        return ConcreteIter(imp, imp.get_size(), shape, r[0], r[1], op_flags, base)
     return ConcreteIter(imp, imp.get_size(), _shape, r[0], r[1], op_flags, base)
 
 def calculate_ndim(op_in, oa_ndim):

From noreply at buildbot.pypy.org  Mon Jul 20 11:30:05 2015
From: noreply at buildbot.pypy.org (krono)
Date: Mon, 20 Jul 2015 11:30:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: Darwin: make 32bit/64bit compiler
 distinction more general
Message-ID: <20150720093005.4C2131C037F@cobra.cs.uni-duesseldorf.de>

Author: Tobias Pape <tobias at netshed.de>
Branch: 
Changeset: r78606:b5e4276e0872
Date: 2015-07-20 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/b5e4276e0872/

Log:	Darwin: make 32bit/64bit compiler distinction more general

	pdb/lldebug: do not depend on prctl on non-Linux

diff --git a/rpython/rtyper/module/ll_pdb.py b/rpython/rtyper/module/ll_pdb.py
--- a/rpython/rtyper/module/ll_pdb.py
+++ b/rpython/rtyper/module/ll_pdb.py
@@ -16,13 +16,19 @@
 
 
 if not _WIN32:
-    eci = ExternalCompilationInfo(includes=['string.h', 'assert.h', 'sys/prctl.h'],
-                                  post_include_bits=["""
+    import sys
+    if sys.platform.startswith('linux'):
+        # Only necessary on Linux
+        eci = ExternalCompilationInfo(includes=['string.h', 'assert.h', 'sys/prctl.h'],
+                                      post_include_bits=["""
 static void pypy__allow_attach(void) {
     prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY);
     return;
 }
-    """])
+        """])
+    else:
+        # Do nothing, there's no prctl
+        eci = ExternalCompilationInfo(post_include_bits=["static void pypy__allow_attach(void) { return; }"])
 
     allow_attach= rffi.llexternal(
         "pypy__allow_attach", [], lltype.Void,
diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -67,9 +67,9 @@
 
 class Darwin_i386(Darwin):
     name = "darwin_i386"
-    link_flags = ('-arch', 'i386', '-mmacosx-version-min=10.4')
-    cflags = ('-arch', 'i386', '-O3', '-fomit-frame-pointer',
-              '-mmacosx-version-min=10.4')
+    DEFAULT_CC = 'clang -arch i386'
+    link_flags = ('-mmacosx-version-min=10.4',)
+    cflags = ('-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4')
 
 class Darwin_PowerPC(Darwin):#xxx fixme, mwp
     name = "darwin_powerpc"
@@ -78,6 +78,6 @@
 
 class Darwin_x86_64(Darwin):
     name = "darwin_x86_64"
-    link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.5')
-    cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer',
-              '-mmacosx-version-min=10.5')
+    DEFAULT_CC = 'clang -arch x86_64'
+    link_flags = ('-mmacosx-version-min=10.5',)
+    cflags = ('-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.5')

From noreply at buildbot.pypy.org  Mon Jul 20 11:43:36 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 20 Jul 2015 11:43:36 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: work on the talk
Message-ID: <20150720094336.9CF721C037F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5543:51066aac7fba
Date: 2015-07-20 11:44 +0200
http://bitbucket.org/pypy/extradoc/changeset/51066aac7fba/

Log:	work on the talk

diff --git a/talk/ep2015/performance/talk.rst b/talk/ep2015/performance/talk.rst
--- a/talk/ep2015/performance/talk.rst
+++ b/talk/ep2015/performance/talk.rst
@@ -9,9 +9,9 @@
 
 - PyPy core devs
 
-- ``pdb++``, ``fancycompleter``, ...
+- ``vmprof``, ``cffi``, ``pdb++``, ``fancycompleter``, ...
 
-- Consultant
+- Consultants
 
 - http://baroquesoftware.com/
 
@@ -19,7 +19,7 @@
 About you
 -------------
 
-- Target audience
+- You are proficient in Python
 
 - Your Python program is slow
 
@@ -37,6 +37,8 @@
 
   - 80% of the time will be spent in 20% of the program
 
+  - 20% of 1 mln is 200 000
+
 * Two golden rules:
 
   1. Identify the slow spots
@@ -49,11 +51,112 @@
 
 * Two parts
 
-  1. PyPy as a tool to make Python faster
+  1. How to identify the slow spots
 
-  2. How to identify the slow spots
+  2. How to address the problems
 
 
+Part 1
+-------
+
+* profiling
+
+* tools
+
+
+What is performance?
+--------------------
+
+* you need something quantifiable by numbers
+
+* usually, time spent doing task X
+
+* sometimes number of requests, latency, etc.
+
+* some statistical properties about that metric (average, minimum, maximum)
+
+Do you have a performance problem?
+----------------------------------
+
+* define what you're trying to measure
+
+* measure it (production, benchmarks, etc.)
+
+* see if Python is the cause here (if it's not, we can't help you,
+  but I'm sure someone can)
+
+* make sure you can change and test stuff quickly (e.g. benchmarks are better
+  than changing stuff in production)
+
+* same as for debugging
+
+We have a python problem
+------------------------
+
+* tools, timers etc.
+
+* systems are too complicated to **guess** which will be faster
+
+* find your bottlenecks
+
+* 20/80 (but 20% of million lines is 200 000 lines, remember that)
+
+Profilers landscape
+-------------------
+
+* cProfile, runSnakeRun (high overhead) - event based profiler
+
+* plop, vmprof - statistical profiler
+
+* cProfile & vmprof work on pypy
+
+vmprof
+------
+
+* inspired by ``gperftools``
+
+* statistical profiler run by an interrupt (~300Hz on modern linux)
+
+* sampling the C stack
+
+* CPython, PyPy, possibly more virtual machines
+
+why not just use gperftools?
+----------------------------
+
+* C stack does not contain python-level frames
+
+* 90% ``PyEval_EvalFrame`` and other internals
+
+* we want python-level functions
+
+* picture is even more confusing in the presence of the JIT
+
+using vmprof
+------------
+
+* demo
+
+* http://vmprof.readthedocs.org
+
+using vmprof in production
+--------------------------
+
+* low overhead (5-10%), possibly lower in the future
+
+* possibility of realtime monitoring (coming)
+
+vmprof future
+-------------
+
+* profiler as a service
+
+* realtime advanced visualization
+
+Part 2
+------
+
+Make it fast
 
 Tools
 ------
@@ -74,6 +177,8 @@
 
   * WARNING: we wrote it, we are biased :)
 
+  * gives you most wins for free (*)
+
 
 
 What is PyPy
@@ -107,6 +212,7 @@
    :scale: 47%
 
 
+
 The JIT
 --------
 
@@ -127,15 +233,12 @@
 .. image:: jit-overview3.pdf
    :scale: 50%
 
-
 JIT overview
 -------------
 
 - Tracing JIT
 
-  * detect and compile "hot" loops
-
-  * (although not only loops)
+  * detect and compile "hot" code
 
 - **Specialization**
 
@@ -224,91 +327,3 @@
   * inefficient code
 
 
-Part 2
--------
-
-* Measure performance
-
-* Identify problems
-
-
-What is performance?
---------------------
-
-* it's a metric
-
-* usually, time spent doing task X
-
-* sometimes number of requests, latency, etc.
-
-* some statistical properties about that metric (average, minimum, maximum)
-
-Do you have a performance problem?
-----------------------------------
-
-* define the metric
-
-* measure it (production, benchmarks, etc.)
-
-* see if Python is the cause here (if it's not, we can't help you,
-  but I'm sure someone help)
-
-* make sure you can change and test stuff quickly (e.g. benchmarks are better
-  than changing stuff in production)
-
-We have a python problem
-------------------------
-
-* tools, timers etc.
-
-* systems are too complicated to **guess** which will be faster
-
-* find your bottlenecks
-
-* 20/80 (but 20% of million lines is 200 000 lines, remember that)
-
-Profilers landscape
--------------------
-
-* cProfile, runSnakeRun (high overhead) - exact profiler
-
-* plop, vmprof - statistical profiler
-
-* cProfile & vmprof work on pypy
-
-vmprof
-------
-
-XXXxxx
-
-using vmprof
-------------
-
-yyyyyyy
-
-interpreting the results
-------------------------
-
-xxxx
-
-using vmprof in production
---------------------------
-
-demo
-----
-
-let's optimize some code
-------------------------
-
-let's optimize some more complex code
--------------------------------------
-
-Extras: what's cool what's not cool on cpython and pypy
-
-CPython vs PyPy
----------------
-
-* very different performance characteristics
-
-* XXX list them
-

From noreply at buildbot.pypy.org  Mon Jul 20 12:14:25 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 20 Jul 2015 12:14:25 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: more points
Message-ID: <20150720101425.C2AE31C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5544:308e60262d5f
Date: 2015-07-20 12:05 +0200
http://bitbucket.org/pypy/extradoc/changeset/308e60262d5f/

Log:	more points

diff --git a/talk/ep2015/performance/talk.rst b/talk/ep2015/performance/talk.rst
--- a/talk/ep2015/performance/talk.rst
+++ b/talk/ep2015/performance/talk.rst
@@ -326,4 +326,30 @@
 
   * inefficient code
 
+Guidos points
+-------------
 
+xzxzcxzcxz
+
+Don't do it on PyPy (or at all)
+-------------------------------
+
+yyyy
+
+More about PyPy
+---------------
+
+* we are going to run a PyPy open space
+
+* come ask more questions
+
+Q&A?
+----
+
+* Thank you!
+
+* http://baroquesoftware.com
+
+* http://pypy.org
+
+* http://vmprof.readthedocs.org

From noreply at buildbot.pypy.org  Mon Jul 20 13:55:09 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 20 Jul 2015 13:55:09 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: checkin my rst2beamer.py version
 so that fijal is happy :)
Message-ID: <20150720115509.15B511C037F@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: extradoc
Changeset: r5545:07e4e2721c53
Date: 2015-07-20 13:55 +0200
http://bitbucket.org/pypy/extradoc/changeset/07e4e2721c53/

Log:	checkin my rst2beamer.py version so that fijal is happy :)

diff --git a/talk/bin/rst2beamer.py b/talk/bin/rst2beamer.py
new file mode 100755
--- /dev/null
+++ b/talk/bin/rst2beamer.py
@@ -0,0 +1,267 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+A docutils script converting restructured text into Beamer-flavoured LaTeX.
+
+Beamer is a LaTeX document class for presentations. Via this script, ReST can
+be used to prepare slides. It can be called::
+
+        rst2beamer.py infile.txt > outfile.tex
+
+where ``infile.tex`` contains the produced Beamer LaTeX.
+
+See <http:www.agapow.net/programming/python/rst2beamer> for more details.
+
+"""
+# TODO: modifications for handout sections?
+# TOOD: sections and subsections?
+# TODO: enable beamer themes?
+# TODO: convert document metadata to front page fields?
+# TODO: toc-conversion?
+# TODO: fix descriptions
+
+# Unless otherwise stated, created by P-M Agapow on 2007-08-21
+# and open for academic & non-commercial use and modification .
+
+__docformat__ = 'restructuredtext en'
+__author__ = "Paul-Michael Agapow <agapow at bbsrc.ac.uk>"
+__version__ = "0.2"
+
+
+### IMPORTS ###
+
+import locale
+from docutils.core import publish_cmdline, default_description
+from docutils.writers.latex2e import Writer as Latex2eWriter
+from docutils.writers.latex2e import LaTeXTranslator, DocumentClass
+from docutils import nodes
+
+## Syntax highlighting:
+
+"""
+        .. sourcecode:: python
+
+            My code goes here.
+
+
+    :copyright: 2007 by Georg Brandl.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from pygments.formatters import HtmlFormatter, LatexFormatter
+
+# The default formatter
+DEFAULT = LatexFormatter()
+
+
+from docutils.parsers.rst import directives
+
+from pygments import highlight
+from pygments.lexers import get_lexer_by_name, TextLexer
+
+VARIANTS = {
+    'linenos': LatexFormatter(linenos=True),
+}
+
+def pygments_directive(name, arguments, options, content, lineno,
+                       content_offset, block_text, state, state_machine):
+    try:
+        lexer = get_lexer_by_name(arguments[0])
+    except ValueError:
+        # no lexer found - use the text one instead of an exception
+        lexer = TextLexer()
+    formatter = DEFAULT
+    parsed = highlight(u'\n'.join(content), lexer, formatter)
+    return [nodes.raw('', parsed, format='latex')]
+
+pygments_directive.arguments = (1, 0, 1)
+pygments_directive.content = 1
+pygments_directive.options = dict([(key, directives.flag) for key in VARIANTS])
+
+directives.register_directive('sourcecode', pygments_directive)
+
+
+## multiple images as a single animation
+
+"""
+        .. animage:: foo-p*.pdf
+           :align: center
+           :scale: 50%
+"""
+
+from glob import glob
+import copy
+from docutils.parsers.rst import directives
+from docutils.parsers.rst.directives.images import Image
+import docutils
+
+class Animage(Image): # Animated Image :-)
+    
+    def run(self):
+        def raw(text):
+            return docutils.nodes.raw('', text, format='latex')
+        
+        nodes = Image.run(self)
+        img = nodes[0]
+        if not isinstance(img, docutils.nodes.image):
+            return nodes # not an image, WTF?
+        newnodes = []
+        pattern = img.attributes['uri']
+        filenames = sorted(glob(pattern))
+        for i, filename in enumerate(filenames):
+            newimg = copy.deepcopy(img)
+            newimg.attributes['uri'] = filename
+            newnodes += [raw(r'\only<%d>{' % (i+1)),
+                         newimg,
+                         raw('}')]
+        return newnodes
+
+directives.register_directive('animage', Animage)
+
+
+
+
+## CONSTANTS & DEFINES: ###
+
+BEAMER_SPEC =   (
+        'Beamer options',
+        'These are derived almost entirely from the LaTeX2e options',
+        tuple (
+                [
+                        (
+                                'Specify theme.',
+                                ['--theme'],
+                                {'default': '', }
+                        ),
+                        (
+                                'Specify document options.       Multiple options can be given, '
+                                'separated by commas.  Default is "10pt,a4paper".',
+                                ['--documentoptions'],
+                                {'default': '', }
+                        ),
+                ] + list (Latex2eWriter.settings_spec[2][2:])
+        ),
+)
+
+BEAMER_DEFAULTS = {
+        'output_encoding': 'latin-1',
+        'documentclass': 'beamer',
+}
+
+
+### IMPLEMENTATION ###
+
+try:
+         locale.setlocale (locale.LC_ALL, '')
+except:
+         pass
+
+class BeamerTranslator (LaTeXTranslator):
+        """
+        A converter for docutils elements to beamer-flavoured latex.
+        """
+
+        def __init__ (self, document):
+                LaTeXTranslator.__init__ (self, document)
+                self.head_prefix = [x for x in self.head_prefix if ('{typearea}' not in x)]
+                hyperref_posn = [i for i in range (len (self.head_prefix)) if ('{hyperref}' in self.head_prefix[i])]
+                if not hyperref_posn:
+                        self.head_prefix.append(None)
+                        hyperref_posn = [-1] # XXX hack
+                self.head_prefix[hyperref_posn[0]] = ('\\usepackage{hyperref}\n' +
+                                                      '\\usepackage{fancyvrb}\n' +
+                                                      LatexFormatter(style="manni").get_style_defs() +
+                                                      "\n")
+
+                self.head_prefix.extend ([
+                        '\\definecolor{rrblitbackground}{rgb}{0.55, 0.3, 0.1}\n',
+                        '\\newenvironment{rtbliteral}{\n',
+                        '\\begin{ttfamily}\n',
+                        '\\color{rrblitbackground}\n',
+                        '}{\n',
+                        '\\end{ttfamily}\n',
+                        '}\n',
+                ])
+                # this fixes the hardcoded section titles in docutils 0.4
+                self.d_class = DocumentClass ('article')
+
+        def begin_frametag (self, node):
+                if "verbatim" in str(node).lower():
+                    return '\\begin{frame}[containsverbatim,fragile]\n'
+                else:
+                    return '\\begin{frame}\n'
+
+        def end_frametag (self):
+                return '\\end{frame}\n'
+
+        def visit_section (self, node):
+                if (self.section_level == 0):
+                        self.body.append (self.begin_frametag(node))
+                LaTeXTranslator.visit_section (self, node)
+
+        def depart_section (self, node):
+                # Remove counter for potential subsections:
+                LaTeXTranslator.depart_section (self, node)
+                if (self.section_level == 0):
+                        self.body.append (self.end_frametag())
+
+        def visit_title (self, node):
+                if (self.section_level == 1):
+                        self.body.append ('\\frametitle{%s}\n\n' % self.encode(node.astext()))
+                        raise nodes.SkipNode
+                else:
+                        LaTeXTranslator.visit_title (self, node)
+
+        def depart_title (self, node):
+                if (self.section_level != 1):
+                        LaTeXTranslator.depart_title (self, node)
+
+        def visit_literal_block(self, node):
+                 if not self.active_table.is_open():
+                          self.body.append('\n\n\\smallskip\n\\begin{rtbliteral}\n')
+                          self.context.append('\\end{rtbliteral}\n\\smallskip\n\n')
+                 else:
+                          self.body.append('\n')
+                          self.context.append('\n')
+                 if (self.settings.use_verbatim_when_possible and (len(node) == 1)
+                                 # in case of a parsed-literal containing just a "**bold**" word:
+                                 and isinstance(node[0], nodes.Text)):
+                          self.verbatim = 1
+                          self.body.append('\\begin{verbatim}\n')
+                 else:
+                          self.literal_block = 1
+                          self.insert_none_breaking_blanks = 1
+
+        def depart_literal_block(self, node):
+                if self.verbatim:
+                        self.body.append('\n\\end{verbatim}\n')
+                        self.verbatim = 0
+                else:
+                        self.body.append('\n')
+                        self.insert_none_breaking_blanks = 0
+                        self.literal_block = 0
+                self.body.append(self.context.pop())
+
+
+class BeamerWriter (Latex2eWriter):
+        """
+        A docutils writer that modifies the translator and settings for beamer.
+        """
+        settings_spec = BEAMER_SPEC
+        settings_defaults = BEAMER_DEFAULTS
+
+        def __init__(self):
+                Latex2eWriter.__init__(self)
+                self.translator_class = BeamerTranslator
+
+
+
+
+if __name__ == '__main__':
+        description = (
+                "Generates Beamer-flavoured LaTeX for PDF-based presentations." + default_description)
+        publish_cmdline (writer=BeamerWriter(), description=description)
+
+
+### END ######################################################################
+
diff --git a/talk/ep2015/performance/Makefile b/talk/ep2015/performance/Makefile
--- a/talk/ep2015/performance/Makefile
+++ b/talk/ep2015/performance/Makefile
@@ -5,7 +5,7 @@
 # https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414
 
 talk.pdf: talk.rst author.latex stylesheet.latex
-	python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	python ../../bin/rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
 	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
 	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
 	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit

From noreply at buildbot.pypy.org  Mon Jul 20 14:58:07 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 20 Jul 2015 14:58:07 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: Don't use a property in Chunk
Message-ID: <20150720125807.8A4341C0262@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78607:bb115bf87d39
Date: 2015-07-20 13:21 +0100
http://bitbucket.org/pypy/pypy/changeset/bb115bf87d39/

Log:	Don't use a property in Chunk

diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -18,13 +18,10 @@
         self.stop = stop
         self.step = step
         self.lgt = lgt
-
-    @property
-    def out_dim(self):
         if self.step == 0:
-            return 0
+            self.out_dim = 0
         else:
-            return 1
+            self.out_dim = 1
 
     def compute(self, space, base_length, base_stride):
         stride = base_stride * self.step

From noreply at buildbot.pypy.org  Mon Jul 20 16:06:15 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 20 Jul 2015 16:06:15 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: talk
Message-ID: <20150720140615.13E921C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5546:12bda7716989
Date: 2015-07-20 16:06 +0200
http://bitbucket.org/pypy/extradoc/changeset/12bda7716989/

Log:	talk

diff --git a/talk/ep2015/performance/guido.png b/talk/ep2015/performance/guido.png
new file mode 100644
index 0000000000000000000000000000000000000000..273f0b7686a15c691af1880e3ad5a3cc1865c741
GIT binary patch

[cut]

diff --git a/talk/ep2015/performance/talk.pdf b/talk/ep2015/performance/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..874cebbd96fb24e3f93148dfd82afb0985fc7145
GIT binary patch

[cut]

diff --git a/talk/ep2015/performance/talk.rst b/talk/ep2015/performance/talk.rst
--- a/talk/ep2015/performance/talk.rst
+++ b/talk/ep2015/performance/talk.rst
@@ -329,12 +329,25 @@
 Guidos points
 -------------
 
-xzxzcxzcxz
+.. image:: guido.png
+   :scale: 50%
 
 Don't do it on PyPy (or at all)
 -------------------------------
 
-yyyy
+* simple is better than complicated
+
+* avoid string concatenation in the loop
+
+* avoid replacing simple loop with itertools monsters
+
+* "move stuff to C" is (almost) never a good idea
+
+* use ``cffi`` when calling C
+
+* avoid C extensions using CPython C API
+
+* avoid creating classes at runtime
 
 More about PyPy
 ---------------

From noreply at buildbot.pypy.org  Mon Jul 20 18:05:47 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 20 Jul 2015 18:05:47 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: Document the branch
Message-ID: <20150720160547.193F51C0262@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78608:be6254daf0c7
Date: 2015-07-20 17:03 +0100
http://bitbucket.org/pypy/pypy/changeset/be6254daf0c7/

Log:	Document the branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,6 @@
 Improve compatibility with numpy dtypes; handle offsets to create unions,
 fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
 allow subclassing dtype
+
+.. branch: indexing
+Refactor array indexing to support ellipses.

From noreply at buildbot.pypy.org  Mon Jul 20 18:05:48 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 20 Jul 2015 18:05:48 +0200 (CEST)
Subject: [pypy-commit] pypy indexing: Close branch before merging
Message-ID: <20150720160548.341661C089E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: indexing
Changeset: r78609:626f1d3730ba
Date: 2015-07-20 17:04 +0100
http://bitbucket.org/pypy/pypy/changeset/626f1d3730ba/

Log:	Close branch before merging


From noreply at buildbot.pypy.org  Mon Jul 20 18:05:49 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 20 Jul 2015 18:05:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge branch 'indexing'
Message-ID: <20150720160549.749E81C0262@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78610:f9f6f00b3d78
Date: 2015-07-20 17:05 +0100
http://bitbucket.org/pypy/pypy/changeset/f9f6f00b3d78/

Log:	Merge branch 'indexing'

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,6 @@
 Improve compatibility with numpy dtypes; handle offsets to create unions,
 fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
 allow subclassing dtype
+
+.. branch: indexing
+Refactor array indexing to support ellipses.
diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py
--- a/pypy/module/micronumpy/arrayops.py
+++ b/pypy/module/micronumpy/arrayops.py
@@ -5,7 +5,7 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
 from pypy.module.micronumpy.converters import clipmode_converter
 from pypy.module.micronumpy.strides import (
-    Chunk, Chunks, shape_agreement, shape_agreement_multiple)
+    Chunk, new_view, shape_agreement, shape_agreement_multiple)
 from .casting import find_binop_result_dtype, find_result_type
 
 
@@ -148,7 +148,8 @@
             continue
         chunks[axis] = Chunk(axis_start, axis_start + arr.get_shape()[axis], 1,
                              arr.get_shape()[axis])
-        Chunks(chunks).apply(space, res).implementation.setslice(space, arr)
+        view = new_view(space, res, chunks)
+        view.implementation.setslice(space, arr)
         axis_start += arr.get_shape()[axis]
     return res
 
@@ -162,8 +163,9 @@
         shape = [arr.get_shape()[0] * repeats]
         w_res = W_NDimArray.from_shape(space, shape, arr.get_dtype(), w_instance=arr)
         for i in range(repeats):
-            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
-                 orig_size)]).apply(space, w_res).implementation.setslice(space, arr)
+            chunks = [Chunk(i, shape[0] - repeats + i, repeats, orig_size)]
+            view = new_view(space, w_res, chunks)
+            view.implementation.setslice(space, arr)
     else:
         axis = space.int_w(w_axis)
         shape = arr.get_shape()[:]
@@ -174,7 +176,8 @@
         for i in range(repeats):
             chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
                                  orig_size)
-            Chunks(chunks).apply(space, w_res).implementation.setslice(space, arr)
+            view = new_view(space, w_res, chunks)
+            view.implementation.setslice(space, arr)
     return w_res
 
 
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib import jit, rgc
 from rpython.rlib.buffer import Buffer
-from rpython.rlib.debug import make_sure_not_resized, debug_print
+from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
     raw_storage_getitem, raw_storage_setitem, RAW_STORAGE
 from rpython.rtyper.lltypesystem import rffi, lltype, llmemory
@@ -9,13 +9,12 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray, \
     ArrayArgumentException, W_NumpyObject
 from pypy.module.micronumpy.iterators import ArrayIter
-from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
-    RecordChunk, calc_strides, calc_new_strides, shape_agreement,
+from pypy.module.micronumpy.strides import (
+    IntegerChunk, SliceChunk, NewAxisChunk, EllipsisChunk, new_view,
+    calc_strides, calc_new_strides, shape_agreement,
     calculate_broadcast_strides, calc_backstrides, calc_start, is_c_contiguous,
     is_f_contiguous)
 from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.annlowlevel import cast_gcref_to_instance
-from pypy.interpreter.baseobjspace import W_Root
 
 class BaseConcreteArray(object):
     _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]',
@@ -204,6 +203,8 @@
                 if (isinstance(w_item, W_NDimArray) or
                     space.isinstance_w(w_item, space.w_list)):
                     raise ArrayArgumentException
+                elif space.is_w(w_item, space.w_Ellipsis):
+                    raise IndexError
             return self._lookup_by_index(space, view_w)
         if shape_len == 0:
             raise oefmt(space.w_IndexError, "too many indices for array")
@@ -215,39 +216,47 @@
     @jit.unroll_safe
     def _prepare_slice_args(self, space, w_idx):
         if space.isinstance_w(w_idx, space.w_str):
-            idx = space.str_w(w_idx)
-            dtype = self.dtype
-            if not dtype.is_record():
-                raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
-                    "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
-                    "boolean arrays are valid indices")
-            elif idx not in dtype.fields:
-                raise oefmt(space.w_ValueError, "field named %s not found", idx)
-            return RecordChunk(idx)
-        elif (space.isinstance_w(w_idx, space.w_int) or
-                space.isinstance_w(w_idx, space.w_slice)):
+            raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
+                "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
+                "boolean arrays are valid indices")
+        if space.isinstance_w(w_idx, space.w_slice):
             if len(self.get_shape()) == 0:
                 raise oefmt(space.w_ValueError, "cannot slice a 0-d array")
-            return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))])
+            return [SliceChunk(w_idx), EllipsisChunk()]
+        elif space.isinstance_w(w_idx, space.w_int):
+            return [IntegerChunk(w_idx), EllipsisChunk()]
         elif isinstance(w_idx, W_NDimArray) and w_idx.is_scalar():
             w_idx = w_idx.get_scalar_value().item(space)
             if not space.isinstance_w(w_idx, space.w_int) and \
                     not space.isinstance_w(w_idx, space.w_bool):
                 raise OperationError(space.w_IndexError, space.wrap(
                     "arrays used as indices must be of integer (or boolean) type"))
-            return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))])
+            return [IntegerChunk(w_idx), EllipsisChunk()]
         elif space.is_w(w_idx, space.w_None):
-            return Chunks([NewAxisChunk()])
+            return [NewAxisChunk(), EllipsisChunk()]
         result = []
         i = 0
+        has_ellipsis = False
         for w_item in space.fixedview(w_idx):
-            if space.is_w(w_item, space.w_None):
+            if space.is_w(w_item, space.w_Ellipsis):
+                if has_ellipsis:
+                    # in CNumPy, this is only a deprecation warning
+                    raise oefmt(space.w_ValueError,
+                        "an index can only have a single Ellipsis (`...`); "
+                        "replace all but one with slices (`:`).")
+                result.append(EllipsisChunk())
+                has_ellipsis = True
+            elif space.is_w(w_item, space.w_None):
                 result.append(NewAxisChunk())
+            elif space.isinstance_w(w_item, space.w_slice):
+                result.append(SliceChunk(w_item))
+                i += 1
             else:
-                result.append(Chunk(*space.decode_index4(w_item,
-                                                         self.get_shape()[i])))
+                result.append(IntegerChunk(w_item))
                 i += 1
-        return Chunks(result)
+        if not has_ellipsis:
+            result.append(EllipsisChunk())
+        return result
 
     def descr_getitem(self, space, orig_arr, w_index):
         try:
@@ -256,7 +265,7 @@
         except IndexError:
             # not a single result
             chunks = self._prepare_slice_args(space, w_index)
-            return chunks.apply(space, orig_arr)
+            return new_view(space, orig_arr, chunks)
 
     def descr_setitem(self, space, orig_arr, w_index, w_value):
         try:
@@ -265,7 +274,7 @@
         except IndexError:
             w_value = convert_to_array(space, w_value)
             chunks = self._prepare_slice_args(space, w_index)
-            view = chunks.apply(space, orig_arr)
+            view = new_view(space, orig_arr, chunks)
             view.implementation.setslice(space, w_value)
 
     def transpose(self, orig_array, axes=None):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -18,8 +18,9 @@
 from pypy.module.micronumpy.converters import multi_axis_converter, \
     order_converter, shape_converter, searchside_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
-from pypy.module.micronumpy.strides import get_shape_from_iterable, \
-    shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
+from pypy.module.micronumpy.strides import (
+    get_shape_from_iterable, shape_agreement, shape_agreement_multiple,
+    is_c_contiguous, is_f_contiguous, calc_strides, new_view)
 from pypy.module.micronumpy.casting import can_cast_array
 
 
@@ -178,7 +179,7 @@
         if iter_shape is None:
             # w_index is a list of slices, return a view
             chunks = self.implementation._prepare_slice_args(space, w_index)
-            return chunks.apply(space, self)
+            return new_view(space, self, chunks)
         shape = res_shape + self.get_shape()[len(indexes):]
         w_res = W_NDimArray.from_shape(space, shape, self.get_dtype(),
                                        self.get_order(), w_instance=self)
@@ -194,7 +195,7 @@
         if iter_shape is None:
             # w_index is a list of slices
             chunks = self.implementation._prepare_slice_args(space, w_index)
-            view = chunks.apply(space, self)
+            view = new_view(space, self, chunks)
             view.implementation.setslice(space, val_arr)
             return
         if support.product(iter_shape) == 0:
@@ -203,6 +204,10 @@
                                prefix)
 
     def descr_getitem(self, space, w_idx):
+        if self.get_dtype().is_record():
+            if space.isinstance_w(w_idx, space.w_str):
+                idx = space.str_w(w_idx)
+                return self.getfield(space, idx)
         if space.is_w(w_idx, space.w_Ellipsis):
             return self
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool():
@@ -229,6 +234,13 @@
         self.implementation.setitem_index(space, index_list, w_value)
 
     def descr_setitem(self, space, w_idx, w_value):
+        if self.get_dtype().is_record():
+            if space.isinstance_w(w_idx, space.w_str):
+                idx = space.str_w(w_idx)
+                view = self.getfield(space, idx)
+                w_value = convert_to_array(space, w_value)
+                view.implementation.setslice(space, w_value)
+                return
         if space.is_w(w_idx, space.w_Ellipsis):
             self.implementation.setslice(space, convert_to_array(space, w_value))
             return
@@ -241,6 +253,28 @@
         except ArrayArgumentException:
             self.setitem_array_int(space, w_idx, w_value)
 
+    def getfield(self, space, field):
+        dtype = self.get_dtype()
+        if field not in dtype.fields:
+            raise oefmt(space.w_ValueError, "field named %s not found", field)
+        arr = self.implementation
+        ofs, subdtype = arr.dtype.fields[field][:2]
+        # ofs only changes start
+        # create a view of the original array by extending
+        # the shape, strides, backstrides of the array
+        strides, backstrides = calc_strides(subdtype.shape,
+                                            subdtype.subdtype, arr.order)
+        final_shape = arr.shape + subdtype.shape
+        final_strides = arr.get_strides() + strides
+        final_backstrides = arr.get_backstrides() + backstrides
+        final_dtype = subdtype
+        if subdtype.subdtype:
+            final_dtype = subdtype.subdtype
+        return W_NDimArray.new_slice(space, arr.start + ofs, final_strides,
+                                     final_backstrides,
+                                     final_shape, arr, self, final_dtype)
+
+
     def descr_delitem(self, space, w_idx):
         raise OperationError(space.w_ValueError, space.wrap(
             "cannot delete array elements"))
@@ -1298,7 +1332,6 @@
 def descr_new_array(space, w_subtype, w_shape, w_dtype=None, w_buffer=None,
                     offset=0, w_strides=None, w_order=None):
     from pypy.module.micronumpy.concrete import ConcreteArray
-    from pypy.module.micronumpy.strides import calc_strides
     dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
         space.gettypefor(descriptor.W_Dtype), w_dtype))
     shape = shape_converter(space, w_shape, dtype)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,78 +10,92 @@
     pass
 
 
-class RecordChunk(BaseChunk):
-    def __init__(self, name):
-        self.name = name
-
-    def apply(self, space, orig_arr):
-        arr = orig_arr.implementation
-        ofs, subdtype = arr.dtype.fields[self.name][:2]
-        # ofs only changes start
-        # create a view of the original array by extending
-        # the shape, strides, backstrides of the array
-        strides, backstrides = calc_strides(subdtype.shape,
-                                            subdtype.subdtype, arr.order)
-        final_shape = arr.shape + subdtype.shape
-        final_strides = arr.get_strides() + strides
-        final_backstrides = arr.get_backstrides() + backstrides
-        final_dtype = subdtype
-        if subdtype.subdtype:
-            final_dtype = subdtype.subdtype
-        return W_NDimArray.new_slice(space, arr.start + ofs, final_strides,
-                                     final_backstrides,
-                                     final_shape, arr, orig_arr, final_dtype)
-
-
-class Chunks(BaseChunk):
-    def __init__(self, l):
-        self.l = l
-
-    @jit.unroll_safe
-    def extend_shape(self, old_shape):
-        shape = []
-        i = -1
-        for i, c in enumerate_chunks(self.l):
-            if c.step != 0:
-                shape.append(c.lgt)
-        s = i + 1
-        assert s >= 0
-        return shape[:] + old_shape[s:]
-
-    def apply(self, space, orig_arr):
-        arr = orig_arr.implementation
-        shape = self.extend_shape(arr.shape)
-        r = calculate_slice_strides(arr.shape, arr.start, arr.get_strides(),
-                                    arr.get_backstrides(), self.l)
-        _, start, strides, backstrides = r
-        return W_NDimArray.new_slice(space, start, strides[:], backstrides[:],
-                                     shape[:], arr, orig_arr)
-
-
 class Chunk(BaseChunk):
-    axis_step = 1
+    input_dim = 1
 
     def __init__(self, start, stop, step, lgt):
         self.start = start
         self.stop = stop
         self.step = step
         self.lgt = lgt
+        if self.step == 0:
+            self.out_dim = 0
+        else:
+            self.out_dim = 1
+
+    def compute(self, space, base_length, base_stride):
+        stride = base_stride * self.step
+        backstride = base_stride * max(0, self.lgt - 1) * self.step
+        return self.start, self.lgt, stride, backstride
 
     def __repr__(self):
         return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step,
                                           self.lgt)
 
+class IntegerChunk(BaseChunk):
+    input_dim = 1
+    out_dim = 0
+    def __init__(self, w_idx):
+        self.w_idx = w_idx
+
+    def compute(self, space, base_length, base_stride):
+        start, _, _, _ = space.decode_index4(self.w_idx, base_length)
+        return start, 0, 0, 0
+
+
+class SliceChunk(BaseChunk):
+    input_dim = 1
+    out_dim = 1
+
+    def __init__(self, w_slice):
+        self.w_slice = w_slice
+
+    def compute(self, space, base_length, base_stride):
+        start, stop, step, length = space.decode_index4(self.w_slice, base_length)
+        stride = base_stride * step
+        backstride = base_stride * max(0, length - 1) * step
+        return start, length, stride, backstride
 
 class NewAxisChunk(Chunk):
-    start = 0
-    stop = 1
-    step = 1
-    lgt = 1
-    axis_step = 0 # both skip this axis in calculate_slice_strides and set stride => 0
+    input_dim = 0
+    out_dim = 1
 
     def __init__(self):
         pass
 
+    def compute(self, space, base_length, base_stride):
+        return 0, 1, 0, 0
+
+class EllipsisChunk(BaseChunk):
+    input_dim = 0
+    out_dim = 0
+    def __init__(self):
+        pass
+
+    def compute(self, space, base_length, base_stride):
+        backstride = base_stride * max(0, base_length - 1)
+        return 0, base_length, base_stride, backstride
+
+
+def new_view(space, w_arr, chunks):
+    arr = w_arr.implementation
+    r = calculate_slice_strides(space, arr.shape, arr.start, arr.get_strides(),
+                                arr.get_backstrides(), chunks)
+    shape, start, strides, backstrides = r
+    return W_NDimArray.new_slice(space, start, strides[:], backstrides[:],
+                                 shape[:], arr, w_arr)
+
+ at jit.unroll_safe
+def _extend_shape(old_shape, chunks):
+    shape = []
+    i = -1
+    for i, c in enumerate_chunks(chunks):
+        if c.out_dim > 0:
+            shape.append(c.lgt)
+    s = i + 1
+    assert s >= 0
+    return shape[:] + old_shape[s:]
+
 
 class BaseTransform(object):
     pass
@@ -103,41 +117,56 @@
     result = []
     i = -1
     for chunk in chunks:
-        i += chunk.axis_step
+        i += chunk.input_dim
         result.append((i, chunk))
     return result
 
 
- at jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks:
+ at jit.look_inside_iff(lambda space, shape, start, strides, backstrides, chunks:
                      jit.isconstant(len(chunks)))
-def calculate_slice_strides(shape, start, strides, backstrides, chunks):
+def calculate_slice_strides(space, shape, start, strides, backstrides, chunks):
+    """
+    Note: `chunks` must contain exactly one EllipsisChunk object.
+    """
     size = 0
+    used_dims = 0
     for chunk in chunks:
-        if chunk.step != 0:
-            size += 1
-    rstrides = [0] * size
-    rbackstrides = [0] * size
+        used_dims += chunk.input_dim
+        size += chunk.out_dim
+    if used_dims > len(shape):
+        raise oefmt(space.w_IndexError, "too many indices for array")
+    else:
+        extra_dims = len(shape) - used_dims
+    rstrides = [0] * (size + extra_dims)
+    rbackstrides = [0] * (size + extra_dims)
     rstart = start
-    rshape = [0] * size
-    i = -1
-    j = 0
-    for i, chunk in enumerate_chunks(chunks):
-        try:
-            s_i = strides[i]
-        except IndexError:
+    rshape = [0] * (size + extra_dims)
+    rstart = start
+    i = 0  # index of the current dimension in the input array
+    j = 0  # index of the current dimension in the result view
+    for chunk in chunks:
+        if isinstance(chunk, NewAxisChunk):
+            rshape[j] = 1
+            j += 1
             continue
-        if chunk.step != 0:
-            rstrides[j] = s_i * chunk.step * chunk.axis_step
-            rbackstrides[j] = s_i * max(0, chunk.lgt - 1) * chunk.step
-            rshape[j] = chunk.lgt
-            j += 1
-        rstart += s_i * chunk.start
-    # add a reminder
-    s = i + 1
-    assert s >= 0
-    rstrides += strides[s:]
-    rbackstrides += backstrides[s:]
-    rshape += shape[s:]
+        elif isinstance(chunk, EllipsisChunk):
+            for k in range(extra_dims):
+                start, length, stride, backstride = chunk.compute(
+                        space, shape[i], strides[i])
+                rshape[j] = length
+                rstrides[j] = stride
+                rbackstrides[j] = backstride
+                j += 1
+                i += 1
+            continue
+        start, length, stride, backstride = chunk.compute(space, shape[i], strides[i])
+        if chunk.out_dim == 1:
+            rshape[j] = length
+            rstrides[j] = stride
+            rbackstrides[j] = backstride
+            j += chunk.out_dim
+        rstart += strides[i] * start
+        i += chunk.input_dim
     return rshape, rstart, rstrides, rbackstrides
 
 
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -4,7 +4,7 @@
 
 from pypy.conftest import option
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
-from pypy.module.micronumpy.strides import Chunk, Chunks
+from pypy.module.micronumpy.strides import Chunk, new_view, EllipsisChunk
 from pypy.module.micronumpy.ndarray import W_NDimArray
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
@@ -22,7 +22,9 @@
 
 
 def create_slice(space, a, chunks):
-    return Chunks(chunks).apply(space, W_NDimArray(a)).implementation
+    if not any(isinstance(c, EllipsisChunk) for c in chunks):
+        chunks.append(EllipsisChunk())
+    return new_view(space, W_NDimArray(a), chunks).implementation
 
 
 def create_array(*args, **kwargs):
@@ -2488,6 +2490,13 @@
         assert b.shape == b[...].shape
         assert (b == b[...]).all()
 
+        a = np.arange(6).reshape(2, 3)
+        if '__pypy__' in sys.builtin_module_names:
+            raises(ValueError, "a[..., ...]")
+        b = a [..., 0]
+        assert (b == [0, 3]).all()
+        assert b.base is a
+
     def test_empty_indexing(self):
         import numpy as np
         r = np.ones(3)

From noreply at buildbot.pypy.org  Mon Jul 20 18:25:31 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 20 Jul 2015 18:25:31 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: all / any removed rffi.cast when numpy
 Bool is created in convert_to
Message-ID: <20150720162531.0C4FE1C0262@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78611:51d259166ee5
Date: 2015-07-20 18:19 +0200
http://bitbucket.org/pypy/pypy/changeset/51d259166ee5/

Log:	all / any removed rffi.cast when numpy Bool is created in convert_to

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -358,8 +358,7 @@
 
     @specialize.argtype(1)
     def box(self, value):
-        box = Primitive.box(self, value)
-        if box.value:
+        if value:
             return self._True
         else:
             return self._False

From noreply at buildbot.pypy.org  Mon Jul 20 18:31:18 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 20 Jul 2015 18:31:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal,
	plan_rich) fix for JIT force_cast(Bool, xyz)
Message-ID: <20150720163118.D02031C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78612:d287a8893182
Date: 2015-07-20 18:29 +0200
http://bitbucket.org/pypy/pypy/changeset/d287a8893182/

Log:	(fijal, plan_rich) fix for JIT force_cast(Bool, xyz)

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1273,7 +1273,9 @@
             return
 
         result = []
-        if min2:
+        if v_result.concretetype is lltype.Bool:
+            result.append(SpaceOperation('int_is_true', [v_arg], v_result))
+        elif min2:
             c_bytes = Constant(size2, lltype.Signed)
             result.append(SpaceOperation('int_signext', [v_arg, c_bytes],
                                          v_result))
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -795,6 +795,7 @@
             (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
             (rffi.ULONG, rffi.SHORT, "int_signext %i0, $2 -> %i1"),
             (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"),
+            (rffi.LONG, lltype.Bool, "int_is_true %i0 -> %i1"),
             (rffi.ULONG, rffi.LONG, ""),
             (rffi.ULONG, rffi.ULONG, ""),
             ]:
@@ -1022,6 +1023,7 @@
     """Check that the test is correctly written..."""
     import re
     r = re.compile('(\w+) \%i\d, \$(-?\d+)')
+    r2 = re.compile('(\w+) \%i\d')
     #
     value = rffi.cast(FROM, value)
     value = rffi.cast(lltype.Signed, value)
@@ -1031,6 +1033,8 @@
     #
     for op in operations:
         match = r.match(op)
+        if match is None:
+            match = r2.match(op)
         assert match, "line %r does not match regexp" % (op,)
         opname = match.group(1)
         if opname == 'int_and':
@@ -1038,6 +1042,8 @@
         elif opname == 'int_signext':
             numbytes = int(match.group(2))
             value = int_signext(value, numbytes)
+        elif opname == 'int_is_true':
+            value = bool(value)
         else:
             assert 0, opname
     #

From noreply at buildbot.pypy.org  Mon Jul 20 18:31:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 20 Jul 2015 18:31:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150720163119.F320C1C0262@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78613:4b6cfd24db0b
Date: 2015-07-20 18:31 +0200
http://bitbucket.org/pypy/pypy/changeset/4b6cfd24db0b/

Log:	merge

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,6 @@
 Improve compatibility with numpy dtypes; handle offsets to create unions,
 fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
 allow subclassing dtype
+
+.. branch: indexing
+Refactor array indexing to support ellipses.
diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py
--- a/pypy/module/micronumpy/arrayops.py
+++ b/pypy/module/micronumpy/arrayops.py
@@ -5,7 +5,7 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
 from pypy.module.micronumpy.converters import clipmode_converter
 from pypy.module.micronumpy.strides import (
-    Chunk, Chunks, shape_agreement, shape_agreement_multiple)
+    Chunk, new_view, shape_agreement, shape_agreement_multiple)
 from .casting import find_binop_result_dtype, find_result_type
 
 
@@ -148,7 +148,8 @@
             continue
         chunks[axis] = Chunk(axis_start, axis_start + arr.get_shape()[axis], 1,
                              arr.get_shape()[axis])
-        Chunks(chunks).apply(space, res).implementation.setslice(space, arr)
+        view = new_view(space, res, chunks)
+        view.implementation.setslice(space, arr)
         axis_start += arr.get_shape()[axis]
     return res
 
@@ -162,8 +163,9 @@
         shape = [arr.get_shape()[0] * repeats]
         w_res = W_NDimArray.from_shape(space, shape, arr.get_dtype(), w_instance=arr)
         for i in range(repeats):
-            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
-                 orig_size)]).apply(space, w_res).implementation.setslice(space, arr)
+            chunks = [Chunk(i, shape[0] - repeats + i, repeats, orig_size)]
+            view = new_view(space, w_res, chunks)
+            view.implementation.setslice(space, arr)
     else:
         axis = space.int_w(w_axis)
         shape = arr.get_shape()[:]
@@ -174,7 +176,8 @@
         for i in range(repeats):
             chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
                                  orig_size)
-            Chunks(chunks).apply(space, w_res).implementation.setslice(space, arr)
+            view = new_view(space, w_res, chunks)
+            view.implementation.setslice(space, arr)
     return w_res
 
 
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib import jit, rgc
 from rpython.rlib.buffer import Buffer
-from rpython.rlib.debug import make_sure_not_resized, debug_print
+from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
     raw_storage_getitem, raw_storage_setitem, RAW_STORAGE
 from rpython.rtyper.lltypesystem import rffi, lltype, llmemory
@@ -9,13 +9,12 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray, \
     ArrayArgumentException, W_NumpyObject
 from pypy.module.micronumpy.iterators import ArrayIter
-from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
-    RecordChunk, calc_strides, calc_new_strides, shape_agreement,
+from pypy.module.micronumpy.strides import (
+    IntegerChunk, SliceChunk, NewAxisChunk, EllipsisChunk, new_view,
+    calc_strides, calc_new_strides, shape_agreement,
     calculate_broadcast_strides, calc_backstrides, calc_start, is_c_contiguous,
     is_f_contiguous)
 from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.annlowlevel import cast_gcref_to_instance
-from pypy.interpreter.baseobjspace import W_Root
 
 class BaseConcreteArray(object):
     _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]',
@@ -204,6 +203,8 @@
                 if (isinstance(w_item, W_NDimArray) or
                     space.isinstance_w(w_item, space.w_list)):
                     raise ArrayArgumentException
+                elif space.is_w(w_item, space.w_Ellipsis):
+                    raise IndexError
             return self._lookup_by_index(space, view_w)
         if shape_len == 0:
             raise oefmt(space.w_IndexError, "too many indices for array")
@@ -215,39 +216,47 @@
     @jit.unroll_safe
     def _prepare_slice_args(self, space, w_idx):
         if space.isinstance_w(w_idx, space.w_str):
-            idx = space.str_w(w_idx)
-            dtype = self.dtype
-            if not dtype.is_record():
-                raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
-                    "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
-                    "boolean arrays are valid indices")
-            elif idx not in dtype.fields:
-                raise oefmt(space.w_ValueError, "field named %s not found", idx)
-            return RecordChunk(idx)
-        elif (space.isinstance_w(w_idx, space.w_int) or
-                space.isinstance_w(w_idx, space.w_slice)):
+            raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
+                "ellipsis (`...`), numpy.newaxis (`None`) and integer or "
+                "boolean arrays are valid indices")
+        if space.isinstance_w(w_idx, space.w_slice):
             if len(self.get_shape()) == 0:
                 raise oefmt(space.w_ValueError, "cannot slice a 0-d array")
-            return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))])
+            return [SliceChunk(w_idx), EllipsisChunk()]
+        elif space.isinstance_w(w_idx, space.w_int):
+            return [IntegerChunk(w_idx), EllipsisChunk()]
         elif isinstance(w_idx, W_NDimArray) and w_idx.is_scalar():
             w_idx = w_idx.get_scalar_value().item(space)
             if not space.isinstance_w(w_idx, space.w_int) and \
                     not space.isinstance_w(w_idx, space.w_bool):
                 raise OperationError(space.w_IndexError, space.wrap(
                     "arrays used as indices must be of integer (or boolean) type"))
-            return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))])
+            return [IntegerChunk(w_idx), EllipsisChunk()]
         elif space.is_w(w_idx, space.w_None):
-            return Chunks([NewAxisChunk()])
+            return [NewAxisChunk(), EllipsisChunk()]
         result = []
         i = 0
+        has_ellipsis = False
         for w_item in space.fixedview(w_idx):
-            if space.is_w(w_item, space.w_None):
+            if space.is_w(w_item, space.w_Ellipsis):
+                if has_ellipsis:
+                    # in CNumPy, this is only a deprecation warning
+                    raise oefmt(space.w_ValueError,
+                        "an index can only have a single Ellipsis (`...`); "
+                        "replace all but one with slices (`:`).")
+                result.append(EllipsisChunk())
+                has_ellipsis = True
+            elif space.is_w(w_item, space.w_None):
                 result.append(NewAxisChunk())
+            elif space.isinstance_w(w_item, space.w_slice):
+                result.append(SliceChunk(w_item))
+                i += 1
             else:
-                result.append(Chunk(*space.decode_index4(w_item,
-                                                         self.get_shape()[i])))
+                result.append(IntegerChunk(w_item))
                 i += 1
-        return Chunks(result)
+        if not has_ellipsis:
+            result.append(EllipsisChunk())
+        return result
 
     def descr_getitem(self, space, orig_arr, w_index):
         try:
@@ -256,7 +265,7 @@
         except IndexError:
             # not a single result
             chunks = self._prepare_slice_args(space, w_index)
-            return chunks.apply(space, orig_arr)
+            return new_view(space, orig_arr, chunks)
 
     def descr_setitem(self, space, orig_arr, w_index, w_value):
         try:
@@ -265,7 +274,7 @@
         except IndexError:
             w_value = convert_to_array(space, w_value)
             chunks = self._prepare_slice_args(space, w_index)
-            view = chunks.apply(space, orig_arr)
+            view = new_view(space, orig_arr, chunks)
             view.implementation.setslice(space, w_value)
 
     def transpose(self, orig_array, axes=None):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -18,8 +18,9 @@
 from pypy.module.micronumpy.converters import multi_axis_converter, \
     order_converter, shape_converter, searchside_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
-from pypy.module.micronumpy.strides import get_shape_from_iterable, \
-    shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
+from pypy.module.micronumpy.strides import (
+    get_shape_from_iterable, shape_agreement, shape_agreement_multiple,
+    is_c_contiguous, is_f_contiguous, calc_strides, new_view)
 from pypy.module.micronumpy.casting import can_cast_array
 
 
@@ -178,7 +179,7 @@
         if iter_shape is None:
             # w_index is a list of slices, return a view
             chunks = self.implementation._prepare_slice_args(space, w_index)
-            return chunks.apply(space, self)
+            return new_view(space, self, chunks)
         shape = res_shape + self.get_shape()[len(indexes):]
         w_res = W_NDimArray.from_shape(space, shape, self.get_dtype(),
                                        self.get_order(), w_instance=self)
@@ -194,7 +195,7 @@
         if iter_shape is None:
             # w_index is a list of slices
             chunks = self.implementation._prepare_slice_args(space, w_index)
-            view = chunks.apply(space, self)
+            view = new_view(space, self, chunks)
             view.implementation.setslice(space, val_arr)
             return
         if support.product(iter_shape) == 0:
@@ -203,6 +204,10 @@
                                prefix)
 
     def descr_getitem(self, space, w_idx):
+        if self.get_dtype().is_record():
+            if space.isinstance_w(w_idx, space.w_str):
+                idx = space.str_w(w_idx)
+                return self.getfield(space, idx)
         if space.is_w(w_idx, space.w_Ellipsis):
             return self
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool():
@@ -229,6 +234,13 @@
         self.implementation.setitem_index(space, index_list, w_value)
 
     def descr_setitem(self, space, w_idx, w_value):
+        if self.get_dtype().is_record():
+            if space.isinstance_w(w_idx, space.w_str):
+                idx = space.str_w(w_idx)
+                view = self.getfield(space, idx)
+                w_value = convert_to_array(space, w_value)
+                view.implementation.setslice(space, w_value)
+                return
         if space.is_w(w_idx, space.w_Ellipsis):
             self.implementation.setslice(space, convert_to_array(space, w_value))
             return
@@ -241,6 +253,28 @@
         except ArrayArgumentException:
             self.setitem_array_int(space, w_idx, w_value)
 
+    def getfield(self, space, field):
+        dtype = self.get_dtype()
+        if field not in dtype.fields:
+            raise oefmt(space.w_ValueError, "field named %s not found", field)
+        arr = self.implementation
+        ofs, subdtype = arr.dtype.fields[field][:2]
+        # ofs only changes start
+        # create a view of the original array by extending
+        # the shape, strides, backstrides of the array
+        strides, backstrides = calc_strides(subdtype.shape,
+                                            subdtype.subdtype, arr.order)
+        final_shape = arr.shape + subdtype.shape
+        final_strides = arr.get_strides() + strides
+        final_backstrides = arr.get_backstrides() + backstrides
+        final_dtype = subdtype
+        if subdtype.subdtype:
+            final_dtype = subdtype.subdtype
+        return W_NDimArray.new_slice(space, arr.start + ofs, final_strides,
+                                     final_backstrides,
+                                     final_shape, arr, self, final_dtype)
+
+
     def descr_delitem(self, space, w_idx):
         raise OperationError(space.w_ValueError, space.wrap(
             "cannot delete array elements"))
@@ -1298,7 +1332,6 @@
 def descr_new_array(space, w_subtype, w_shape, w_dtype=None, w_buffer=None,
                     offset=0, w_strides=None, w_order=None):
     from pypy.module.micronumpy.concrete import ConcreteArray
-    from pypy.module.micronumpy.strides import calc_strides
     dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
         space.gettypefor(descriptor.W_Dtype), w_dtype))
     shape = shape_converter(space, w_shape, dtype)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,78 +10,92 @@
     pass
 
 
-class RecordChunk(BaseChunk):
-    def __init__(self, name):
-        self.name = name
-
-    def apply(self, space, orig_arr):
-        arr = orig_arr.implementation
-        ofs, subdtype = arr.dtype.fields[self.name][:2]
-        # ofs only changes start
-        # create a view of the original array by extending
-        # the shape, strides, backstrides of the array
-        strides, backstrides = calc_strides(subdtype.shape,
-                                            subdtype.subdtype, arr.order)
-        final_shape = arr.shape + subdtype.shape
-        final_strides = arr.get_strides() + strides
-        final_backstrides = arr.get_backstrides() + backstrides
-        final_dtype = subdtype
-        if subdtype.subdtype:
-            final_dtype = subdtype.subdtype
-        return W_NDimArray.new_slice(space, arr.start + ofs, final_strides,
-                                     final_backstrides,
-                                     final_shape, arr, orig_arr, final_dtype)
-
-
-class Chunks(BaseChunk):
-    def __init__(self, l):
-        self.l = l
-
-    @jit.unroll_safe
-    def extend_shape(self, old_shape):
-        shape = []
-        i = -1
-        for i, c in enumerate_chunks(self.l):
-            if c.step != 0:
-                shape.append(c.lgt)
-        s = i + 1
-        assert s >= 0
-        return shape[:] + old_shape[s:]
-
-    def apply(self, space, orig_arr):
-        arr = orig_arr.implementation
-        shape = self.extend_shape(arr.shape)
-        r = calculate_slice_strides(arr.shape, arr.start, arr.get_strides(),
-                                    arr.get_backstrides(), self.l)
-        _, start, strides, backstrides = r
-        return W_NDimArray.new_slice(space, start, strides[:], backstrides[:],
-                                     shape[:], arr, orig_arr)
-
-
 class Chunk(BaseChunk):
-    axis_step = 1
+    input_dim = 1
 
     def __init__(self, start, stop, step, lgt):
         self.start = start
         self.stop = stop
         self.step = step
         self.lgt = lgt
+        if self.step == 0:
+            self.out_dim = 0
+        else:
+            self.out_dim = 1
+
+    def compute(self, space, base_length, base_stride):
+        stride = base_stride * self.step
+        backstride = base_stride * max(0, self.lgt - 1) * self.step
+        return self.start, self.lgt, stride, backstride
 
     def __repr__(self):
         return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step,
                                           self.lgt)
 
+class IntegerChunk(BaseChunk):
+    input_dim = 1
+    out_dim = 0
+    def __init__(self, w_idx):
+        self.w_idx = w_idx
+
+    def compute(self, space, base_length, base_stride):
+        start, _, _, _ = space.decode_index4(self.w_idx, base_length)
+        return start, 0, 0, 0
+
+
+class SliceChunk(BaseChunk):
+    input_dim = 1
+    out_dim = 1
+
+    def __init__(self, w_slice):
+        self.w_slice = w_slice
+
+    def compute(self, space, base_length, base_stride):
+        start, stop, step, length = space.decode_index4(self.w_slice, base_length)
+        stride = base_stride * step
+        backstride = base_stride * max(0, length - 1) * step
+        return start, length, stride, backstride
 
 class NewAxisChunk(Chunk):
-    start = 0
-    stop = 1
-    step = 1
-    lgt = 1
-    axis_step = 0 # both skip this axis in calculate_slice_strides and set stride => 0
+    input_dim = 0
+    out_dim = 1
 
     def __init__(self):
         pass
 
+    def compute(self, space, base_length, base_stride):
+        return 0, 1, 0, 0
+
+class EllipsisChunk(BaseChunk):
+    input_dim = 0
+    out_dim = 0
+    def __init__(self):
+        pass
+
+    def compute(self, space, base_length, base_stride):
+        backstride = base_stride * max(0, base_length - 1)
+        return 0, base_length, base_stride, backstride
+
+
+def new_view(space, w_arr, chunks):
+    arr = w_arr.implementation
+    r = calculate_slice_strides(space, arr.shape, arr.start, arr.get_strides(),
+                                arr.get_backstrides(), chunks)
+    shape, start, strides, backstrides = r
+    return W_NDimArray.new_slice(space, start, strides[:], backstrides[:],
+                                 shape[:], arr, w_arr)
+
+ at jit.unroll_safe
+def _extend_shape(old_shape, chunks):
+    shape = []
+    i = -1
+    for i, c in enumerate_chunks(chunks):
+        if c.out_dim > 0:
+            shape.append(c.lgt)
+    s = i + 1
+    assert s >= 0
+    return shape[:] + old_shape[s:]
+
 
 class BaseTransform(object):
     pass
@@ -103,41 +117,56 @@
     result = []
     i = -1
     for chunk in chunks:
-        i += chunk.axis_step
+        i += chunk.input_dim
         result.append((i, chunk))
     return result
 
 
- at jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks:
+ at jit.look_inside_iff(lambda space, shape, start, strides, backstrides, chunks:
                      jit.isconstant(len(chunks)))
-def calculate_slice_strides(shape, start, strides, backstrides, chunks):
+def calculate_slice_strides(space, shape, start, strides, backstrides, chunks):
+    """
+    Note: `chunks` must contain exactly one EllipsisChunk object.
+    """
     size = 0
+    used_dims = 0
     for chunk in chunks:
-        if chunk.step != 0:
-            size += 1
-    rstrides = [0] * size
-    rbackstrides = [0] * size
+        used_dims += chunk.input_dim
+        size += chunk.out_dim
+    if used_dims > len(shape):
+        raise oefmt(space.w_IndexError, "too many indices for array")
+    else:
+        extra_dims = len(shape) - used_dims
+    rstrides = [0] * (size + extra_dims)
+    rbackstrides = [0] * (size + extra_dims)
     rstart = start
-    rshape = [0] * size
-    i = -1
-    j = 0
-    for i, chunk in enumerate_chunks(chunks):
-        try:
-            s_i = strides[i]
-        except IndexError:
+    rshape = [0] * (size + extra_dims)
+    rstart = start
+    i = 0  # index of the current dimension in the input array
+    j = 0  # index of the current dimension in the result view
+    for chunk in chunks:
+        if isinstance(chunk, NewAxisChunk):
+            rshape[j] = 1
+            j += 1
             continue
-        if chunk.step != 0:
-            rstrides[j] = s_i * chunk.step * chunk.axis_step
-            rbackstrides[j] = s_i * max(0, chunk.lgt - 1) * chunk.step
-            rshape[j] = chunk.lgt
-            j += 1
-        rstart += s_i * chunk.start
-    # add a reminder
-    s = i + 1
-    assert s >= 0
-    rstrides += strides[s:]
-    rbackstrides += backstrides[s:]
-    rshape += shape[s:]
+        elif isinstance(chunk, EllipsisChunk):
+            for k in range(extra_dims):
+                start, length, stride, backstride = chunk.compute(
+                        space, shape[i], strides[i])
+                rshape[j] = length
+                rstrides[j] = stride
+                rbackstrides[j] = backstride
+                j += 1
+                i += 1
+            continue
+        start, length, stride, backstride = chunk.compute(space, shape[i], strides[i])
+        if chunk.out_dim == 1:
+            rshape[j] = length
+            rstrides[j] = stride
+            rbackstrides[j] = backstride
+            j += chunk.out_dim
+        rstart += strides[i] * start
+        i += chunk.input_dim
     return rshape, rstart, rstrides, rbackstrides
 
 
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -4,7 +4,7 @@
 
 from pypy.conftest import option
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
-from pypy.module.micronumpy.strides import Chunk, Chunks
+from pypy.module.micronumpy.strides import Chunk, new_view, EllipsisChunk
 from pypy.module.micronumpy.ndarray import W_NDimArray
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
@@ -22,7 +22,9 @@
 
 
 def create_slice(space, a, chunks):
-    return Chunks(chunks).apply(space, W_NDimArray(a)).implementation
+    if not any(isinstance(c, EllipsisChunk) for c in chunks):
+        chunks.append(EllipsisChunk())
+    return new_view(space, W_NDimArray(a), chunks).implementation
 
 
 def create_array(*args, **kwargs):
@@ -2488,6 +2490,13 @@
         assert b.shape == b[...].shape
         assert (b == b[...]).all()
 
+        a = np.arange(6).reshape(2, 3)
+        if '__pypy__' in sys.builtin_module_names:
+            raises(ValueError, "a[..., ...]")
+        b = a [..., 0]
+        assert (b == [0, 3]).all()
+        assert b.base is a
+
     def test_empty_indexing(self):
         import numpy as np
         r = np.ones(3)

From noreply at buildbot.pypy.org  Mon Jul 20 19:10:06 2015
From: noreply at buildbot.pypy.org (krono)
Date: Mon, 20 Jul 2015 19:10:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: Darwin: require 10.5 everywhere (for
	@rpath to work)
Message-ID: <20150720171006.5D0EA1C089E@cobra.cs.uni-duesseldorf.de>

Author: Tobias Pape <tobias at netshed.de>
Branch: 
Changeset: r78614:70a7870e6d1c
Date: 2015-07-20 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/70a7870e6d1c/

Log:	Darwin: require 10.5 everywhere (for @rpath to work)

diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -2,12 +2,22 @@
 
 from rpython.translator.platform import posix
 
+#
+# Although Intel 32bit is supported since Apple Mac OS X 10.4, (and PPC since, ever)
+# the @rpath handling used in Darwin._args_for_shared is only availabe
+# since 10.5, so we use that as minimum requirement.
+#
+DARWIN_VERSION_MIN = '-mmacosx-version-min=10.5'
+
 class Darwin(posix.BasePosix):
     name = "darwin"
 
     standalone_only = ('-mdynamic-no-pic',)
     shared_only = ()
 
+    link_flags = (DARWIN_VERSION_MIN,)
+    cflags = ('-O3', '-fomit-frame-pointer', DARWIN_VERSION_MIN)
+
     so_ext = 'dylib'
     DEFAULT_CC = 'clang'
     rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path/']
@@ -64,20 +74,13 @@
                                 icon=icon)
         return mk
 
+class Darwin_PowerPC(Darwin):#xxx fixme, mwp
+    name = "darwin_powerpc"
 
 class Darwin_i386(Darwin):
     name = "darwin_i386"
     DEFAULT_CC = 'clang -arch i386'
-    link_flags = ('-mmacosx-version-min=10.4',)
-    cflags = ('-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4')
-
-class Darwin_PowerPC(Darwin):#xxx fixme, mwp
-    name = "darwin_powerpc"
-    link_flags = ()
-    cflags = ('-O3', '-fomit-frame-pointer')
 
 class Darwin_x86_64(Darwin):
     name = "darwin_x86_64"
     DEFAULT_CC = 'clang -arch x86_64'
-    link_flags = ('-mmacosx-version-min=10.5',)
-    cflags = ('-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.5')

From noreply at buildbot.pypy.org  Mon Jul 20 19:10:07 2015
From: noreply at buildbot.pypy.org (krono)
Date: Mon, 20 Jul 2015 19:10:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: Automated merge with
	ssh://bitbucket.org/pypy/pypy
Message-ID: <20150720171007.C93C41C089E@cobra.cs.uni-duesseldorf.de>

Author: Tobias Pape <tobias at netshed.de>
Branch: 
Changeset: r78615:0ac77039f462
Date: 2015-07-20 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/0ac77039f462/

Log:	Automated merge with ssh://bitbucket.org/pypy/pypy

diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -2,12 +2,22 @@
 
 from rpython.translator.platform import posix
 
+#
+# Although Intel 32bit is supported since Apple Mac OS X 10.4, (and PPC since, ever)
+# the @rpath handling used in Darwin._args_for_shared is only availabe
+# since 10.5, so we use that as minimum requirement.
+#
+DARWIN_VERSION_MIN = '-mmacosx-version-min=10.5'
+
 class Darwin(posix.BasePosix):
     name = "darwin"
 
     standalone_only = ('-mdynamic-no-pic',)
     shared_only = ()
 
+    link_flags = (DARWIN_VERSION_MIN,)
+    cflags = ('-O3', '-fomit-frame-pointer', DARWIN_VERSION_MIN)
+
     so_ext = 'dylib'
     DEFAULT_CC = 'clang'
     rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path/']
@@ -64,20 +74,13 @@
                                 icon=icon)
         return mk
 
+class Darwin_PowerPC(Darwin):#xxx fixme, mwp
+    name = "darwin_powerpc"
 
 class Darwin_i386(Darwin):
     name = "darwin_i386"
     DEFAULT_CC = 'clang -arch i386'
-    link_flags = ('-mmacosx-version-min=10.4',)
-    cflags = ('-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4')
-
-class Darwin_PowerPC(Darwin):#xxx fixme, mwp
-    name = "darwin_powerpc"
-    link_flags = ()
-    cflags = ('-O3', '-fomit-frame-pointer')
 
 class Darwin_x86_64(Darwin):
     name = "darwin_x86_64"
     DEFAULT_CC = 'clang -arch x86_64'
-    link_flags = ('-mmacosx-version-min=10.5',)
-    cflags = ('-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.5')

From noreply at buildbot.pypy.org  Mon Jul 20 19:32:18 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 20 Jul 2015 19:32:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: Don't attempt to handle complex indices
	on the fast path
Message-ID: <20150720173218.2D3AE1C0962@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78616:e549d8109029
Date: 2015-07-20 18:32 +0100
http://bitbucket.org/pypy/pypy/changeset/e549d8109029/

Log:	Don't attempt to handle complex indices on the fast path

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -187,17 +187,8 @@
             raise ArrayArgumentException
         if space.isinstance_w(w_idx, space.w_tuple):
             view_w = space.fixedview(w_idx)
-            if len(view_w) < shape_len:
+            if len(view_w) != shape_len:
                 raise IndexError
-            if len(view_w) > shape_len:
-                # we can allow for one extra None
-                count = len(view_w)
-                for w_item in view_w:
-                    if space.is_w(w_item, space.w_None):
-                        count -= 1
-                if count == shape_len:
-                    raise IndexError # but it's still not a single item
-                raise oefmt(space.w_IndexError, "invalid index")
             # check for arrays
             for w_item in view_w:
                 if (isinstance(w_item, W_NDimArray) or
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2479,6 +2479,7 @@
             assert a[...].base is a
         a[...] = 4
         assert (a == [4, 4, 4]).all()
+        assert a[..., 0] == 4
 
         b = np.arange(24).reshape(2,3,4)
         b[...] = 100

From noreply at buildbot.pypy.org  Mon Jul 20 20:30:51 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 20 Jul 2015 20:30:51 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Add struct module to bz2 test classes'
	spaceconfig.
Message-ID: <20150720183051.06AAB1C0262@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78617:53daf5e5b539
Date: 2015-07-20 19:55 +0200
http://bitbucket.org/pypy/pypy/changeset/53daf5e5b539/

Log:	Add struct module to bz2 test classes' spaceconfig.

diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py
--- a/pypy/module/bz2/test/test_bz2_compdecomp.py
+++ b/pypy/module/bz2/test/test_bz2_compdecomp.py
@@ -41,7 +41,7 @@
     interp_bz2.SMALLCHUNK = mod.OLD_SMALLCHUNK
 
 class AppTestBZ2Compressor(CheckAllocation):
-    spaceconfig = dict(usemodules=('bz2', 'time'))
+    spaceconfig = dict(usemodules=('bz2', 'time', 'struct'))
 
     def setup_class(cls):
         cls.w_TEXT = cls.space.wrapbytes(TEXT)
@@ -117,7 +117,7 @@
 
 
 class AppTestBZ2Decompressor(CheckAllocation):
-    spaceconfig = dict(usemodules=('bz2', 'time'))
+    spaceconfig = dict(usemodules=('bz2', 'time', 'struct'))
 
     def setup_class(cls):
         cls.w_TEXT = cls.space.wrapbytes(TEXT)

From noreply at buildbot.pypy.org  Mon Jul 20 21:11:31 2015
From: noreply at buildbot.pypy.org (wlav)
Date: Mon, 20 Jul 2015 21:11:31 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: merge default into branch and
 fix error message for dlopen
Message-ID: <20150720191131.620C81C089E@cobra.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r78618:ef58dc5560e2
Date: 2014-07-16 12:42 -0700
http://bitbucket.org/pypy/pypy/changeset/ef58dc5560e2/

Log:	merge default into branch and fix error message for dlopen

diff too long, truncating to 2000 out of 40954 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -6,3 +6,11 @@
 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
+20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
+20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
+0000000000000000000000000000000000000000 release-2.3.0
+394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
+32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
+32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
+32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
+0000000000000000000000000000000000000000 release-2.2=3.1
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,31 +44,33 @@
     Alex Gaynor
     Michael Hudson
     David Schneider
+    Matti Picus
+    Brian Kearns
+    Philip Jenvey
     Holger Krekel
     Christian Tismer
     Hakan Ardo
     Benjamin Peterson
-    Matti Picus
-    Philip Jenvey
+    Manuel Jacob
     Anders Chrigstrom
-    Brian Kearns
     Eric van Riet Paap
+    Wim Lavrijsen
+    Ronan Lamy
     Richard Emslie
     Alexander Schremmer
-    Wim Lavrijsen
     Dan Villiom Podlaski Christiansen
-    Manuel Jacob
     Lukas Diekmann
     Sven Hager
     Anders Lehmann
     Aurelien Campeas
     Niklaus Haldimann
-    Ronan Lamy
     Camillo Bruni
     Laura Creighton
     Toon Verwaest
+    Remi Meier
     Leonardo Santagada
     Seo Sanghyeon
+    Romain Guillebert
     Justin Peel
     Ronny Pfannschmidt
     David Edelsohn
@@ -80,52 +82,62 @@
     Daniel Roberts
     Niko Matsakis
     Adrien Di Mascio
+    Alexander Hesse
     Ludovic Aubry
-    Alexander Hesse
     Jacob Hallen
-    Romain Guillebert
     Jason Creighton
     Alex Martelli
     Michal Bendowski
     Jan de Mooij
+    stian
     Michael Foord
     Stephan Diehl
     Stefan Schwarzer
     Valentino Volonghi
     Tomek Meka
     Patrick Maupin
-    stian
     Bob Ippolito
     Bruno Gola
     Jean-Paul Calderone
     Timo Paulssen
+    Squeaky
     Alexandre Fayolle
     Simon Burton
     Marius Gedminas
     John Witulski
+    Konstantin Lopuhin
     Greg Price
     Dario Bertini
     Mark Pearse
     Simon Cross
-    Konstantin Lopuhin
     Andreas St&#252;hrk
     Jean-Philippe St. Pierre
     Guido van Rossum
     Pavel Vinogradov
+    Pawe&#322; Piotr Przeradowski
     Paul deGrandis
     Ilya Osadchiy
+    Tobias Oberstein
     Adrian Kuhn
     Boris Feigin
+    Stefano Rivera
     tav
+    Taavi Burns
     Georg Brandl
     Bert Freudenberg
     Stian Andreassen
-    Stefano Rivera
+    Laurence Tratt
     Wanja Saatkamp
+    Ivan Sichmann Freitas
     Gerald Klix
     Mike Blume
-    Taavi Burns
     Oscar Nierstrasz
+    Stefan H. Muller
+    Jeremy Thurgood
+    Gregor Wegberg
+    Rami Chowdhury
+    Tobias Pape
+    Edd Barrett
     David Malcolm
     Eugene Oden
     Henry Mason
@@ -135,18 +147,16 @@
     Dusty Phillips
     Lukas Renggli
     Guenter Jantzen
-    Tobias Oberstein
-    Remi Meier
     Ned Batchelder
     Amit Regmi
     Ben Young
     Nicolas Chauvat
     Andrew Durdin
+    Andrew Chambers
     Michael Schneider
     Nicholas Riley
     Jason Chu
     Igor Trindade Oliveira
-    Jeremy Thurgood
     Rocco Moretti
     Gintautas Miliauskas
     Michael Twomey
@@ -159,18 +169,19 @@
     Karl Bartel
     Brian Dorsey
     Victor Stinner
+    Andrews Medina
     Stuart Williams
     Jasper Schulz
+    Christian Hudon
     Toby Watson
     Antoine Pitrou
     Aaron Iles
     Michael Cheng
     Justas Sadzevicius
+    Mikael Sch&#246;nenberg
     Gasper Zejn
     Neil Shepperd
-    Mikael Sch&#246;nenberg
     Elmo M&#228;ntynen
-    Tobias Pape
     Jonathan David Riehl
     Stanislaw Halik
     Anders Qvist
@@ -182,19 +193,18 @@
     Alexander Sedov
     Corbin Simpson
     Christopher Pope
-    Laurence Tratt
-    Guillebert Romain
+    wenzhuman
     Christian Tismer 
+    Marc Abramowitz
     Dan Stromberg
     Stefano Parmesan
-    Christian Hudon
     Alexis Daboville
     Jens-Uwe Mager
     Carl Meyer
     Karl Ramm
     Pieter Zieschang
     Gabriel
-    Pawe&#322; Piotr Przeradowski
+    Lukas Vacek
     Andrew Dalke
     Sylvain Thenault
     Nathan Taylor
@@ -203,8 +213,11 @@
     Alejandro J. Cura
     Jacob Oscarson
     Travis Francis Athougies
+    Ryan Gonzalez
     Kristjan Valur Jonsson
+    Sebastian Pawlu&#347;
     Neil Blakey-Milner
+    anatoly techtonik
     Lutz Paelike
     Lucio Torre
     Lars Wassermann
@@ -218,13 +231,14 @@
     Martin Blais
     Lene Wagner
     Tomo Cocoa
-    Andrews Medina
     roberto at goyle
+    Yury V. Zaytsev
+    Anna Katrina Dominguez
     William Leslie
     Bobby Impollonia
     timo at eistee.fritz.box
     Andrew Thompson
-    Yusei Tahara
+    Ben Darnell
     Roberto De Ioris
     Juan Francisco Cantero Hurtado
     Godefroid Chappelle
@@ -234,28 +248,39 @@
     Michael Hudson-Doyle
     Anders Sigfridsson
     Yasir Suhail
+    rafalgalczynski at gmail.com
     Floris Bruynooghe
+    Laurens Van Houtven
     Akira Li
     Gustavo Niemeyer
     Stephan Busemann
-    Anna Katrina Dominguez
+    Rafa&#322; Ga&#322;czy&#324;ski
+    Yusei Tahara
     Christian Muirhead
     James Lan
     shoma hosaka
-    Daniel Neuh&#228;user
+    Daniel Neuh?user
+    Matthew Miller
     Buck Golemon
     Konrad Delong
     Dinu Gherman
     Chris Lambacher
     coolbutuseless at gmail.com
+    Rodrigo Ara&#250;jo
+    w31rd0
     Jim Baker
-    Rodrigo Ara&#250;jo
+    James Robert
     Armin Ronacher
     Brett Cannon
     yrttyr
+    aliceinwire
+    OlivierBlanvillain
     Zooko Wilcox-O Hearn
     Tomer Chachamu
     Christopher Groskopf
+    Asmo Soinio
+    Stefan Marr
+    jiaaro
     opassembler.py
     Antony Lee
     Jim Hunziker
@@ -263,12 +288,13 @@
     Even Wiik Thomassen
     jbs
     soareschen
+    Kurt Griffiths
+    Mike Bayer
     Flavio Percoco
     Kristoffer Kleine
     yasirs
     Michael Chermside
     Anna Ravencroft
-    Andrew Chambers
     Julien Phalip
     Dan Loewenherz
 
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -56,6 +56,9 @@
         for line in longrepr.splitlines():
             py.builtin.print_(" %s" % line, file=self.logfile)
         for key, text in sections:
+            # py.io.StdCaptureFD may send in unicode
+            if isinstance(text, unicode):
+                text = text.encode('utf-8')
             py.builtin.print_(" ", file=self.logfile)
             py.builtin.print_(" -------------------- %s --------------------"
                               % key.rstrip(), file=self.logfile)
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -389,12 +389,13 @@
             func.__name__ = name_or_ordinal
         return func
 
-class PyDLL(CDLL):
-    """This class represents the Python library itself.  It allows to
-    access Python API functions.  The GIL is not released, and
-    Python exceptions are handled correctly.
-    """
-    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+# Not in PyPy
+#class PyDLL(CDLL):
+#    """This class represents the Python library itself.  It allows to
+#    access Python API functions.  The GIL is not released, and
+#    Python exceptions are handled correctly.
+#    """
+#    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
 
 if _os.name in ("nt", "ce"):
 
@@ -447,15 +448,8 @@
         return self._dlltype(name)
 
 cdll = LibraryLoader(CDLL)
-pydll = LibraryLoader(PyDLL)
-
-if _os.name in ("nt", "ce"):
-    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
-elif _sys.platform == "cygwin":
-    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
-else:
-    pythonapi = PyDLL(None)
-
+# not on PyPy
+#pydll = LibraryLoader(PyDLL)
 
 if _os.name in ("nt", "ce"):
     windll = LibraryLoader(WinDLL)
diff --git a/lib-python/2.7/ctypes/test/test_values.py b/lib-python/2.7/ctypes/test/test_values.py
--- a/lib-python/2.7/ctypes/test/test_values.py
+++ b/lib-python/2.7/ctypes/test/test_values.py
@@ -4,6 +4,7 @@
 
 import unittest
 from ctypes import *
+from ctypes.test import xfail
 
 import _ctypes_test
 
@@ -23,7 +24,8 @@
 
     class Win_ValuesTestCase(unittest.TestCase):
         """This test only works when python itself is a dll/shared library"""
-
+        
+        @xfail
         def test_optimizeflag(self):
             # This test accesses the Py_OptimizeFlag intger, which is
             # exported by the Python dll.
@@ -40,6 +42,7 @@
             else:
                 self.assertEqual(opt, 2)
 
+        @xfail
         def test_frozentable(self):
             # Python exports a PyImport_FrozenModules symbol. This is a
             # pointer to an array of struct _frozen entries.  The end of the
@@ -75,6 +78,7 @@
             from ctypes import _pointer_type_cache
             del _pointer_type_cache[struct_frozen]
 
+        @xfail
         def test_undefined(self):
             self.assertRaises(ValueError, c_int.in_dll, pydll, "Undefined_Symbol")
 
diff --git a/lib-python/2.7/imputil.py b/lib-python/2.7/imputil.py
--- a/lib-python/2.7/imputil.py
+++ b/lib-python/2.7/imputil.py
@@ -422,7 +422,8 @@
     saved back to the filesystem for future imports. The source file's
     modification timestamp must be provided as a Long value.
     """
-    codestring = open(pathname, 'rU').read()
+    with open(pathname, 'rU') as fp:
+        codestring = fp.read()
     if codestring and codestring[-1] != '\n':
         codestring = codestring + '\n'
     code = __builtin__.compile(codestring, pathname, 'exec')
@@ -603,8 +604,8 @@
         self.desc = desc
 
     def import_file(self, filename, finfo, fqname):
-        fp = open(filename, self.desc[1])
-        module = imp.load_module(fqname, fp, filename, self.desc)
+        with open(filename, self.desc[1]) as fp:
+            module = imp.load_module(fqname, fp, filename, self.desc)
         module.__file__ = filename
         return 0, module, { }
 
diff --git a/lib-python/2.7/modulefinder.py b/lib-python/2.7/modulefinder.py
--- a/lib-python/2.7/modulefinder.py
+++ b/lib-python/2.7/modulefinder.py
@@ -109,16 +109,16 @@
 
     def run_script(self, pathname):
         self.msg(2, "run_script", pathname)
-        fp = open(pathname, READ_MODE)
-        stuff = ("", "r", imp.PY_SOURCE)
-        self.load_module('__main__', fp, pathname, stuff)
+        with open(pathname, READ_MODE) as fp:
+            stuff = ("", "r", imp.PY_SOURCE)
+            self.load_module('__main__', fp, pathname, stuff)
 
     def load_file(self, pathname):
         dir, name = os.path.split(pathname)
         name, ext = os.path.splitext(name)
-        fp = open(pathname, READ_MODE)
-        stuff = (ext, "r", imp.PY_SOURCE)
-        self.load_module(name, fp, pathname, stuff)
+        with open(pathname, READ_MODE) as fp:
+            stuff = (ext, "r", imp.PY_SOURCE)
+            self.load_module(name, fp, pathname, stuff)
 
     def import_hook(self, name, caller=None, fromlist=None, level=-1):
         self.msg(3, "import_hook", name, caller, fromlist, level)
@@ -461,6 +461,8 @@
         fp, buf, stuff = self.find_module("__init__", m.__path__)
         self.load_module(fqname, fp, buf, stuff)
         self.msgout(2, "load_package ->", m)
+        if fp:
+            fp.close()
         return m
 
     def add_module(self, fqname):
diff --git a/lib-python/2.7/test/test_argparse.py b/lib-python/2.7/test/test_argparse.py
--- a/lib-python/2.7/test/test_argparse.py
+++ b/lib-python/2.7/test/test_argparse.py
@@ -48,6 +48,9 @@
 
     def tearDown(self):
         os.chdir(self.old_dir)
+        import gc
+        # Force a collection which should close FileType() options
+        gc.collect()
         for root, dirs, files in os.walk(self.temp_dir, topdown=False):
             for name in files:
                 os.chmod(os.path.join(self.temp_dir, name), stat.S_IWRITE)
diff --git a/lib-python/2.7/test/test_gdbm.py b/lib-python/2.7/test/test_gdbm.py
--- a/lib-python/2.7/test/test_gdbm.py
+++ b/lib-python/2.7/test/test_gdbm.py
@@ -74,6 +74,40 @@
         size2 = os.path.getsize(filename)
         self.assertTrue(size1 > size2 >= size0)
 
+    def test_sync(self):
+        # check if sync works at all, not sure how to check it
+        self.g = gdbm.open(filename, 'cf')
+        self.g['x'] = 'x' * 10000
+        self.g.sync()
+
+    def test_get_key(self):
+        self.g = gdbm.open(filename, 'cf')
+        self.g['x'] = 'x' * 10000
+        self.g.close()
+        self.g = gdbm.open(filename, 'r')
+        self.assertEquals(self.g['x'], 'x' * 10000)
+
+    def test_key_with_null_bytes(self):
+        key = 'a\x00b'
+        value = 'c\x00d'
+        self.g = gdbm.open(filename, 'cf')
+        self.g[key] = value
+        self.g.close()
+        self.g = gdbm.open(filename, 'r')
+        self.assertEquals(self.g[key], value)
+        self.assertTrue(key in self.g)
+        self.assertTrue(self.g.has_key(key))
+
+    def test_unicode_key(self):
+        key = u'ab'
+        value = u'cd'
+        self.g = gdbm.open(filename, 'cf')
+        self.g[key] = value
+        self.g.close()
+        self.g = gdbm.open(filename, 'r')
+        self.assertEquals(self.g[key], value)
+        self.assertTrue(key in self.g)
+        self.assertTrue(self.g.has_key(key))
 
 def test_main():
     run_unittest(TestGdbm)
diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py
--- a/lib-python/2.7/timeit.py
+++ b/lib-python/2.7/timeit.py
@@ -55,11 +55,6 @@
 import gc
 import sys
 import time
-try:
-    import itertools
-except ImportError:
-    # Must be an older Python version (see timeit() below)
-    itertools = None
 
 __all__ = ["Timer"]
 
@@ -81,7 +76,8 @@
 def inner(_it, _timer):
     %(setup)s
     _t0 = _timer()
-    for _i in _it:
+    while _it > 0:
+        _it -= 1
         %(stmt)s
     _t1 = _timer()
     return _t1 - _t0
@@ -96,7 +92,8 @@
     def inner(_it, _timer, _func=func):
         setup()
         _t0 = _timer()
-        for _i in _it:
+        while _it > 0:
+            _it -= 1
             _func()
         _t1 = _timer()
         return _t1 - _t0
@@ -133,9 +130,19 @@
             else:
                 raise ValueError("setup is neither a string nor callable")
             self.src = src # Save for traceback display
-            code = compile(src, dummy_src_name, "exec")
-            exec code in globals(), ns
-            self.inner = ns["inner"]
+            def make_inner():
+                # PyPy tweak: recompile the source code each time before
+                # calling inner(). There are situations like Issue #1776
+                # where PyPy tries to reuse the JIT code from before,
+                # but that's not going to work: the first thing the
+                # function does is the "-s" statement, which may declare
+                # new classes (here a namedtuple). We end up with
+                # bridges from the inner loop; more and more of them
+                # every time we call inner().
+                code = compile(src, dummy_src_name, "exec")
+                exec code in globals(), ns
+                return ns["inner"]
+            self.make_inner = make_inner
         elif hasattr(stmt, '__call__'):
             self.src = None
             if isinstance(setup, basestring):
@@ -144,7 +151,8 @@
                     exec _setup in globals(), ns
             elif not hasattr(setup, '__call__'):
                 raise ValueError("setup is neither a string nor callable")
-            self.inner = _template_func(setup, stmt)
+            inner = _template_func(setup, stmt)
+            self.make_inner = lambda: inner
         else:
             raise ValueError("stmt is neither a string nor callable")
 
@@ -185,15 +193,12 @@
         to one million.  The main statement, the setup statement and
         the timer function to be used are passed to the constructor.
         """
-        if itertools:
-            it = itertools.repeat(None, number)
-        else:
-            it = [None] * number
+        inner = self.make_inner()
         gcold = gc.isenabled()
         if '__pypy__' not in sys.builtin_module_names:
             gc.disable()    # only do that on CPython
         try:
-            timing = self.inner(it, self.timer)
+            timing = inner(number, self.timer)
         finally:
             if gcold:
                 gc.enable()
diff --git a/lib-python/2.7/xml/sax/saxutils.py b/lib-python/2.7/xml/sax/saxutils.py
--- a/lib-python/2.7/xml/sax/saxutils.py
+++ b/lib-python/2.7/xml/sax/saxutils.py
@@ -98,13 +98,14 @@
         except AttributeError:
             pass
     # wrap a binary writer with TextIOWrapper
-    class UnbufferedTextIOWrapper(io.TextIOWrapper):
-        def write(self, s):
-            super(UnbufferedTextIOWrapper, self).write(s)
-            self.flush()
-    return UnbufferedTextIOWrapper(buffer, encoding=encoding,
+    return _UnbufferedTextIOWrapper(buffer, encoding=encoding,
                                    errors='xmlcharrefreplace',
                                    newline='\n')
+# PyPy: moved this class outside the function above
+class _UnbufferedTextIOWrapper(io.TextIOWrapper):
+    def write(self, s):
+        super(_UnbufferedTextIOWrapper, self).write(s)
+        self.flush()
 
 class XMLGenerator(handler.ContentHandler):
 
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -219,6 +219,8 @@
             if restype is None:
                 import ctypes
                 restype = ctypes.c_int
+            if self._argtypes_ is None:
+                self._argtypes_ = []
             self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
             self._check_argtypes_for_fastpath()
             return
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -3,6 +3,8 @@
 import sys
 import os
 
+irc_header = "And now for something completely different"
+
 
 def interactive_console(mainmodule=None, quiet=False):
     # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
@@ -15,8 +17,7 @@
     if not quiet:
         try:
             from _pypy_irc_topic import some_topic
-            text = "And now for something completely different: ``%s''" % (
-                some_topic(),)
+            text = "%s: ``%s''" % ( irc_header, some_topic())
             while len(text) >= 80:
                 i = text[:80].rfind(' ')
                 print(text[:i])
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -13,7 +13,15 @@
     k1 = k1.lstrip('0x').rstrip('L')
     k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff)
     k2 = k2.lstrip('0').rstrip('L')
-    output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s%s' %(k1, k2)
+    try:
+        username = os.environ['USER']           #linux, et al
+    except KeyError:
+        try:
+            username = os.environ['USERNAME']   #windows
+        except KeyError:
+            username = os.getuid()
+    output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s_%s%s' % (
+        username, k1, k2)
     if not os.path.exists(output_dir):
         os.mkdir(output_dir)
     return output_dir
diff --git a/lib_pypy/_tkinter/license.terms b/lib_pypy/_tkinter/license.terms
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/license.terms
@@ -0,0 +1,39 @@
+This software is copyrighted by the Regents of the University of
+California, Sun Microsystems, Inc., and other parties.  The following
+terms apply to all files associated with the software unless explicitly
+disclaimed in individual files.
+
+The authors hereby grant permission to use, copy, modify, distribute,
+and license this software and its documentation for any purpose, provided
+that existing copyright notices are retained in all copies and that this
+notice is included verbatim in any distributions. No written agreement,
+license, or royalty fee is required for any of the authorized uses.
+Modifications to this software may be copyrighted by their authors
+and need not follow the licensing terms described here, provided that
+the new terms are clearly indicated on the first page of each file where
+they apply.
+
+IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
+FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
+DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT.  THIS SOFTWARE
+IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE
+NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR
+MODIFICATIONS.
+
+GOVERNMENT USE: If you are acquiring this software on behalf of the
+U.S. government, the Government shall have only "Restricted Rights"
+in the software and related documentation as defined in the Federal 
+Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2).  If you
+are acquiring the software on behalf of the Department of Defense, the
+software shall be classified as "Commercial Computer Software" and the
+Government shall have only "Restricted Rights" as defined in Clause
+252.227-7013 (c) (1) of DFARs.  Notwithstanding the foregoing, the
+authors grant the U.S. Government and others acting in its behalf
+permission to use and distribute the software in accordance with the
+terms specified in this license.
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -121,6 +121,10 @@
     incdirs = []
     linklibs = ['tcl85', 'tk85']
     libdirs = []
+elif sys.platform == 'darwin':
+    incdirs = ['/System/Library/Frameworks/Tk.framework/Versions/Current/Headers/']
+    linklibs = ['tcl', 'tk']
+    libdirs = []
 else:
     incdirs=['/usr/include/tcl']
     linklibs=['tcl', 'tk']
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.8.2"
-__version_info__ = (0, 8, 2)
+__version__ = "0.8.6"
+__version_info__ = (0, 8, 6)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -55,8 +55,7 @@
             # _cffi_backend.so compiled.
             import _cffi_backend as backend
             from . import __version__
-            assert (backend.__version__ == __version__ or
-                    backend.__version__ == __version__[:3])
+            assert backend.__version__ == __version__
             # (If you insist you can also try to pass the option
             # 'backend=backend_ctypes.CTypesBackend()', but don't
             # rely on it!  It's probably not going to work well.)
@@ -443,6 +442,10 @@
                 for enumname, enumval in zip(tp.enumerators, tp.enumvalues):
                     if enumname not in library.__dict__:
                         library.__dict__[enumname] = enumval
+            for key, val in ffi._parser._int_constants.items():
+                if key not in library.__dict__:
+                    library.__dict__[key] = val
+
             copied_enums.append(True)
             if name in library.__dict__:
                 return
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -24,6 +24,7 @@
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
 _r_words = re.compile(r"\w+|\S")
 _parser_cache = None
+_r_int_literal = re.compile(r"^0?x?[0-9a-f]+u?l?$", re.IGNORECASE)
 
 def _get_parser():
     global _parser_cache
@@ -99,6 +100,7 @@
         self._structnode2type = weakref.WeakKeyDictionary()
         self._override = False
         self._packed = False
+        self._int_constants = {}
 
     def _parse(self, csource):
         csource, macros = _preprocess(csource)
@@ -128,9 +130,10 @@
         finally:
             if lock is not None:
                 lock.release()
-        return ast, macros
+        # csource will be used to find buggy source text
+        return ast, macros, csource
 
-    def convert_pycparser_error(self, e, csource):
+    def _convert_pycparser_error(self, e, csource):
         # xxx look for ":NUM:" at the start of str(e) and try to interpret
         # it as a line number
         line = None
@@ -142,6 +145,12 @@
                 csourcelines = csource.splitlines()
                 if 1 <= linenum <= len(csourcelines):
                     line = csourcelines[linenum-1]
+        return line
+
+    def convert_pycparser_error(self, e, csource):
+        line = self._convert_pycparser_error(e, csource)
+
+        msg = str(e)
         if line:
             msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
@@ -160,14 +169,9 @@
             self._packed = prev_packed
 
     def _internal_parse(self, csource):
-        ast, macros = self._parse(csource)
+        ast, macros, csource = self._parse(csource)
         # add the macros
-        for key, value in macros.items():
-            value = value.strip()
-            if value != '...':
-                raise api.CDefError('only supports the syntax "#define '
-                                    '%s ..." for now (literally)' % key)
-            self._declare('macro ' + key, value)
+        self._process_macros(macros)
         # find the first "__dotdotdot__" and use that as a separator
         # between the repeated typedefs and the real csource
         iterator = iter(ast.ext)
@@ -175,27 +179,61 @@
             if decl.name == '__dotdotdot__':
                 break
         #
-        for decl in iterator:
-            if isinstance(decl, pycparser.c_ast.Decl):
-                self._parse_decl(decl)
-            elif isinstance(decl, pycparser.c_ast.Typedef):
-                if not decl.name:
-                    raise api.CDefError("typedef does not declare any name",
-                                        decl)
-                if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
-                        and decl.type.type.names == ['__dotdotdot__']):
-                    realtype = model.unknown_type(decl.name)
-                elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and
-                      isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and
-                      isinstance(decl.type.type.type,
-                                 pycparser.c_ast.IdentifierType) and
-                      decl.type.type.type.names == ['__dotdotdot__']):
-                    realtype = model.unknown_ptr_type(decl.name)
+        try:
+            for decl in iterator:
+                if isinstance(decl, pycparser.c_ast.Decl):
+                    self._parse_decl(decl)
+                elif isinstance(decl, pycparser.c_ast.Typedef):
+                    if not decl.name:
+                        raise api.CDefError("typedef does not declare any name",
+                                            decl)
+                    if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
+                            and decl.type.type.names == ['__dotdotdot__']):
+                        realtype = model.unknown_type(decl.name)
+                    elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and
+                          isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and
+                          isinstance(decl.type.type.type,
+                                     pycparser.c_ast.IdentifierType) and
+                          decl.type.type.type.names == ['__dotdotdot__']):
+                        realtype = model.unknown_ptr_type(decl.name)
+                    else:
+                        realtype = self._get_type(decl.type, name=decl.name)
+                    self._declare('typedef ' + decl.name, realtype)
                 else:
-                    realtype = self._get_type(decl.type, name=decl.name)
-                self._declare('typedef ' + decl.name, realtype)
+                    raise api.CDefError("unrecognized construct", decl)
+        except api.FFIError as e:
+            msg = self._convert_pycparser_error(e, csource)
+            if msg:
+                e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
+            raise
+
+    def _add_constants(self, key, val):
+        if key in self._int_constants:
+            raise api.FFIError(
+                "multiple declarations of constant: %s" % (key,))
+        self._int_constants[key] = val
+
+    def _process_macros(self, macros):
+        for key, value in macros.items():
+            value = value.strip()
+            match = _r_int_literal.search(value)
+            if match is not None:
+                int_str = match.group(0).lower().rstrip("ul")
+
+                # "010" is not valid oct in py3
+                if (int_str.startswith("0") and
+                        int_str != "0" and
+                        not int_str.startswith("0x")):
+                    int_str = "0o" + int_str[1:]
+
+                pyvalue = int(int_str, 0)
+                self._add_constants(key, pyvalue)
+            elif value == '...':
+                self._declare('macro ' + key, value)
             else:
-                raise api.CDefError("unrecognized construct", decl)
+                raise api.CDefError('only supports the syntax "#define '
+                                    '%s ..." (literally) or "#define '
+                                    '%s 0x1FF" for now' % (key, key))
 
     def _parse_decl(self, decl):
         node = decl.type
@@ -227,7 +265,7 @@
                     self._declare('variable ' + decl.name, tp)
 
     def parse_type(self, cdecl):
-        ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)
+        ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2]
         assert not macros
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
@@ -306,7 +344,8 @@
                 if ident == 'void':
                     return model.void_type
                 if ident == '__dotdotdot__':
-                    raise api.FFIError('bad usage of "..."')
+                    raise api.FFIError(':%d: bad usage of "..."' %
+                            typenode.coord.line)
                 return resolve_common_type(ident)
             #
             if isinstance(type, pycparser.c_ast.Struct):
@@ -333,7 +372,8 @@
             return self._get_struct_union_enum_type('union', typenode, name,
                                                     nested=True)
         #
-        raise api.FFIError("bad or unsupported type declaration")
+        raise api.FFIError(":%d: bad or unsupported type declaration" %
+                typenode.coord.line)
 
     def _parse_function_type(self, typenode, funcname=None):
         params = list(getattr(typenode.args, 'params', []))
@@ -499,6 +539,10 @@
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '-'):
             return -self._parse_constant(exprnode.expr)
+        # load previously defined int constant
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
+                exprnode.name in self._int_constants):
+            return self._int_constants[exprnode.name]
         #
         if partial_length_ok:
             if (isinstance(exprnode, pycparser.c_ast.ID) and
@@ -506,8 +550,8 @@
                 self._partial_length = True
                 return '...'
         #
-        raise api.FFIError("unsupported expression: expected a "
-                           "simple numeric constant")
+        raise api.FFIError(":%d: unsupported expression: expected a "
+                           "simple numeric constant" % exprnode.coord.line)
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
@@ -522,6 +566,7 @@
                 if enum.value is not None:
                     nextenumvalue = self._parse_constant(enum.value)
                 enumvalues.append(nextenumvalue)
+                self._add_constants(enum.name, nextenumvalue)
                 nextenumvalue += 1
             enumvalues = tuple(enumvalues)
             tp = model.EnumType(explicit_name, enumerators, enumvalues)
@@ -535,3 +580,5 @@
             kind = name.split(' ', 1)[0]
             if kind in ('typedef', 'struct', 'union', 'enum'):
                 self._declare(name, tp)
+        for k, v in other._int_constants.items():
+            self._add_constants(k, v)
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -38,6 +38,7 @@
     import distutils.errors
     #
     dist = Distribution({'ext_modules': [ext]})
+    dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -89,43 +89,54 @@
         # by generate_cpy_function_method().
         prnt('static PyMethodDef _cffi_methods[] = {')
         self._generate("method")
-        prnt('  {"_cffi_setup", _cffi_setup, METH_VARARGS},')
-        prnt('  {NULL, NULL}    /* Sentinel */')
+        prnt('  {"_cffi_setup", _cffi_setup, METH_VARARGS, NULL},')
+        prnt('  {NULL, NULL, 0, NULL}    /* Sentinel */')
         prnt('};')
         prnt()
         #
         # standard init.
         modname = self.verifier.get_module_name()
-        if sys.version_info >= (3,):
-            prnt('static struct PyModuleDef _cffi_module_def = {')
-            prnt('  PyModuleDef_HEAD_INIT,')
-            prnt('  "%s",' % modname)
-            prnt('  NULL,')
-            prnt('  -1,')
-            prnt('  _cffi_methods,')
-            prnt('  NULL, NULL, NULL, NULL')
-            prnt('};')
-            prnt()
-            initname = 'PyInit_%s' % modname
-            createmod = 'PyModule_Create(&_cffi_module_def)'
-            errorcase = 'return NULL'
-            finalreturn = 'return lib'
-        else:
-            initname = 'init%s' % modname
-            createmod = 'Py_InitModule("%s", _cffi_methods)' % modname
-            errorcase = 'return'
-            finalreturn = 'return'
+        constants = self._chained_list_constants[False]
+        prnt('#if PY_MAJOR_VERSION >= 3')
+        prnt()
+        prnt('static struct PyModuleDef _cffi_module_def = {')
+        prnt('  PyModuleDef_HEAD_INIT,')
+        prnt('  "%s",' % modname)
+        prnt('  NULL,')
+        prnt('  -1,')
+        prnt('  _cffi_methods,')
+        prnt('  NULL, NULL, NULL, NULL')
+        prnt('};')
+        prnt()
         prnt('PyMODINIT_FUNC')
-        prnt('%s(void)' % initname)
+        prnt('PyInit_%s(void)' % modname)
         prnt('{')
         prnt('  PyObject *lib;')
-        prnt('  lib = %s;' % createmod)
-        prnt('  if (lib == NULL || %s < 0)' % (
-            self._chained_list_constants[False],))
-        prnt('    %s;' % errorcase)
-        prnt('  _cffi_init();')
-        prnt('  %s;' % finalreturn)
+        prnt('  lib = PyModule_Create(&_cffi_module_def);')
+        prnt('  if (lib == NULL)')
+        prnt('    return NULL;')
+        prnt('  if (%s < 0 || _cffi_init() < 0) {' % (constants,))
+        prnt('    Py_DECREF(lib);')
+        prnt('    return NULL;')
+        prnt('  }')
+        prnt('  return lib;')
         prnt('}')
+        prnt()
+        prnt('#else')
+        prnt()
+        prnt('PyMODINIT_FUNC')
+        prnt('init%s(void)' % modname)
+        prnt('{')
+        prnt('  PyObject *lib;')
+        prnt('  lib = Py_InitModule("%s", _cffi_methods);' % modname)
+        prnt('  if (lib == NULL)')
+        prnt('    return;')
+        prnt('  if (%s < 0 || _cffi_init() < 0)' % (constants,))
+        prnt('    return;')
+        prnt('  return;')
+        prnt('}')
+        prnt()
+        prnt('#endif')
 
     def load_library(self):
         # XXX review all usages of 'self' here!
@@ -394,7 +405,7 @@
             meth = 'METH_O'
         else:
             meth = 'METH_VARARGS'
-        self._prnt('  {"%s", _cffi_f_%s, %s},' % (name, name, meth))
+        self._prnt('  {"%s", _cffi_f_%s, %s, NULL},' % (name, name, meth))
 
     _loading_cpy_function = _loaded_noop
 
@@ -481,8 +492,8 @@
         if tp.fldnames is None:
             return     # nothing to do with opaque structs
         layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
-        self._prnt('  {"%s", %s, METH_NOARGS},' % (layoutfuncname,
-                                                   layoutfuncname))
+        self._prnt('  {"%s", %s, METH_NOARGS, NULL},' % (layoutfuncname,
+                                                         layoutfuncname))
 
     def _loading_struct_or_union(self, tp, prefix, name, module):
         if tp.fldnames is None:
@@ -589,13 +600,7 @@
                                                           'variable type'),))
             assert delayed
         else:
-            prnt('  if (LONG_MIN <= (%s) && (%s) <= LONG_MAX)' % (name, name))
-            prnt('    o = PyInt_FromLong((long)(%s));' % (name,))
-            prnt('  else if ((%s) <= 0)' % (name,))
-            prnt('    o = PyLong_FromLongLong((long long)(%s));' % (name,))
-            prnt('  else')
-            prnt('    o = PyLong_FromUnsignedLongLong('
-                 '(unsigned long long)(%s));' % (name,))
+            prnt('  o = _cffi_from_c_int_const(%s);' % name)
         prnt('  if (o == NULL)')
         prnt('    return -1;')
         if size_too:
@@ -632,13 +637,18 @@
     # ----------
     # enums
 
+    def _enum_funcname(self, prefix, name):
+        # "$enum_$1" => "___D_enum____D_1"
+        name = name.replace('$', '___D_')
+        return '_cffi_e_%s_%s' % (prefix, name)
+
     def _generate_cpy_enum_decl(self, tp, name, prefix='enum'):
         if tp.partial:
             for enumerator in tp.enumerators:
                 self._generate_cpy_const(True, enumerator, delayed=False)
             return
         #
-        funcname = '_cffi_e_%s_%s' % (prefix, name)
+        funcname = self._enum_funcname(prefix, name)
         prnt = self._prnt
         prnt('static int %s(PyObject *lib)' % funcname)
         prnt('{')
@@ -760,17 +770,30 @@
 #include <Python.h>
 #include <stddef.h>
 
-#ifdef MS_WIN32
-#include <malloc.h>   /* for alloca() */
-typedef __int8 int8_t;
-typedef __int16 int16_t;
-typedef __int32 int32_t;
-typedef __int64 int64_t;
-typedef unsigned __int8 uint8_t;
-typedef unsigned __int16 uint16_t;
-typedef unsigned __int32 uint32_t;
-typedef unsigned __int64 uint64_t;
-typedef unsigned char _Bool;
+/* this block of #ifs should be kept exactly identical between
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+#if defined(_MSC_VER)
+# include <malloc.h>   /* for alloca() */
+# if _MSC_VER < 1600   /* MSVC < 2010 */
+   typedef __int8 int8_t;
+   typedef __int16 int16_t;
+   typedef __int32 int32_t;
+   typedef __int64 int64_t;
+   typedef unsigned __int8 uint8_t;
+   typedef unsigned __int16 uint16_t;
+   typedef unsigned __int32 uint32_t;
+   typedef unsigned __int64 uint64_t;
+# else
+#  include <stdint.h>
+# endif
+# if _MSC_VER < 1800   /* MSVC < 2013 */
+   typedef unsigned char _Bool;
+# endif
+#else
+# include <stdint.h>
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+#  include <alloca.h>
+# endif
 #endif
 
 #if PY_MAJOR_VERSION < 3
@@ -795,6 +818,15 @@
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
 
+#define _cffi_from_c_int_const(x)                                        \
+    (((x) > 0) ?                                                         \
+        ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ?      \
+            PyInt_FromLong((long)(x)) :                                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)(x)) :       \
+        ((long long)(x) >= (long long)LONG_MIN) ?                        \
+            PyInt_FromLong((long)(x)) :                                  \
+            PyLong_FromLongLong((long long)(x)))
+
 #define _cffi_from_c_int(x, type)                                        \
     (((type)-1) > 0 ?   /* unsigned */                                   \
         (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) :               \
@@ -804,14 +836,14 @@
                                         PyLong_FromLongLong(x)))
 
 #define _cffi_to_c_int(o, type)                                          \
-    (sizeof(type) == 1 ? (((type)-1) > 0 ? _cffi_to_c_u8(o)              \
-                                         : _cffi_to_c_i8(o)) :           \
-     sizeof(type) == 2 ? (((type)-1) > 0 ? _cffi_to_c_u16(o)             \
-                                         : _cffi_to_c_i16(o)) :          \
-     sizeof(type) == 4 ? (((type)-1) > 0 ? _cffi_to_c_u32(o)             \
-                                         : _cffi_to_c_i32(o)) :          \
-     sizeof(type) == 8 ? (((type)-1) > 0 ? _cffi_to_c_u64(o)             \
-                                         : _cffi_to_c_i64(o)) :          \
+    (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \
+                                         : (type)_cffi_to_c_i8(o)) :     \
+     sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o)       \
+                                         : (type)_cffi_to_c_i16(o)) :    \
+     sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o)       \
+                                         : (type)_cffi_to_c_i32(o)) :    \
+     sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o)       \
+                                         : (type)_cffi_to_c_i64(o)) :    \
      (Py_FatalError("unsupported size for type " #type), 0))
 
 #define _cffi_to_c_i8                                                    \
@@ -885,25 +917,32 @@
     return PyBool_FromLong(was_alive);
 }
 
-static void _cffi_init(void)
+static int _cffi_init(void)
 {
-    PyObject *module = PyImport_ImportModule("_cffi_backend");
-    PyObject *c_api_object;
+    PyObject *module, *c_api_object = NULL;
 
+    module = PyImport_ImportModule("_cffi_backend");
     if (module == NULL)
-        return;
+        goto failure;
 
     c_api_object = PyObject_GetAttrString(module, "_C_API");
     if (c_api_object == NULL)
-        return;
+        goto failure;
     if (!PyCapsule_CheckExact(c_api_object)) {
-        Py_DECREF(c_api_object);
         PyErr_SetNone(PyExc_ImportError);
-        return;
+        goto failure;
     }
     memcpy(_cffi_exports, PyCapsule_GetPointer(c_api_object, "cffi"),
            _CFFI_NUM_EXPORTS * sizeof(void *));
+
+    Py_DECREF(module);
     Py_DECREF(c_api_object);
+    return 0;
+
+  failure:
+    Py_XDECREF(module);
+    Py_XDECREF(c_api_object);
+    return -1;
 }
 
 #define _cffi_type(num) ((CTypeDescrObject *)PyList_GET_ITEM(_cffi_types, num))
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -249,10 +249,10 @@
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         self.export_symbols.append(layoutfuncname)
-        prnt('ssize_t %s(ssize_t i)' % (layoutfuncname,))
+        prnt('intptr_t %s(intptr_t i)' % (layoutfuncname,))
         prnt('{')
         prnt('  struct _cffi_aligncheck { char x; %s y; };' % cname)
-        prnt('  static ssize_t nums[] = {')
+        prnt('  static intptr_t nums[] = {')
         prnt('    sizeof(%s),' % cname)
         prnt('    offsetof(struct _cffi_aligncheck, y),')
         for fname, ftype, fbitsize in tp.enumfields():
@@ -276,7 +276,7 @@
             return     # nothing to do with opaque structs
         layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
         #
-        BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0]
+        BFunc = self.ffi._typeof_locked("intptr_t(*)(intptr_t)")[0]
         function = module.load_function(BFunc, layoutfuncname)
         layout = []
         num = 0
@@ -410,13 +410,18 @@
     # ----------
     # enums
 
+    def _enum_funcname(self, prefix, name):
+        # "$enum_$1" => "___D_enum____D_1"
+        name = name.replace('$', '___D_')
+        return '_cffi_e_%s_%s' % (prefix, name)
+
     def _generate_gen_enum_decl(self, tp, name, prefix='enum'):
         if tp.partial:
             for enumerator in tp.enumerators:
                 self._generate_gen_const(True, enumerator)
             return
         #
-        funcname = '_cffi_e_%s_%s' % (prefix, name)
+        funcname = self._enum_funcname(prefix, name)
         self.export_symbols.append(funcname)
         prnt = self._prnt
         prnt('int %s(char *out_error)' % funcname)
@@ -430,14 +435,14 @@
                     enumerator, enumerator, enumvalue))
             prnt('    char buf[64];')
             prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+            prnt('        sprintf(buf, "%%ld", (long)(%s));' % enumerator)
             prnt('    else')
-            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+            prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
                  enumerator)
-            prnt('    snprintf(out_error, 255,'
+            prnt('    sprintf(out_error,'
                              ' "%s has the real value %s, not %s",')
             prnt('            "%s", buf, "%d");' % (
-                enumerator, enumvalue))
+                enumerator[:100], enumvalue))
             prnt('    return -1;')
             prnt('  }')
         prnt('  return 0;')
@@ -453,7 +458,7 @@
         else:
             BType = self.ffi._typeof_locked("char[]")[0]
             BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
-            funcname = '_cffi_e_%s_%s' % (prefix, name)
+            funcname = self._enum_funcname(prefix, name)
             function = module.load_function(BFunc, funcname)
             p = self.ffi.new(BType, 256)
             if function(p) < 0:
@@ -547,20 +552,29 @@
 #include <errno.h>
 #include <sys/types.h>   /* XXX for ssize_t on some platforms */
 
-#ifdef _WIN32
-#  include <Windows.h>
-#  define snprintf _snprintf
-typedef __int8 int8_t;
-typedef __int16 int16_t;
-typedef __int32 int32_t;
-typedef __int64 int64_t;
-typedef unsigned __int8 uint8_t;
-typedef unsigned __int16 uint16_t;
-typedef unsigned __int32 uint32_t;
-typedef unsigned __int64 uint64_t;
-typedef SSIZE_T ssize_t;
-typedef unsigned char _Bool;
+/* this block of #ifs should be kept exactly identical between
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+#if defined(_MSC_VER)
+# include <malloc.h>   /* for alloca() */
+# if _MSC_VER < 1600   /* MSVC < 2010 */
+   typedef __int8 int8_t;
+   typedef __int16 int16_t;
+   typedef __int32 int32_t;
+   typedef __int64 int64_t;
+   typedef unsigned __int8 uint8_t;
+   typedef unsigned __int16 uint16_t;
+   typedef unsigned __int32 uint32_t;
+   typedef unsigned __int64 uint64_t;
+# else
+#  include <stdint.h>
+# endif
+# if _MSC_VER < 1800   /* MSVC < 2013 */
+   typedef unsigned char _Bool;
+# endif
 #else
-#  include <stdint.h>
+# include <stdint.h>
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+#  include <alloca.h>
+# endif
 #endif
 '''
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/gdbm.py
@@ -0,0 +1,176 @@
+import cffi, os
+
+ffi = cffi.FFI()
+ffi.cdef('''
+#define GDBM_READER ...
+#define GDBM_WRITER ...
+#define GDBM_WRCREAT ...
+#define GDBM_NEWDB ...
+#define GDBM_FAST ...
+#define GDBM_SYNC ...
+#define GDBM_NOLOCK ...
+#define GDBM_REPLACE ...
+
+void* gdbm_open(char *, int, int, int, void (*)());
+void gdbm_close(void*);
+
+typedef struct {
+    char *dptr;
+    int   dsize;
+} datum;
+
+datum gdbm_fetch(void*, datum);
+int gdbm_delete(void*, datum);
+int gdbm_store(void*, datum, datum, int);
+int gdbm_exists(void*, datum);
+
+int gdbm_reorganize(void*);
+
+datum gdbm_firstkey(void*);
+datum gdbm_nextkey(void*, datum);
+void gdbm_sync(void*);
+
+char* gdbm_strerror(int);
+int gdbm_errno;
+
+void free(void*);
+''')
+
+try:
+    lib = ffi.verify('''
+    #include "gdbm.h"
+    ''', libraries=['gdbm'])
+except cffi.VerificationError as e:
+    # distutils does not preserve the actual message,
+    # but the verification is simple enough that the
+    # failure must be due to missing gdbm dev libs
+    raise ImportError('%s: %s' %(e.__class__.__name__, e))
+
+class error(Exception):
+    pass
+
+def _fromstr(key):
+    if isinstance(key, unicode):
+        key = key.encode("ascii")
+    if not isinstance(key, str):
+        raise TypeError("gdbm mappings have string indices only")
+    return {'dptr': ffi.new("char[]", key), 'dsize': len(key)}
+
+class gdbm(object):
+    ll_dbm = None
+
+    def __init__(self, filename, iflags, mode):
+        res = lib.gdbm_open(filename, 0, iflags, mode, ffi.NULL)
+        self.size = -1
+        if not res:
+            self._raise_from_errno()
+        self.ll_dbm = res
+
+    def close(self):
+        if self.ll_dbm:
+            lib.gdbm_close(self.ll_dbm)
+            self.ll_dbm = None
+
+    def _raise_from_errno(self):
+        if ffi.errno:
+            raise error(ffi.errno, os.strerror(ffi.errno))
+        raise error(lib.gdbm_errno, lib.gdbm_strerror(lib.gdbm_errno))
+
+    def __len__(self):
+        if self.size < 0:
+            self.size = len(self.keys())
+        return self.size
+
+    def __setitem__(self, key, value):
+        self._check_closed()
+        self._size = -1
+        r = lib.gdbm_store(self.ll_dbm, _fromstr(key), _fromstr(value),
+                           lib.GDBM_REPLACE)
+        if r < 0:
+            self._raise_from_errno()
+
+    def __delitem__(self, key):
+        self._check_closed()
+        res = lib.gdbm_delete(self.ll_dbm, _fromstr(key))
+        if res < 0:
+            raise KeyError(key)
+
+    def __contains__(self, key):
+        self._check_closed()
+        return lib.gdbm_exists(self.ll_dbm, _fromstr(key))
+    has_key = __contains__
+
+    def __getitem__(self, key):
+        self._check_closed()
+        drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key))
+        if not drec.dptr:
+            raise KeyError(key)
+        res = str(ffi.buffer(drec.dptr, drec.dsize))
+        lib.free(drec.dptr)
+        return res
+
+    def keys(self):
+        self._check_closed()
+        l = []
+        key = lib.gdbm_firstkey(self.ll_dbm)
+        while key.dptr:
+            l.append(str(ffi.buffer(key.dptr, key.dsize)))
+            nextkey = lib.gdbm_nextkey(self.ll_dbm, key)
+            lib.free(key.dptr)
+            key = nextkey
+        return l
+
+    def firstkey(self):
+        self._check_closed()
+        key = lib.gdbm_firstkey(self.ll_dbm)
+        if key.dptr:
+            res = str(ffi.buffer(key.dptr, key.dsize))
+            lib.free(key.dptr)
+            return res
+
+    def nextkey(self, key):
+        self._check_closed()
+        key = lib.gdbm_nextkey(self.ll_dbm, _fromstr(key))
+        if key.dptr:
+            res = str(ffi.buffer(key.dptr, key.dsize))
+            lib.free(key.dptr)
+            return res
+
+    def reorganize(self):
+        self._check_closed()
+        if lib.gdbm_reorganize(self.ll_dbm) < 0:
+            self._raise_from_errno()
+
+    def _check_closed(self):
+        if not self.ll_dbm:
+            raise error(0, "GDBM object has already been closed")
+
+    __del__ = close
+
+    def sync(self):
+        self._check_closed()
+        lib.gdbm_sync(self.ll_dbm)
+
+def open(filename, flags='r', mode=0666):
+    if flags[0] == 'r':
+        iflags = lib.GDBM_READER
+    elif flags[0] == 'w':
+        iflags = lib.GDBM_WRITER
+    elif flags[0] == 'c':
+        iflags = lib.GDBM_WRCREAT
+    elif flags[0] == 'n':
+        iflags = lib.GDBM_NEWDB
+    else:
+        raise error(0, "First flag must be one of 'r', 'w', 'c' or 'n'")
+    for flag in flags[1:]:
+        if flag == 'f':
+            iflags |= lib.GDBM_FAST
+        elif flag == 's':
+            iflags |= lib.GDBM_SYNC
+        elif flag == 'u':
+            iflags |= lib.GDBM_NOLOCK
+        else:
+            raise error(0, "Flag '%s' not supported" % flag)
+    return gdbm(filename, iflags, mode)
+
+open_flags = "rwcnfsu"
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -113,7 +113,7 @@
             try:
                 for name in modlist:
                     __import__(name)
-            except (ImportError, CompilationError, py.test.skip.Exception), e:
+            except (ImportError, CompilationError, py.test.skip.Exception) as e:
                 errcls = e.__class__.__name__
                 raise Exception(
                     "The module %r is disabled\n" % (modname,) +
diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile
--- a/pypy/doc/Makefile
+++ b/pypy/doc/Makefile
@@ -7,63 +7,80 @@
 PAPER         =
 BUILDDIR      = _build
 
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
-.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex man changes linkcheck doctest
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  html      to make standalone HTML files"
-	@echo "  dirhtml   to make HTML files named index.html in directories"
-	@echo "  pickle    to make pickle files"
-	@echo "  json      to make JSON files"
-	@echo "  htmlhelp  to make HTML files and a HTML help project"
-	@echo "  qthelp    to make HTML files and a qthelp project"
-	@echo "  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
-	@echo "  man       to make manual pages"
-	@echo "  changes   to make an overview of all changed/added/deprecated items"
-	@echo "  linkcheck to check all external links for integrity"
-	@echo "  doctest   to run all doctests embedded in the documentation (if enabled)"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 
 clean:
-	-rm -rf $(BUILDDIR)/*
+	rm -rf $(BUILDDIR)/*
 
 html:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
 dirhtml:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
 pickle:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 	@echo
 	@echo "Build finished; now you can process the pickle files."
 
 json:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 	@echo
 	@echo "Build finished; now you can process the JSON files."
 
 htmlhelp:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 
 qthelp:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -72,35 +89,89 @@
 	@echo "To view the help file:"
 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc"
 
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/PyPy"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyPy"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
 latex:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
-	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
-	      "run these through (pdf)latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
 
 man:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	@echo
-	@echo "Build finished. The manual pages are in $(BUILDDIR)/man"
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
 changes:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo
 	@echo "The overview file is in $(BUILDDIR)/changes."
 
 linkcheck:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."
 
 doctest:
-	# python config/generate.py #readthedocs will not run this Makefile
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/doctest/output.txt."
+
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -105,7 +105,7 @@
         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise       # re-raise other app-level exceptions
                 break
@@ -348,8 +348,12 @@
 
 **objects**
 
-  Normal rules apply. Special methods are not honoured, except ``__init__``,
-  ``__del__`` and ``__iter__``.
+  Normal rules apply. The only special methods that are honoured are
+  ``__init__``, ``__del__``, ``__len__``, ``__getitem__``, ``__setitem__``,
+  ``__getslice__``, ``__setslice__``, and ``__iter__``. To handle slicing,
+  ``__getslice__`` and ``__setslice__`` must be used; using ``__getitem__`` and
+   ``__setitem__`` for slicing isn't supported. Additionally, using negative
+   indices for slicing is still not support, even when using ``__getslice__``.
 
 This layout makes the number of types to take care about quite limited.
 
@@ -567,7 +571,7 @@
 
     try:
         ...
-    except OperationError, e:
+    except OperationError as e:
         if not e.match(space, space.w_XxxError):
             raise
         ...
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -18,11 +18,31 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.append(os.path.abspath('.'))
 
+
+# -- Read The Docs theme config ------------------------------------------------
+
+# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    try:
+        import sphinx_rtd_theme
+        html_theme = 'sphinx_rtd_theme'
+        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+    except ImportError:
+        print('sphinx_rtd_theme is not installed')
+        html_theme = 'default'
+
+# otherwise, readthedocs.org uses their theme by default, so no need to specify it
+
+
 # -- General configuration -----------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.ifconfig', 'sphinx.ext.graphviz', 'pypyconfig']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx',
+              'sphinx.ext.todo', 'sphinx.ext.ifconfig', 'sphinx.ext.graphviz',
+              'pypyconfig']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -45,9 +65,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.2'
+version = '2.3'
 # The full version, including alpha/beta/rc tags.
-release = '2.2.1'
+release = '2.3.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -91,7 +111,7 @@
 
 # The theme to use for HTML and HTML Help pages.  Major themes that come with
 # Sphinx are currently 'default' and 'sphinxdoc'.
-html_theme = 'default'
+#html_theme = 'default'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
diff --git a/pypy/doc/config/translation.log.txt b/pypy/doc/config/translation.log.txt
--- a/pypy/doc/config/translation.log.txt
+++ b/pypy/doc/config/translation.log.txt
@@ -2,4 +2,4 @@
 
 These must be enabled by setting the PYPYLOG environment variable.
 The exact set of features supported by PYPYLOG is described in
-pypy/translation/c/src/debug_print.h.
+rpython/translator/c/src/debug_print.h.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -99,6 +99,7 @@
   Stian Andreassen
   Laurence Tratt
   Wanja Saatkamp
+  Ivan Sichmann Freitas
   Gerald Klix
   Mike Blume
   Oscar Nierstrasz
@@ -183,7 +184,9 @@
   Alejandro J. Cura
   Jacob Oscarson
   Travis Francis Athougies
+  Ryan Gonzalez
   Kristjan Valur Jonsson
+  Sebastian Pawlu&#347;
   Neil Blakey-Milner
   anatoly techtonik
   Lutz Paelike
@@ -216,6 +219,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Yasir Suhail
+  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -245,6 +249,8 @@
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
+  Asmo Soinio
+  Stefan Marr
   jiaaro
   opassembler.py
   Antony Lee
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -348,4 +348,9 @@
   type and vice versa. For builtin types, a dictionary will be returned that
   cannot be changed (but still looks and behaves like a normal dictionary).
 
+* PyPy prints a random line from past #pypy IRC topics at startup in
+  interactive mode. In a released version, this behaviour is supressed, but
+  setting the environment variable PYPY_IRC_TOPIC will bring it back. Note that
+  downstream package providers have been known to totally disable this feature.
+
 .. include:: _ref.txt
diff --git a/pypy/doc/ctypes-implementation.rst b/pypy/doc/ctypes-implementation.rst
--- a/pypy/doc/ctypes-implementation.rst
+++ b/pypy/doc/ctypes-implementation.rst
@@ -72,13 +72,11 @@
 Here is a list of the limitations and missing features of the
 current implementation:
 
-* ``ctypes.pythonapi`` lets you access the CPython C API emulation layer
-  of PyPy, at your own risks and without doing anything sensible about
-  the GIL.  Since PyPy 2.3, these functions are also named with an extra
-  "Py", for example ``PyPyInt_FromLong()``.  Basically, don't use this,
-  but it might more or less work in simple cases if you do.  (Obviously,
-  assuming the PyObject pointers you get have any particular fields in
-  any particular order is just going to crash.)
+* ``ctypes.pythonapi`` is missing.  In previous versions, it was present
+  and redirected to the `cpyext` C API emulation layer, but our
+  implementation did not do anything sensible about the GIL and the
+  functions were named with an extra "Py", for example
+  ``PyPyInt_FromLong()``.  It was removed for being unhelpful.
 
 * We copy Python strings instead of having pointers to raw buffers
 
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -8,6 +8,9 @@
 *Articles about PyPy published so far, most recent first:* (bibtex_ file)
 
 
+* `A Way Forward in Parallelising Dynamic Languages`_,
+  R. Meier, A. Rigo
+
 * `Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`_,
   C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo
 
@@ -71,6 +74,7 @@
 
 
 .. _bibtex: https://bitbucket.org/pypy/extradoc/raw/tip/talk/bibtex.bib
+.. _`A Way Forward in Parallelising Dynamic Languages`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf
 .. _`Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
 .. _`Allocation Removal by Partial Evaluation in a Tracing JIT`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
 .. _`Towards a Jitting VM for Prolog Execution`: http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
@@ -93,6 +97,11 @@
 Talks and Presentations 
 ----------------------------------
 
+*This part is no longer updated.*  The complete list is here__ (in
+alphabetical order).
+
+.. __: https://bitbucket.org/pypy/extradoc/src/extradoc/talk/
+
 Talks in 2010
 +++++++++++++
 
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -171,16 +171,21 @@
 You might be interested in our `benchmarking site`_ and our 
 `jit documentation`_.
 
-Note that the JIT has a very high warm-up cost, meaning that the
-programs are slow at the beginning.  If you want to compare the timings
-with CPython, even relatively simple programs need to run *at least* one
-second, preferrably at least a few seconds.  Large, complicated programs
-need even more time to warm-up the JIT.
+`Your tests are not a benchmark`_: tests tend to be slow under PyPy
+because they run exactly once; if they are good tests, they exercise
+various corner cases in your code.  This is a bad case for JIT
+compilers.  Note also that our JIT has a very high warm-up cost, meaning
+that any program is slow at the beginning.  If you want to compare the
+timings with CPython, even relatively simple programs need to run *at
+least* one second, preferrably at least a few seconds.  Large,
+complicated programs need even more time to warm-up the JIT.
 
 .. _`benchmarking site`: http://speed.pypy.org
 
 .. _`jit documentation`: jit/index.html
 
+.. _`your tests are not a benchmark`: http://alexgaynor.net/2013/jul/15/your-tests-are-not-benchmark/
+
 ---------------------------------------------------------------
 Couldn't the JIT dump and reload already-compiled machine code?
 ---------------------------------------------------------------
@@ -465,9 +470,13 @@
 
 This is documented (here__ and here__).  It needs 4 GB of RAM to run
 "rpython targetpypystandalone" on top of PyPy, a bit more when running
-on CPython.  If you have less than 4 GB it will just swap forever (or
-fail if you don't have enough swap).  On 32-bit, divide the numbers by
-two.
+on top of CPython.  If you have less than 4 GB free, it will just swap
+forever (or fail if you don't have enough swap).  And we mean *free:*
+if the machine has 4 GB *in total,* then it will swap.
+
+On 32-bit, divide the numbers by two.  (We didn't try recently, but in
+the past it was possible to compile a 32-bit version on a 2 GB Linux
+machine with nothing else running: no Gnome/KDE, for example.)
 
 .. __: http://pypy.org/download.html#building-from-source
 .. __: https://pypy.readthedocs.org/en/latest/getting-started-python.html#translating-the-pypy-python-interpreter
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -28,8 +28,6 @@
   pypy/doc/tool/makecontributor.py generates the list of contributors
 * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
   and create a fresh whatsnew_head.rst after the release
-* update README
-* change the tracker to have a new release tag to file bugs against
 * go to pypy/tool/release and run:
   force-builds.py <release branch>

From noreply at buildbot.pypy.org  Mon Jul 20 21:11:32 2015
From: noreply at buildbot.pypy.org (wlav)
Date: Mon, 20 Jul 2015 21:11:32 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: update bound classes for added
	tests
Message-ID: <20150720191132.907191C0962@cobra.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r78619:d3bef9990ae7
Date: 2014-07-16 13:00 -0700
http://bitbucket.org/pypy/pypy/changeset/d3bef9990ae7/

Log:	update bound classes for added tests

diff --git a/pypy/module/cppyy/test/example01_LinkDef.h b/pypy/module/cppyy/test/example01_LinkDef.h
--- a/pypy/module/cppyy/test/example01_LinkDef.h
+++ b/pypy/module/cppyy/test/example01_LinkDef.h
@@ -7,6 +7,9 @@
 #pragma link C++ class example01;
 #pragma link C++ typedef example01_t;
 #pragma link C++ class example01a;
+#pragma link C++ class example01b;
+#pragma link C++ class example01c;
+#pragma link C++ class example01d;
 #pragma link C++ class payload;
 #pragma link C++ class ArgPasser;
 #pragma link C++ class z_;

From noreply at buildbot.pypy.org  Mon Jul 20 21:11:33 2015
From: noreply at buildbot.pypy.org (wlav)
Date: Mon, 20 Jul 2015 21:11:33 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: close reflex-support
Message-ID: <20150720191133.A7F091C089E@cobra.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r78620:a5036b5b1dd8
Date: 2015-07-20 12:10 -0700
http://bitbucket.org/pypy/pypy/changeset/a5036b5b1dd8/

Log:	close reflex-support


From noreply at buildbot.pypy.org  Tue Jul 21 13:30:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Jul 2015 13:30:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add another test (passing)
Message-ID: <20150721113058.9848D1C089E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78621:a1acfcae73b3
Date: 2015-07-21 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/a1acfcae73b3/

Log:	Add another test (passing)

diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -783,6 +783,7 @@
             (rffi.USHORT, rffi.USHORT, ""),
             (rffi.USHORT, rffi.LONG, ""),
             (rffi.USHORT, rffi.ULONG, ""),
+            (rffi.USHORT, lltype.Bool, "int_is_true %i0 -> %i1"),
 
             (rffi.LONG, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"),

From noreply at buildbot.pypy.org  Tue Jul 21 13:30:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Jul 2015 13:30:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a test (passing at least on C
	compilers with _Bool)
Message-ID: <20150721113059.D338B1C0962@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78622:ff0ebb4b3a7d
Date: 2015-07-21 12:22 +0200
http://bitbucket.org/pypy/pypy/changeset/ff0ebb4b3a7d/

Log:	Add a test (passing at least on C compilers with _Bool)

diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py
--- a/rpython/translator/c/test/test_lltyped.py
+++ b/rpython/translator/c/test/test_lltyped.py
@@ -981,3 +981,13 @@
         assert fn(0) == 3
         assert fn(10) == 42
         assert fn(100) == -10
+
+    def test_cast_to_bool(self):
+        def f(n):
+            return rffi.cast(Bool, n)
+
+        fn = self.getcompiled(f, [int])
+        assert fn(0) == False
+        assert fn(1) == True
+        assert fn(256) == True
+        assert fn(-2**24) == True

From noreply at buildbot.pypy.org  Tue Jul 21 13:31:00 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Jul 2015 13:31:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2086: attempted fix
Message-ID: <20150721113100.F07FF1C1035@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78623:55c0403ff1d9
Date: 2015-07-21 13:15 +0200
http://bitbucket.org/pypy/pypy/changeset/55c0403ff1d9/

Log:	Issue #2086: attempted fix

diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h
--- a/pypy/module/cpyext/include/modsupport.h
+++ b/pypy/module/cpyext/include/modsupport.h
@@ -95,6 +95,13 @@
 
 PyAPI_DATA(char *) _Py_PackageContext;
 
+/* hack hack hack */
+#ifndef __va_copy
+# ifdef va_copy
+#  define __va_copy(a,b) va_copy(a,b)
+# endif
+#endif
+
 #ifdef __cplusplus
 }
 #endif

From noreply at buildbot.pypy.org  Tue Jul 21 13:31:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Jul 2015 13:31:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: More tests,
 one fix for cast_primitive(Bool, r_longlong())
Message-ID: <20150721113102.1A51F1C13F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78624:530d71942802
Date: 2015-07-21 13:27 +0200
http://bitbucket.org/pypy/pypy/changeset/530d71942802/

Log:	More tests, one fix for cast_primitive(Bool, r_longlong())

diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -480,7 +480,8 @@
     }
 _cast_from_Signed = {
     lltype.Signed:         None,
-    lltype.Bool:           'int_is_true',
+    #lltype.Bool:           'int_is_true', ---disabled, see test_cast_to_bool_1
+    #                                      in translator/c/test/test_lltyped.py
     lltype.Char:           'cast_int_to_char',
     lltype.UniChar:        'cast_int_to_unichar',
     lltype.Float:          'cast_int_to_float',
diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py
--- a/rpython/translator/c/test/test_lltyped.py
+++ b/rpython/translator/c/test/test_lltyped.py
@@ -982,7 +982,27 @@
         assert fn(10) == 42
         assert fn(100) == -10
 
-    def test_cast_to_bool(self):
+    def test_cast_to_bool_1(self):
+        def f(n):
+            return cast_primitive(Bool, n)
+
+        fn = self.getcompiled(f, [int])
+        assert fn(0) == False
+        assert fn(1) == True
+        assert fn(256) == True
+        assert fn(-2**24) == True
+
+    def test_cast_to_bool_1_longlong(self):
+        def f(n):
+            return cast_primitive(Bool, n)
+
+        fn = self.getcompiled(f, [r_longlong])
+        assert fn(r_longlong(0)) == False
+        assert fn(r_longlong(1)) == True
+        assert fn(r_longlong(256)) == True
+        assert fn(r_longlong(2**32)) == True
+
+    def test_cast_to_bool_2(self):
         def f(n):
             return rffi.cast(Bool, n)
 
@@ -991,3 +1011,13 @@
         assert fn(1) == True
         assert fn(256) == True
         assert fn(-2**24) == True
+
+    def test_cast_to_bool_2_longlong(self):
+        def f(n):
+            return rffi.cast(Bool, n)
+
+        fn = self.getcompiled(f, [r_longlong])
+        assert fn(r_longlong(0)) == False
+        assert fn(r_longlong(1)) == True
+        assert fn(r_longlong(256)) == True
+        assert fn(r_longlong(2**32)) == True

From noreply at buildbot.pypy.org  Tue Jul 21 14:07:50 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Jul 2015 14:07:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for casts from r_longlong to bool
	on 32-bit
Message-ID: <20150721120750.0543F1C145B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78625:27cf9aad1e7d
Date: 2015-07-21 14:07 +0200
http://bitbucket.org/pypy/pypy/changeset/27cf9aad1e7d/

Log:	Fix for casts from r_longlong to bool on 32-bit

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1220,6 +1220,15 @@
         if longlong_arg and longlong_res:
             return
         elif longlong_arg:
+            if v_result.concretetype is lltype.Bool:
+                longlong_zero = rffi.cast(v_arg.concretetype, 0)
+                c_longlong_zero = Constant(longlong_zero, v_arg.concretetype)
+                if unsigned1:
+                    name = 'ullong_ne'
+                else:
+                    name = 'llong_ne'
+                op1 = SpaceOperation(name, [v_arg, c_longlong_zero], v_result)
+                return self.rewrite_operation(op1)
             v = varoftype(lltype.Signed)
             op1 = self.rewrite_operation(
                 SpaceOperation('truncate_longlong_to_int', [v_arg], v)
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -789,6 +789,7 @@
             (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
             (rffi.LONG, rffi.SHORT, "int_signext %i0, $2 -> %i1"),
             (rffi.LONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"),
+            (rffi.LONG, lltype.Bool, "int_is_true %i0 -> %i1"),
             (rffi.LONG, rffi.LONG, ""),
             (rffi.LONG, rffi.ULONG, ""),
 
@@ -796,7 +797,7 @@
             (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
             (rffi.ULONG, rffi.SHORT, "int_signext %i0, $2 -> %i1"),
             (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"),
-            (rffi.LONG, lltype.Bool, "int_is_true %i0 -> %i1"),
+            (rffi.ULONG, lltype.Bool, "int_is_true %i0 -> %i1"),
             (rffi.ULONG, rffi.LONG, ""),
             (rffi.ULONG, rffi.ULONG, ""),
             ]:
@@ -818,8 +819,15 @@
                         FROM = rffi.LONGLONG
                     else:
                         FROM = rffi.ULONGLONG
-                    expected.insert(0,
-                        "residual_call_irf_i $<* fn llong_to_int>, I[], R[], F[%f0], <Descr> -> %i0")
+                    if TO == lltype.Bool:
+                        prefix = 'u' if FROM == rffi.ULONGLONG else ''
+                        expected = [
+                            "residual_call_irf_i $<* fn %sllong_ne>, I[], R[], F[%%f0, $0L], <Descr> -> %%i0" % prefix,
+                            "int_return %i0",
+                        ]
+                    else:
+                        expected.insert(0,
+                            "residual_call_irf_i $<* fn llong_to_int>, I[], R[], F[%f0], <Descr> -> %i0")
                     expectedstr = '\n'.join(expected)
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
diff --git a/rpython/jit/metainterp/test/test_longlong.py b/rpython/jit/metainterp/test/test_longlong.py
--- a/rpython/jit/metainterp/test/test_longlong.py
+++ b/rpython/jit/metainterp/test/test_longlong.py
@@ -1,4 +1,5 @@
 import py, sys
+from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint, intmask
 from rpython.jit.metainterp.test.support import LLJitMixin
 
@@ -228,6 +229,20 @@
         res = self.interp_operations(f, [0x56789ABC])
         assert intmask(res) == intmask(0xABC00000)
 
+    def test_cast_longlong_to_bool(self):
+        def f(n):
+            m = r_longlong(n) << 20
+            return lltype.cast_primitive(lltype.Bool, m)
+        res = self.interp_operations(f, [2**12])
+        assert res == 1
+
+    def test_cast_ulonglong_to_bool(self):
+        def f(n):
+            m = r_ulonglong(n) << 20
+            return lltype.cast_primitive(lltype.Bool, m)
+        res = self.interp_operations(f, [2**12])
+        assert res == 1
+
 
 class TestLLtype(LongLongTests, LLJitMixin):
     pass

From noreply at buildbot.pypy.org  Tue Jul 21 14:48:55 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 21 Jul 2015 14:48:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: silence this warning,
 it seems we never look at them anyway
Message-ID: <20150721124855.B92481C0627@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78626:1c952f7cf89a
Date: 2015-07-21 14:49 +0200
http://bitbucket.org/pypy/pypy/changeset/1c952f7cf89a/

Log:	silence this warning, it seems we never look at them anyway

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -916,9 +916,9 @@
                             name, None)
                         if attrvalue is None:
                             # Ellipsis from get_reusable_prebuilt_instance()
-                            if value is not Ellipsis:
-                                warning("prebuilt instance %r has no "
-                                        "attribute %r" % (value, name))
+                            #if value is not Ellipsis:
+                                #warning("prebuilt instance %r has no "
+                                #        "attribute %r" % (value, name))
                             llattrvalue = r.lowleveltype._defl()
                         else:
                             llattrvalue = r.convert_desc_or_const(attrvalue)

From noreply at buildbot.pypy.org  Tue Jul 21 18:19:44 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 21 Jul 2015 18:19:44 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Add 'n' and 'N' struct conversion codes.
Message-ID: <20150721161944.853161C089E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.3
Changeset: r78627:a6a689eac5da
Date: 2015-07-21 18:19 +0200
http://bitbucket.org/pypy/pypy/changeset/a6a689eac5da/

Log:	Add 'n' and 'N' struct conversion codes.

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -127,6 +127,7 @@
                   <= calcsize('l') == calcsize('L')
                   <= calcsize('q') == calcsize('Q'))
         assert 4 <= calcsize('f') <= 8 <= calcsize('d')
+        assert calcsize('n') == calcsize('N') >= calcsize('P')
         assert calcsize('13s') == 13
         assert calcsize('500p') == 500
         assert 4 <= calcsize('P') <= 8
diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -88,11 +88,13 @@
                'i': 'signed int',
                'l': 'signed long',
                'q': 'signed long long',
+               'n': 'ssize_t',
                'B': 'unsigned char',
                'H': 'unsigned short',
                'I': 'unsigned int',
                'L': 'unsigned long',
                'Q': 'unsigned long long',
+               'N': 'size_t',
                'P': 'char *',
                'f': 'float',
                'd': 'double',
@@ -100,6 +102,7 @@
                }
 
     pre_include_bits = ["""
+        #include <sys/types.h>
         #ifdef _MSC_VER
         #define _Bool char
         #endif"""]

From noreply at buildbot.pypy.org  Tue Jul 21 19:39:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Jul 2015 19:39:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: Unsure,
 but it may be the case that int_is_true is better...
Message-ID: <20150721173911.547051C0627@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78628:8250500bac26
Date: 2015-07-21 19:39 +0200
http://bitbucket.org/pypy/pypy/changeset/8250500bac26/

Log:	Unsure, but it may be the case that int_is_true is better...

diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -480,8 +480,6 @@
     }
 _cast_from_Signed = {
     lltype.Signed:         None,
-    #lltype.Bool:           'int_is_true', ---disabled, see test_cast_to_bool_1
-    #                                      in translator/c/test/test_lltyped.py
     lltype.Char:           'cast_int_to_char',
     lltype.UniChar:        'cast_int_to_unichar',
     lltype.Float:          'cast_int_to_float',
@@ -503,6 +501,8 @@
             if op:
                 v_value = llops.genop(op, [v_value], resulttype=TGT)
             return v_value
+        elif ORIG is lltype.Signed and TGT is lltype.Bool:
+            return llops.genop('int_is_true', [v_value], resulttype=lltype.Bool)
         else:
             # use the generic operation if there is no alternative
             return llops.genop('cast_primitive', [v_value], resulttype=TGT)
diff --git a/rpython/rtyper/test/test_rbuiltin.py b/rpython/rtyper/test/test_rbuiltin.py
--- a/rpython/rtyper/test/test_rbuiltin.py
+++ b/rpython/rtyper/test/test_rbuiltin.py
@@ -541,6 +541,14 @@
             return lltype.cast_primitive(lltype.Signed, v)
         res = self.interpret(llf, [rffi.r_short(123)], policy=LowLevelAnnotatorPolicy())
         assert res == 123
+        def llf(v):
+            return lltype.cast_primitive(lltype.Bool, v)
+        res = self.interpret(llf, [2**24], policy=LowLevelAnnotatorPolicy())
+        assert res == True
+        def llf(v):
+            return lltype.cast_primitive(lltype.Bool, v)
+        res = self.interpret(llf, [rffi.r_longlong(2**48)], policy=LowLevelAnnotatorPolicy())
+        assert res == True
 
     def test_force_cast(self):
         def llfn(v):

From noreply at buildbot.pypy.org  Tue Jul 21 19:48:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Jul 2015 19:48:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2081: mips64 patch for stacklet
Message-ID: <20150721174821.5ED1E1C0962@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78629:1fcd70b936df
Date: 2015-07-21 19:48 +0200
http://bitbucket.org/pypy/pypy/changeset/1fcd70b936df/

Log:	Issue #2081: mips64 patch for stacklet

diff --git a/rpython/translator/c/src/stacklet/slp_platformselect.h b/rpython/translator/c/src/stacklet/slp_platformselect.h
--- a/rpython/translator/c/src/stacklet/slp_platformselect.h
+++ b/rpython/translator/c/src/stacklet/slp_platformselect.h
@@ -10,6 +10,8 @@
 #include "switch_x86_gcc.h" /* gcc on X86 */
 #elif defined(__GNUC__) && defined(__arm__)
 #include "switch_arm_gcc.h" /* gcc on arm */
+#elif defined(__GNUC__) && defined(__mips__) && defined(_ABI64)
+#include "switch_mips64_gcc.h" /* gcc on mips64 */
 #else
 #error "Unsupported platform!"
 #endif

From noreply at buildbot.pypy.org  Tue Jul 21 20:40:13 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 21 Jul 2015 20:40:13 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-docstrings: close branch before merging
Message-ID: <20150721184013.99A291C0627@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: numpy-docstrings
Changeset: r78630:446ec1af195f
Date: 2015-07-21 19:38 +0100
http://bitbucket.org/pypy/pypy/changeset/446ec1af195f/

Log:	close branch before merging


From noreply at buildbot.pypy.org  Tue Jul 21 20:40:14 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 21 Jul 2015 20:40:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge branch 'numpy-docstrings'
Message-ID: <20150721184014.D78FD1C0627@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78631:89862db1c9f1
Date: 2015-07-21 19:38 +0100
http://bitbucket.org/pypy/pypy/changeset/89862db1c9f1/

Log:	merge branch 'numpy-docstrings'

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -2,7 +2,9 @@
 
 
 class MultiArrayModule(MixedModule):
-    appleveldefs = {'arange': 'app_numpy.arange'}
+    appleveldefs = {
+        'arange': 'app_numpy.arange',
+        'add_docstring': 'app_numpy.add_docstring'}
     interpleveldefs = {
         'ndarray': 'ndarray.W_NDimArray',
         'dtype': 'descriptor.W_Dtype',
@@ -29,6 +31,8 @@
         'set_string_function': 'appbridge.set_string_function',
         'typeinfo': 'descriptor.get_dtype_cache(space).w_typeinfo',
         'nditer': 'nditer.W_NDIter',
+
+        'set_docstring': 'support.descr_set_docstring',
     }
     for c in ['MAXDIMS', 'CLIP', 'WRAP', 'RAISE']:
         interpleveldefs[c] = 'space.wrap(constants.%s)' % c
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -3,6 +3,7 @@
 import math
 
 import _numpypy
+from _numpypy.multiarray import set_docstring
 
 def arange(start, stop=None, step=1, dtype=None):
     '''arange([start], stop[, step], dtype=None)
@@ -22,3 +23,13 @@
         arr[j] = i
         i += step
     return arr
+
+
+def add_docstring(obj, docstring):
+    old_doc = getattr(obj, '__doc__', None)
+    if old_doc is not None:
+        raise RuntimeError("%s already has a docstring" % obj)
+    try:
+        set_docstring(obj, docstring)
+    except:
+        raise TypeError("Cannot set a docstring for %s" % obj)
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,8 +1,12 @@
-from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import rffi, lltype
 
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.gateway import unwrap_spec, appdef
+from pypy.interpreter.typedef import GetSetProperty
+from pypy.objspace.std.typeobject import W_TypeObject
+from pypy.objspace.std.objspace import StdObjSpace
 
 def issequence_w(space, w_obj):
     from pypy.module.micronumpy.base import W_NDimArray
@@ -172,3 +176,27 @@
     elif req_order == 'A':
         return proto_order
 
+
+def descr_set_docstring(space, w_obj, w_docstring):
+    if not isinstance(space, StdObjSpace):
+        raise oefmt(space.w_NotImplementedError,
+                    "This only works with the real object space")
+    if isinstance(w_obj, W_TypeObject):
+        w_obj.w_doc = w_docstring
+        return
+    elif isinstance(w_obj, GetSetProperty):
+        if space.is_none(w_docstring):
+            doc = None
+        else:
+            doc = space.str_w(w_docstring)
+        w_obj.doc = doc
+        return
+    app_set_docstring(space, w_obj, w_docstring)
+
+app_set_docstring = appdef("""app_set_docstring_(obj, docstring):
+    import types
+    if isinstance(obj, types.MethodType):
+        obj.im_func.__doc__ = docstring
+    else:
+        obj.__doc__ = docstring
+""")
diff --git a/pypy/module/micronumpy/test/test_support_app.py b/pypy/module/micronumpy/test/test_support_app.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_support_app.py
@@ -0,0 +1,49 @@
+"""App-level tests for support.py"""
+import sys
+import py
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.conftest import option
+
+class AppTestSupport(BaseNumpyAppTest):
+    def setup_class(cls):
+        if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
+        BaseNumpyAppTest.setup_class.im_func(cls)
+
+    def test_add_docstring(self):
+        import numpy as np
+        foo = lambda: None
+        np.add_docstring(foo, "Does a thing")
+        assert foo.__doc__ == "Does a thing"
+
+    def test_type_docstring(self):
+        import numpy as np
+        import types
+        obj = types.ModuleType
+        doc = obj.__doc__
+        try:
+            np.set_docstring(obj, 'foo')
+            assert obj.__doc__ == 'foo'
+        finally:
+            np.set_docstring(obj, doc)
+
+        raises(RuntimeError, np.add_docstring, obj, 'foo')
+
+    def test_method_docstring(self):
+        import numpy as np
+        doc = int.bit_length.__doc__
+        try:
+            np.set_docstring(int.bit_length, 'foo')
+            assert int.bit_length.__doc__ == 'foo'
+        finally:
+            np.set_docstring(int.bit_length, doc)
+
+    def test_property_docstring(self):
+        import numpy as np
+        doc = np.flatiter.base.__doc__
+        try:
+            np.set_docstring(np.flatiter.base, 'foo')
+            assert np.flatiter.base.__doc__ == 'foo'
+        finally:
+            np.set_docstring(np.flatiter.base, doc)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1361,3 +1361,26 @@
         assert np.add(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
         assert np.subtract(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
         assert np.divide(np.zeros(5, dtype=np.int8), 257).dtype == np.int16
+
+    def test_add_doc(self):
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            skip('')
+        try:
+            from numpy import set_docstring
+        except ImportError:
+            from _numpypy.multiarray import set_docstring
+        import numpy as np
+        assert np.add.__doc__ is None
+        add_doc = np.add.__doc__
+        ufunc_doc = np.ufunc.__doc__
+        try:
+            np.add.__doc__ = 'np.add'
+            assert np.add.__doc__ == 'np.add'
+            # Test for interferences between ufunc objects and their class
+            set_docstring(np.ufunc, 'np.ufunc')
+            assert np.ufunc.__doc__ == 'np.ufunc'
+            assert np.add.__doc__ == 'np.add'
+        finally:
+            set_docstring(np.ufunc, ufunc_doc)
+            np.add.__doc__ = add_doc
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -86,6 +86,7 @@
         "identity", "int_only", "allow_bool", "allow_complex",
         "complex_to_float", "nargs", "nout", "signature"
     ]
+    w_doc = None
 
     def __init__(self, name, promote_to_largest, promote_to_float, promote_bools,
                  identity, int_only, allow_bool, allow_complex, complex_to_float):
@@ -105,6 +106,15 @@
     def descr_repr(self, space):
         return space.wrap("<ufunc '%s'>" % self.name)
 
+    def get_doc(self, space):
+        # Note: allows any object to be set as docstring, because why not?
+        if self.w_doc is None:
+            return space.w_None
+        return self.w_doc
+
+    def set_doc(self, space, w_doc):
+        self.w_doc = w_doc
+
     def descr_get_identity(self, space):
         if self.identity is None:
             return space.w_None
@@ -1144,6 +1154,7 @@
     __call__ = interp2app(W_Ufunc.descr_call),
     __repr__ = interp2app(W_Ufunc.descr_repr),
     __name__ = GetSetProperty(W_Ufunc.descr_get_name),
+    __doc__ = GetSetProperty(W_Ufunc.get_doc, W_Ufunc.set_doc),
 
     identity = GetSetProperty(W_Ufunc.descr_get_identity),
     accumulate = interp2app(W_Ufunc.descr_accumulate),
@@ -1157,8 +1168,6 @@
 )
 
 
-
-
 def ufunc_dtype_caller(space, ufunc_name, op_name, nin, bool_result):
     def get_op(dtype):
         try:
@@ -1481,7 +1490,7 @@
     if w_ret.external_loop:
         _parse_signature(space, w_ret, w_ret.signature)
     if doc:
-        w_ret.w_doc = space.wrap(doc)
+        w_ret.set_doc(space, space.wrap(doc))
     return w_ret
 
 # Instantiated in cpyext/ndarrayobject. It is here since ufunc calls

From noreply at buildbot.pypy.org  Tue Jul 21 20:40:16 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 21 Jul 2015 20:40:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <20150721184016.060E41C0627@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78632:919340f70c2a
Date: 2015-07-21 19:40 +0100
http://bitbucket.org/pypy/pypy/changeset/919340f70c2a/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -45,4 +45,9 @@
 allow subclassing dtype
 
 .. branch: indexing
+
 Refactor array indexing to support ellipses.
+
+.. branch: numpy-docstrings
+
+Allow the docstrings of built-in numpy objects to be set at run-time.

From noreply at buildbot.pypy.org  Tue Jul 21 23:12:21 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 21 Jul 2015 23:12:21 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-buffered: more strides -> strideops
	module substitution
Message-ID: <20150721211221.6D3CC1C0627@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-buffered
Changeset: r78633:3673d97c6f8c
Date: 2015-07-20 16:16 -0400
http://bitbucket.org/pypy/pypy/changeset/3673d97c6f8c/

Log:	more strides -> strideops module substitution

diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py
--- a/pypy/module/micronumpy/arrayops.py
+++ b/pypy/module/micronumpy/arrayops.py
@@ -4,7 +4,7 @@
 from pypy.module.micronumpy import constants as NPY
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
 from pypy.module.micronumpy.converters import clipmode_converter
-from pypy.module.micronumpy.strides import (
+from pypy.module.micronumpy.strideops import (
     Chunk, Chunks, shape_agreement, shape_agreement_multiple)
 from .casting import find_binop_result_dtype, find_result_type
 
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -38,7 +38,7 @@
     @staticmethod
     def from_shape(space, shape, dtype, order='C', w_instance=None, zero=True):
         from pypy.module.micronumpy import concrete, descriptor, boxes
-        from pypy.module.micronumpy.strides import calc_strides
+        from pypy.module.micronumpy.strideops import calc_strides
         strides, backstrides = calc_strides(shape, dtype.base, order)
         impl = concrete.ConcreteArray(shape, dtype.base, order, strides,
                                       backstrides, zero=zero)
@@ -53,7 +53,7 @@
                                order='C', owning=False, w_subtype=None,
                                w_base=None, writable=True, strides=None, start=0):
         from pypy.module.micronumpy import concrete
-        from pypy.module.micronumpy.strides import (calc_strides,
+        from pypy.module.micronumpy.strideops import (calc_strides,
                                                     calc_backstrides)
         isize = dtype.elsize
         if storage_bytes > 0 :
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -9,7 +9,7 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray, \
     ArrayArgumentException, W_NumpyObject
 from pypy.module.micronumpy.iterators import ArrayIter
-from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
+from pypy.module.micronumpy.strideops import (Chunk, Chunks, NewAxisChunk,
     RecordChunk, calc_strides, calc_new_strides, shape_agreement,
     calculate_broadcast_strides, calc_backstrides, calc_start, is_c_contiguous,
     is_f_contiguous)
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -82,7 +82,7 @@
     return w_res
 
 def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
-    from pypy.module.micronumpy import strides
+    from pypy.module.micronumpy import strideops
 
     # for anything that isn't already an array, try __array__ method first
     if not isinstance(w_object, W_NDimArray):
@@ -141,7 +141,7 @@
                     w_base=w_base, start=imp.start)
     else:
         # not an array
-        shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
+        shape, elems_w = strideops.find_shape_and_elems(space, w_object, dtype)
     if dtype is None and space.isinstance_w(w_object, space.w_buffer):
         dtype = descriptor.get_dtype_cache(space).w_uint8dtype
     if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1):
@@ -163,7 +163,7 @@
 def numpify(space, w_object):
     """Convert the object to a W_NumpyObject"""
     # XXX: code duplication with _array()
-    from pypy.module.micronumpy import strides
+    from pypy.module.micronumpy import strideops
     if isinstance(w_object, W_NumpyObject):
         return w_object
     # for anything that isn't already an array, try __array__ method first
@@ -171,7 +171,7 @@
     if w_array is not None:
         return w_array
 
-    shape, elems_w = strides.find_shape_and_elems(space, w_object, None)
+    shape, elems_w = strideops.find_shape_and_elems(space, w_object, None)
     dtype = find_dtype_for_seq(space, elems_w, None)
     if dtype is None:
         dtype = descriptor.get_dtype_cache(space).w_float64dtype
diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py
--- a/pypy/module/micronumpy/flagsobj.py
+++ b/pypy/module/micronumpy/flagsobj.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import constants as NPY
-from pypy.module.micronumpy.strides import is_c_contiguous, is_f_contiguous
+from pypy.module.micronumpy.strideops import is_c_contiguous, is_f_contiguous
 
 def enable_flags(arr, flags):
     arr.flags |= flags
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -18,8 +18,9 @@
 from pypy.module.micronumpy.converters import multi_axis_converter, \
     order_converter, shape_converter, searchside_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
-from pypy.module.micronumpy.strideops import get_shape_from_iterable, \
-    shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
+from pypy.module.micronumpy.strideops import (get_shape_from_iterable,
+    shape_agreement, shape_agreement_multiple, is_c_contiguous,
+    is_f_contiguous, calc_strides)
 from pypy.module.micronumpy.casting import can_cast_array
 
 
@@ -1298,7 +1299,6 @@
 def descr_new_array(space, w_subtype, w_shape, w_dtype=None, w_buffer=None,
                     offset=0, w_strides=None, w_order=None):
     from pypy.module.micronumpy.concrete import ConcreteArray
-    from pypy.module.micronumpy.strides import calc_strides
     dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
         space.gettypefor(descriptor.W_Dtype), w_dtype))
     shape = shape_converter(space, w_shape, dtype)
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -212,7 +212,7 @@
 def get_iter(space, order, arr, shape, dtype, op_flags, base):
     imp = arr.implementation
     backward = is_backward(imp, order)
-    if len(shape) == 1:
+    if len(shape) == 1 and len(imp.shape) > 0:
         min_dim = 0
         min_stride = 0xefffffff
         for i in range(len(imp.shape)):
@@ -316,6 +316,7 @@
     shape = [s+1 for s in old_iter.shape_m1]
     strides = old_iter.strides
     backstrides = old_iter.backstrides
+    print shape, strides, backstrides
     if order == 'F':
         new_shape = shape[1:]
         new_strides = strides[1:]
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -22,7 +22,7 @@
 from rpython.tool.sourcetools import func_with_new_name
 from pypy.module.micronumpy import boxes, support
 from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage, V_OBJECTSTORE
-from pypy.module.micronumpy.strides import calc_strides
+from pypy.module.micronumpy.strideops import calc_strides
 from . import constants as NPY
 
 degToRad = math.pi / 180.0
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -17,7 +17,7 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
 from pypy.module.micronumpy.ctors import numpify
 from pypy.module.micronumpy.nditer import W_NDIter, coalesce_iter
-from pypy.module.micronumpy.strides import shape_agreement
+from pypy.module.micronumpy.strideops import shape_agreement
 from pypy.module.micronumpy.support import (_parse_signature, product,
         get_storage_as_int, is_rhs_priority_higher)
 from .casting import (
@@ -861,6 +861,7 @@
                       w_itershape)
         # coalesce each iterators, according to inner_dimensions
         for i in range(len(inargs) + len(outargs)):
+            print 'ndim', self.core_num_dims[i]
             for j in range(self.core_num_dims[i]):
                 new_iter = coalesce_iter(nd_it.iters[i][0], nd_it.op_flags[i],
                                 nd_it, nd_it.order, flat=False)

From noreply at buildbot.pypy.org  Tue Jul 21 23:12:22 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 21 Jul 2015 23:12:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: test, fix issue #2090
Message-ID: <20150721211222.979371C0627@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78634:b0a67e1d9a20
Date: 2015-07-21 17:11 -0400
http://bitbucket.org/pypy/pypy/changeset/b0a67e1d9a20/

Log:	test, fix issue #2090

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -400,11 +400,17 @@
 
 class ConcreteArrayNotOwning(BaseConcreteArray):
     def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0):
+        if len(shape) > NPY.MAXDIMS:
+            raise oefmt(dtype.itemtype.space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
         make_sure_not_resized(shape)
         make_sure_not_resized(strides)
         make_sure_not_resized(backstrides)
         self.shape = shape
-        self.size = support.product(shape) * dtype.elsize
+        try:
+            self.size = support.product(shape) * dtype.elsize
+        except OverflowError as e:
+            raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big")
         self.order = order
         self.dtype = dtype
         self.strides = strides
@@ -445,8 +451,14 @@
                  storage=lltype.nullptr(RAW_STORAGE), zero=True):
         gcstruct = V_OBJECTSTORE
         flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
+        if len(shape) > NPY.MAXDIMS:
+            raise oefmt(dtype.itemtype.space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
         if storage == lltype.nullptr(RAW_STORAGE):
-            length = support.product(shape)
+            try:
+                length = support.product(shape)
+            except OverflowError as e:
+                raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big")
             if dtype.num == NPY.OBJECT:
                 storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True)
                 gcstruct = _create_objectstore(storage, length, dtype.elsize)
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -267,6 +267,11 @@
         y = array(x.T, copy=False)
         assert (y == x.T).all()
 
+        exc = raises(ValueError, ndarray, [1,2,256]*10000)
+        assert exc.value[0] == 'sequence too large; must be smaller than 32'
+        exc = raises(ValueError, ndarray, [1,2,256]*10)
+        assert exc.value[0] == 'array is too big'
+
     def test_ndmin(self):
         from numpy import array
 

From noreply at buildbot.pypy.org  Wed Jul 22 16:58:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 22 Jul 2015 16:58:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2081: Oups, missed the "hg add".
Message-ID: <20150722145815.CF59B1C037F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78635:98e9b6aa3536
Date: 2015-07-22 16:58 +0200
http://bitbucket.org/pypy/pypy/changeset/98e9b6aa3536/

Log:	Issue #2081: Oups, missed the "hg add".

diff --git a/rpython/translator/c/src/stacklet/switch_mips64_gcc.h b/rpython/translator/c/src/stacklet/switch_mips64_gcc.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/stacklet/switch_mips64_gcc.h
@@ -0,0 +1,63 @@
+static void *slp_switch(void *(*save_state)(void*, void*),
+                        void *(*restore_state)(void*, void*),
+                        void *extra)
+{
+  void *result;
+  __asm__ volatile (
+     "daddiu $sp, $sp, -0x50\n"
+     "sd $s0, 0x0($sp)\n" /* push the registers specified as caller-save */
+     "sd $s1, 0x8($sp)\n"
+     "sd $s2, 0x10($sp)\n"
+     "sd $s3, 0x18($sp)\n"
+     "sd $s4, 0x20($sp)\n"
+     "sd $s5, 0x28($sp)\n"
+     "sd $s6, 0x30($sp)\n"
+     "sd $s7, 0x38($sp)\n"
+     "sd $fp, 0x40($sp)\n"
+     "sd $ra, 0x48($sp)\n"
+
+     "move $s0, %[rstate]\n" /* save 'restore_state' for later */
+     "move $s1, %[extra]\n" /* save 'extra' for later */
+
+     "move $a1, %[extra]\n"/* arg 2: extra */
+     "move $a0, $sp\n" /* arg 1: current (old) stack pointer */
+                           
+     "move $t9, %[sstate]\n"
+     "jalr $t9\n" /* call save_state() */
+
+     "beqz $v0, 0f\n" /* skip the rest if the return value is null */
+
+     "move $sp, $v0\n" /* change the stack pointer */
+
+     /* From now on, the stack pointer is modified, but the content of the
+        stack is not restored yet.  It contains only garbage here. */
+
+     "move $a1, $s1\n" /* arg 2: extra */
+     "move $a0, $v0\n" /* arg 1: current (new) stack pointer */
+     "move $t9, $s0\n"
+     "jalr $t9\n" /* call restore_state() */
+
+     /* The stack's content is now restored. */
+
+     "0:\n"
+     "move %[result], $v0\n"
+     "ld $s0, 0x0($sp)\n"
+     "ld $s1, 0x8($sp)\n"
+     "ld $s2, 0x10($sp)\n"
+     "ld $s3, 0x18($sp)\n"
+     "ld $s4, 0x20($sp)\n"
+     "ld $s5, 0x28($sp)\n"
+     "ld $s6, 0x30($sp)\n"
+     "ld $s7, 0x38($sp)\n"
+     "ld $fp, 0x40($sp)\n"
+     "ld $ra, 0x48($sp)\n"
+     "daddiu $sp, $sp, 0x50\n"
+
+     : [result]"=&r"(result)
+     : [sstate]"r"(save_state),
+       [rstate]"r"(restore_state),
+       [extra]"r"(extra)
+     : "memory", "v0", "a0", "a1", "t9"
+     );
+  return result;
+}

From noreply at buildbot.pypy.org  Thu Jul 23 15:15:14 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 23 Jul 2015 15:15:14 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: probably fix some register saving
 around gil-release calls (still get crashes on raytrace)
Message-ID: <20150723131514.6041A1C15C8@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78636:c6df9651efef
Date: 2015-07-23 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/c6df9651efef/

Log:	probably fix some register saving around gil-release calls (still
	get crashes on raytrace)

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -128,15 +128,50 @@
         self.float_const_neg_addr = float_constants
         self.float_const_abs_addr = float_constants + 16
 
+
+    def _stm_save_regs_and_leave_transactional_zone(self, mc):
+        assert IS_X86_64 and self.cpu.supports_floats
+        # cannot save regs to frame as we are outside a transactional
+        # zone after "leaving". Thus, the frame must not be accessed afterwards.
+        # (there should also be no unsaved references left in registers)
+        gpr_regs = [gpr for gpr in gpr_reg_mgr_cls.save_around_call_regs if gpr != eax]
+        xmm_regs = xmm_reg_mgr_cls.all_regs
+
+        # Push the general purpose registers
+        for gpr in gpr_regs:
+            mc.PUSH(gpr)
+        # Push the XMM regs
+        xmm_space = len(xmm_regs) * WORD
+        mc.SUB_ri(esp.value, xmm_space)
+        for off, xmm in enumerate(xmm_regs):
+            mc.MOVSD_sx(off * WORD, xmm.value)
+
+        # align stack and CALL
+        pushed = len(gpr_regs) + len(xmm_regs) + 1  # +1 bc we are not aligned initially
+        aligned = bool(pushed % 2 == 0) # works for X86_64
+        if not aligned:
+            mc.SUB_ri(esp.value, WORD)
+        mc.CALL(imm(rstm.adr_stm_leave_noninevitable_transactional_zone))
+        if not aligned:
+            mc.ADD_ri(esp.value, WORD)
+
+        # Push the XMM regs
+        for off, xmm in enumerate(xmm_regs):
+            mc.MOVSD_xs(xmm.value, off * WORD)
+        mc.ADD_ri(esp.value, xmm_space)
+
+        # POP the general purpose registers
+        for gpr in reversed(gpr_regs):
+            mc.POP(gpr)
+
+
     def _build_stm_enter_leave_transactional_zone_helpers(self):
         assert IS_X86_64 and self.cpu.supports_floats
         # a helper to call _stm_leave_noninevitable_transactional_zone(),
         # preserving all registers that are used to pass arguments.
         # (Push an odd total number of registers, to align the stack.)
         mc = codebuf.MachineCodeBlockWrapper()
-        self._push_all_regs_to_frame(mc, [eax], True, callee_only=True)
-        mc.CALL(imm(rstm.adr_stm_leave_noninevitable_transactional_zone))
-        self._pop_all_regs_from_frame(mc, [eax], True, callee_only=True)
+        self._stm_save_regs_and_leave_transactional_zone(mc)
         mc.RET()
         self._stm_leave_noninevitable_tr_slowpath = mc.materialize(
             self.cpu, [])
@@ -146,12 +181,12 @@
         mc = codebuf.MachineCodeBlockWrapper()
         mc.SUB_ri(esp.value, 3 * WORD)     # 3 instead of 2 to align the stack
         mc.MOV_sr(0, eax.value)     # not edx, we're not running 32-bit
-        mc.MOVSD_sx(1, xmm0.value)
+        mc.MOVSD_sx(1 * WORD, xmm0.value)
         # load the value of 'tl->self_or_0_if_atomic' into edi as argument
         mc.MOV(edi, self.heap_stm_thread_local_self_or_0_if_atomic())
         mc.CALL(imm(rstm.adr_stm_reattach_transaction))
         # pop
-        mc.MOVSD_xs(xmm0.value, 1)
+        mc.MOVSD_xs(xmm0.value, 1 * WORD)
         mc.MOV_rs(eax.value, 0)
         mc.ADD_ri(esp.value, 3 * WORD)
         mc.RET()

From noreply at buildbot.pypy.org  Thu Jul 23 15:46:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 23 Jul 2015 15:46:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal,
	arigo) add a fast-path straight from cpython
Message-ID: <20150723134647.056E61C0433@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78637:7ebfda5c3881
Date: 2015-07-23 15:45 +0200
http://bitbucket.org/pypy/pypy/changeset/7ebfda5c3881/

Log:	(fijal, arigo) add a fast-path straight from cpython

diff --git a/pypy/module/__builtin__/test/test_abstractinst.py b/pypy/module/__builtin__/test/test_abstractinst.py
--- a/pypy/module/__builtin__/test/test_abstractinst.py
+++ b/pypy/module/__builtin__/test/test_abstractinst.py
@@ -202,3 +202,18 @@
             __subclass__ = set([int])
         assert issubclass(int, Integer)
         assert issubclass(int, (Integer,))
+
+    def test_dont_call_instancecheck_fast_path(self):
+        called = []
+        
+        class M(type):
+            def __instancecheck__(self, obj):
+                saddsadsa
+                called.append("called")
+
+        class C:
+            __metaclass__ = M
+
+        c = C()
+        assert isinstance(c, C)
+        assert not called
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -521,6 +521,8 @@
         return space.get_and_call_function(w_check, w_type, w_sub)
 
     def isinstance_allow_override(space, w_inst, w_type):
+        if space.type(w_inst) is w_type:
+            return space.w_True # fast path copied from cpython
         w_check = space.lookup(w_type, "__instancecheck__")
         if w_check is not None:
             return space.get_and_call_function(w_check, w_type, w_inst)

From noreply at buildbot.pypy.org  Thu Jul 23 17:14:14 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 23 Jul 2015 17:14:14 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: fix: hint_commit_soon can now
 malloc and break a transaction itself
Message-ID: <20150723151414.D86A91C0433@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78638:75d83967e132
Date: 2015-07-23 17:15 +0200
http://bitbucket.org/pypy/pypy/changeset/75d83967e132/

Log:	fix: hint_commit_soon can now malloc and break a transaction itself

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -24,6 +24,8 @@
             return
         # ----------  transaction breaks  ----------
         if opnum == rop.STM_HINT_COMMIT_SOON:
+            self.emitting_an_operation_that_can_collect()
+            self.next_op_may_be_in_new_transaction()
             self._do_stm_call('stm_hint_commit_soon', [], None)
             return
         # ----------  jump, finish, guard_not_forced_2  ----------
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -88,9 +88,14 @@
     def check_rewrite(self, frm_operations, to_operations, **namespace):
         inev = ("call(ConstClass(stm_try_inevitable),"
                 " descr=stm_try_inevitable_descr)")
+        hcs  = ("call(ConstClass(stm_hint_commit_soon),"
+                " descr=stm_hint_commit_soon_descr)")
+
         dummyalloc = "p999 = call_malloc_nursery(16)"
         frm_operations = frm_operations.replace('$INEV', inev)
+        frm_operations = frm_operations.replace('$HCS', hcs)
         to_operations  = to_operations .replace('$INEV', inev)
+        to_operations  = to_operations .replace('$HCS', hcs)
         to_operations  = to_operations .replace('$DUMMYALLOC', dummyalloc)
         for name, value in self.gc_ll_descr.__dict__.items():
             if name.endswith('descr') and name[1] == '2' and len(name) == 8:
@@ -705,6 +710,39 @@
             jump()
         """)
 
+    def test_hint_commit_soon(self):
+        T = rffi.CArrayPtr(rffi.TIME_T)
+        calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T)
+        self.check_rewrite("""
+            [i2, p7, p1]
+            i1 = getfield_gc(p1, descr=tydescr) # noptr
+            setfield_gc(p7, 10, descr=tydescr) #noptr
+            stm_hint_commit_soon()
+            setfield_gc(p7, 20, descr=tydescr) #noptr
+            i3 = getfield_gc(p1, descr=tydescr) # noptr
+            jump(i2, p7, i1)
+        """, """
+            [i2, p7, p1]
+            i1 = getfield_gc(p1, descr=tydescr)
+            stm_read(p1)
+
+            cond_call_gc_wb(p7, descr=wbdescr)
+            setfield_gc(p7, 10, descr=tydescr)
+
+            $HCS
+
+            cond_call_gc_wb(p7, descr=wbdescr)
+            setfield_gc(p7, 20, descr=tydescr)
+
+            i3 = getfield_gc(p1, descr=tydescr) # noptr
+            stm_read(p1)
+
+            $DUMMYALLOC
+            jump(i2, p7, i1)
+        """, calldescr2=calldescr2)
+
+
+
     def test_call_release_gil(self):
         T = rffi.CArrayPtr(rffi.TIME_T)
         calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T)
diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -231,6 +231,7 @@
 
     # sync with lloperation.py
     gct_stm_become_inevitable                       = _gct_with_roots_pushed
+    gct_stm_hint_commit_soon                        = _gct_with_roots_pushed
     gct_stm_stop_all_other_threads                  = _gct_with_roots_pushed
     gct_stm_transaction_break                       = _gct_with_roots_pushed
     gct_stm_collect                                 = _gct_with_roots_pushed
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -449,7 +449,10 @@
     # stm_memclearinit(gcref, offset, size) clears the memory in this address space
     'stm_memclearinit': LLOp(),
 
-    'stm_hint_commit_soon':   LLOp(canrun=True),
+    # if it stays that way, should be renamed to something like
+    # try_break_transaction(). (before, it would also work in an
+    # atomic transaction)
+    'stm_hint_commit_soon':   LLOp(canrun=True, canmallocgc=True),
 
     'stm_increment_atomic':   LLOp(),
     'stm_decrement_atomic':   LLOp(),
diff --git a/rpython/translator/stm/breakfinder.py b/rpython/translator/stm/breakfinder.py
--- a/rpython/translator/stm/breakfinder.py
+++ b/rpython/translator/stm/breakfinder.py
@@ -5,6 +5,7 @@
 TRANSACTION_BREAK = set([
     'stm_enter_transactional_zone',
     'stm_leave_transactional_zone',
+    'stm_hint_commit_soon',
     #'jit_assembler_call',
     'stm_enter_callback_call',
     'stm_leave_callback_call',

From noreply at buildbot.pypy.org  Thu Jul 23 20:42:01 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 23 Jul 2015 20:42:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix failing bitshift test on 32 bit
 (was converting long to int32), cleanup
Message-ID: <20150723184201.1F39A1C0695@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78639:cb76e099cd71
Date: 2015-07-23 21:40 +0300
http://bitbucket.org/pypy/pypy/changeset/cb76e099cd71/

Log:	fix failing bitshift test on 32 bit (was converting long to int32),
	cleanup

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -710,10 +710,14 @@
                           ((w_arg1.is_scalar() and not w_arg2.is_scalar()) or
                            (not w_arg1.is_scalar() and w_arg2.is_scalar())))
         in_casting = safe_casting_mode(casting)
+        if use_min_scalar:
+            w_arg1 = convert_to_array(space, w_arg1)
+            w_arg2 = convert_to_array(space, w_arg2)
+        elif in_casting == 'safe' and l_dtype.num == 7 and r_dtype.num == 7:
+            # while long (7) can be cast to int32 (5) on 32 bit, don't do it
+            return l_dtype, l_dtype
         for dt_in, dt_out in self.dtypes:
             if use_min_scalar:
-                w_arg1 = convert_to_array(space, w_arg1)
-                w_arg2 = convert_to_array(space, w_arg2)
                 if not (can_cast_array(space, w_arg1, dt_in, in_casting) and
                         can_cast_array(space, w_arg2, dt_in, in_casting)):
                     continue

From noreply at buildbot.pypy.org  Thu Jul 23 21:51:29 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 23 Jul 2015 21:51:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: (arigo,
	fijal) hack to support prebuilt string builders
Message-ID: <20150723195129.EFF231C037F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78640:993ddbf39539
Date: 2015-07-23 21:51 +0200
http://bitbucket.org/pypy/pypy/changeset/993ddbf39539/

Log:	(arigo, fijal) hack to support prebuilt string builders

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -494,18 +494,22 @@
     # This is not the real implementation!
 
     def __init__(self, init_size=INIT_SIZE):
+        "NOT_RPYTHON"
         self._l = []
         self._size = 0
 
     def _grow(self, size):
+        "NOT_RPYTHON"
         self._size += size
 
     def append(self, s):
+        "NOT_RPYTHON"
         assert isinstance(s, self._tp)
         self._l.append(s)
         self._grow(len(s))
 
     def append_slice(self, s, start, end):
+        "NOT_RPYTHON"
         assert isinstance(s, self._tp)
         assert 0 <= start <= end <= len(s)
         s = s[start:end]
@@ -513,11 +517,13 @@
         self._grow(len(s))
 
     def append_multiple_char(self, c, times):
+        "NOT_RPYTHON"
         assert isinstance(c, self._tp)
         self._l.append(c * times)
         self._grow(times)
 
     def append_charpsize(self, s, size):
+        "NOT_RPYTHON"
         assert size >= 0
         l = []
         for i in xrange(size):
@@ -526,12 +532,14 @@
         self._grow(size)
 
     def build(self):
+        "NOT_RPYTHON"
         result = self._tp("").join(self._l)
         assert len(result) == self._size
         self._l = [result]
         return result
 
     def getlength(self):
+        "NOT_RPYTHON"
         return self._size
 
 
@@ -672,11 +680,22 @@
     _about_ = StringBuilder
     use_unicode = False
 
-
 class UnicodeBuilderEntry(BaseEntry, ExtRegistryEntry):
     _about_ = UnicodeBuilder
     use_unicode = True
 
+class PrebuiltStringBuilderEntry(ExtRegistryEntry):
+    _type_ = StringBuilder
+
+    def compute_annotation(self):
+        return SomeStringBuilder()
+
+class PrebuiltUnicodeBuilderEntry(ExtRegistryEntry):
+    _type_ = UnicodeBuilder
+
+    def compute_annotation(self):
+        return SomeUnicodeBuilder()
+
 
 #___________________________________________________________________
 # Support functions for SomeString.no_nul
diff --git a/rpython/rtyper/lltypesystem/rbuilder.py b/rpython/rtyper/lltypesystem/rbuilder.py
--- a/rpython/rtyper/lltypesystem/rbuilder.py
+++ b/rpython/rtyper/lltypesystem/rbuilder.py
@@ -10,6 +10,8 @@
     string_repr, unichar_repr, unicode_repr)
 from rpython.rtyper.rbuilder import AbstractStringBuilderRepr
 from rpython.tool.sourcetools import func_with_new_name
+from rpython.rtyper.annlowlevel import llstr, llunicode
+        
 
 
 # ------------------------------------------------------------
@@ -413,6 +415,7 @@
 class StringBuilderRepr(BaseStringBuilderRepr):
     lowleveltype = lltype.Ptr(STRINGBUILDER)
     basetp = STR
+    convert_to_ll = staticmethod(llstr)
     string_repr = string_repr
     char_repr = char_repr
     raw_ptr_repr = PtrRepr(
@@ -435,6 +438,7 @@
 class UnicodeBuilderRepr(BaseStringBuilderRepr):
     lowleveltype = lltype.Ptr(UNICODEBUILDER)
     basetp = UNICODE
+    convert_to_ll = staticmethod(llunicode)
     string_repr = unicode_repr
     char_repr = unichar_repr
     raw_ptr_repr = PtrRepr(
diff --git a/rpython/rtyper/rbuilder.py b/rpython/rtyper/rbuilder.py
--- a/rpython/rtyper/rbuilder.py
+++ b/rpython/rtyper/rbuilder.py
@@ -55,6 +55,9 @@
         return hop.gendirectcall(self.ll_bool, *vlist)
 
     def convert_const(self, value):
-        if not value is None:
-            raise TypeError("Prebuilt builedrs that are not none unsupported")
-        return self.empty()
+        if value is None:
+            return self.empty()
+        s = value.build()
+        ll_obj = self.ll_new(len(s))
+        self.ll_append(ll_obj, self.convert_to_ll(s))
+        return ll_obj
diff --git a/rpython/rtyper/test/test_rbuilder.py b/rpython/rtyper/test/test_rbuilder.py
--- a/rpython/rtyper/test/test_rbuilder.py
+++ b/rpython/rtyper/test/test_rbuilder.py
@@ -194,3 +194,23 @@
         assert not res
         res = self.interpret(func, [1])
         assert res
+
+    def test_prebuilt_string_builder(self):
+        s = StringBuilder(100)
+        s.append("abc")
+        
+        def f():
+            return len(s.build())
+
+        res = self.interpret(f, [])
+        assert res == 3
+
+    def test_prebuilt_unicode_builder(self):
+        s = UnicodeBuilder(100)
+        s.append(u"abc")
+        
+        def f():
+            return len(s.build())
+
+        res = self.interpret(f, [])
+        assert res == 3

From noreply at buildbot.pypy.org  Thu Jul 23 21:51:31 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 23 Jul 2015 21:51:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150723195131.34D871C037F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78641:90275c8df29a
Date: 2015-07-23 21:51 +0200
http://bitbucket.org/pypy/pypy/changeset/90275c8df29a/

Log:	merge

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -710,10 +710,14 @@
                           ((w_arg1.is_scalar() and not w_arg2.is_scalar()) or
                            (not w_arg1.is_scalar() and w_arg2.is_scalar())))
         in_casting = safe_casting_mode(casting)
+        if use_min_scalar:
+            w_arg1 = convert_to_array(space, w_arg1)
+            w_arg2 = convert_to_array(space, w_arg2)
+        elif in_casting == 'safe' and l_dtype.num == 7 and r_dtype.num == 7:
+            # while long (7) can be cast to int32 (5) on 32 bit, don't do it
+            return l_dtype, l_dtype
         for dt_in, dt_out in self.dtypes:
             if use_min_scalar:
-                w_arg1 = convert_to_array(space, w_arg1)
-                w_arg2 = convert_to_array(space, w_arg2)
                 if not (can_cast_array(space, w_arg1, dt_in, in_casting) and
                         can_cast_array(space, w_arg2, dt_in, in_casting)):
                     continue

From noreply at buildbot.pypy.org  Thu Jul 23 22:06:53 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 23 Jul 2015 22:06:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix union
Message-ID: <20150723200653.326221C1035@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78642:df782d94c19b
Date: 2015-07-23 22:07 +0200
http://bitbucket.org/pypy/pypy/changeset/df782d94c19b/

Log:	fix union

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -684,6 +684,16 @@
     _about_ = UnicodeBuilder
     use_unicode = True
 
+class __extend__(pairtype(SomeStringBuilder, SomeStringBuilder)):
+
+    def union((obj1, obj2)):
+        return obj1
+
+class __extend__(pairtype(SomeUnicodeBuilder, SomeUnicodeBuilder)):
+
+    def union((obj1, obj2)):
+        return obj1
+
 class PrebuiltStringBuilderEntry(ExtRegistryEntry):
     _type_ = StringBuilder
 
diff --git a/rpython/rtyper/test/test_rbuilder.py b/rpython/rtyper/test/test_rbuilder.py
--- a/rpython/rtyper/test/test_rbuilder.py
+++ b/rpython/rtyper/test/test_rbuilder.py
@@ -214,3 +214,15 @@
 
         res = self.interpret(f, [])
         assert res == 3
+
+    def test_string_builder_union(self):
+        s = StringBuilder()
+
+        def f(i):
+            if i % 2:
+                s2 = StringBuilder()
+            else:
+                s2 = s
+            return s2.build()
+
+        self.interpret(f, [3])

From noreply at buildbot.pypy.org  Fri Jul 24 02:58:58 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 24 Jul 2015 02:58:58 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: Replace 'cumulative' flag with
 'variant' enum in reduce()
Message-ID: <20150724005858.34E231C034E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78643:5cb612e5ab3a
Date: 2015-07-23 19:59 +0100
http://bitbucket.org/pypy/pypy/changeset/5cb612e5ab3a/

Log:	Replace 'cumulative' flag with 'variant' enum in reduce()

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1149,7 +1149,7 @@
 
     # ----------------------- reduce -------------------------------
 
-    def _reduce_ufunc_impl(ufunc_name, cumulative=False, bool_result=False):
+    def _reduce_ufunc_impl(ufunc_name, name, variant=ufuncs.REDUCE, bool_result=False):
         @unwrap_spec(keepdims=bool)
         def impl(self, space, w_axis=None, w_dtype=None, w_out=None, keepdims=False):
             if space.is_none(w_out):
@@ -1161,18 +1161,19 @@
             if bool_result:
                 w_dtype = descriptor.get_dtype_cache(space).w_booldtype
             return getattr(ufuncs.get(space), ufunc_name).reduce(
-                space, self, w_axis, keepdims, out, w_dtype, cumulative=cumulative)
-        return func_with_new_name(impl, "reduce_%s_impl_%d" % (ufunc_name, cumulative))
+                space, self, w_axis, keepdims, out, w_dtype, variant=variant)
+        impl.__name__ = name
+        return impl
 
-    descr_sum = _reduce_ufunc_impl("add")
-    descr_prod = _reduce_ufunc_impl("multiply")
-    descr_max = _reduce_ufunc_impl("maximum")
-    descr_min = _reduce_ufunc_impl("minimum")
-    descr_all = _reduce_ufunc_impl('logical_and', bool_result=True)
-    descr_any = _reduce_ufunc_impl('logical_or', bool_result=True)
+    descr_sum = _reduce_ufunc_impl("add", "descr_sum")
+    descr_prod = _reduce_ufunc_impl("multiply", "descr_prod")
+    descr_max = _reduce_ufunc_impl("maximum", "descr_max")
+    descr_min = _reduce_ufunc_impl("minimum", "descr_min")
+    descr_all = _reduce_ufunc_impl('logical_and', "descr_all", bool_result=True)
+    descr_any = _reduce_ufunc_impl('logical_or', "descr_any", bool_result=True)
 
-    descr_cumsum = _reduce_ufunc_impl('add', cumulative=True)
-    descr_cumprod = _reduce_ufunc_impl('multiply', cumulative=True)
+    descr_cumsum = _reduce_ufunc_impl('add', "descr_cumsum", variant=ufuncs.ACCUMULATE)
+    descr_cumprod = _reduce_ufunc_impl('multiply', "descr_cumprod", variant=ufuncs.ACCUMULATE)
 
     def _reduce_argmax_argmin_impl(raw_name):
         op_name = "arg%s" % raw_name
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -25,6 +25,8 @@
     find_result_type, promote_types)
 from .boxes import W_GenericBox, W_ObjectBox
 
+REDUCE, ACCUMULATE, REDUCEAT = range(3)
+
 def done_if_true(dtype, val):
     return dtype.itemtype.bool(val)
 
@@ -171,7 +173,7 @@
         else:
             out = w_out
         return self.reduce(space, w_obj, w_axis, True, #keepdims must be true
-                           out, w_dtype, cumulative=True)
+                           out, w_dtype, variant=ACCUMULATE)
 
     @unwrap_spec(keepdims=bool)
     def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
@@ -241,7 +243,7 @@
         return self.reduce(space, w_obj, w_axis, keepdims, out, w_dtype)
 
     def reduce(self, space, w_obj, w_axis, keepdims=False, out=None, dtype=None,
-               cumulative=False):
+               variant=REDUCE):
         if self.nin != 2:
             raise oefmt(space.w_ValueError,
                         "reduce only supported for binary functions")
@@ -292,7 +294,7 @@
                             "zero-size array to reduction operation %s "
                             "which has no identity", self.name)
 
-        if cumulative:
+        if variant == ACCUMULATE:
             dtype = self.find_binop_type(space, dtype)
         else:
             _, dtype, _ = self.find_specialization(space, dtype, dtype, out,
@@ -300,7 +302,7 @@
         call__array_wrap__ = True
         if shapelen > 1 and axis < shapelen:
             temp = None
-            if cumulative:
+            if variant == ACCUMULATE:
                 shape = obj_shape[:]
                 temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]
                 if out:
@@ -339,13 +341,13 @@
                 if self.identity is not None:
                     out.fill(space, self.identity.convert_to(space, dtype))
                 return out
-            loop.do_axis_reduce(space, shape, self.func, obj, dtype,
-                                       axis, out, self.identity, cumulative,
-                                       temp)
+            loop.do_axis_reduce(space, shape, self.func, obj, dtype, axis,
+                                out, self.identity, (variant == ACCUMULATE),
+                                temp)
             if call__array_wrap__:
                 out = space.call_method(obj, '__array_wrap__', out)
             return out
-        if cumulative:
+        if variant == ACCUMULATE:
             if out:
                 call__array_wrap__ = False
                 if out.get_shape() != [obj.get_size()]:
@@ -786,7 +788,7 @@
         self.external_loop = external_loop
 
     def reduce(self, space, w_obj, w_axis, keepdims=False, out=None, dtype=None,
-               cumulative=False):
+               variant=REDUCE):
         raise oefmt(space.w_NotImplementedError, 'not implemented yet')
 
     def call(self, space, args_w, sig, casting, extobj):

From noreply at buildbot.pypy.org  Fri Jul 24 02:58:59 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 24 Jul 2015 02:58:59 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: specialize W_Ufunc.reduce()
Message-ID: <20150724005859.634761C034E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78644:a3dc2396dd37
Date: 2015-07-24 01:53 +0100
http://bitbucket.org/pypy/pypy/changeset/a3dc2396dd37/

Log:	specialize W_Ufunc.reduce()

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -9,7 +9,7 @@
 from rpython.rlib.rawstorage import (
     raw_storage_setitem, free_raw_storage, alloc_raw_storage)
 from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here, specialize
 
 from pypy.module.micronumpy import loop, constants as NPY
 from pypy.module.micronumpy.descriptor import (
@@ -26,6 +26,7 @@
 from .boxes import W_GenericBox, W_ObjectBox
 
 REDUCE, ACCUMULATE, REDUCEAT = range(3)
+_reduce_type = ["reduce", "acccumulate", "reduceat"]
 
 def done_if_true(dtype, val):
     return dtype.itemtype.bool(val)
@@ -242,16 +243,19 @@
             out = w_out
         return self.reduce(space, w_obj, w_axis, keepdims, out, w_dtype)
 
+    @specialize.arg(7)
     def reduce(self, space, w_obj, w_axis, keepdims=False, out=None, dtype=None,
                variant=REDUCE):
         if self.nin != 2:
             raise oefmt(space.w_ValueError,
-                        "reduce only supported for binary functions")
+                        "%s only supported for binary functions",
+                        _reduce_type[variant])
         assert isinstance(self, W_Ufunc2)
         obj = convert_to_array(space, w_obj)
         if obj.get_dtype().is_flexible():
             raise oefmt(space.w_TypeError,
-                        "cannot perform reduce with flexible type")
+                        "cannot perform %s with flexible type",
+                        _reduce_type[variant])
         obj_shape = obj.get_shape()
         if obj.is_scalar():
             return obj.get_scalar_value()

From noreply at buildbot.pypy.org  Fri Jul 24 03:48:44 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 24 Jul 2015 03:48:44 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: Split off loop.py implementations
 of reduce() and accumulate() into separate functions
Message-ID: <20150724014844.9257F1C034E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78645:67853b0b87af
Date: 2015-07-24 02:21 +0100
http://bitbucket.org/pypy/pypy/changeset/67853b0b87af/

Log:	Split off loop.py implementations of reduce() and accumulate() into
	separate functions

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -242,6 +242,40 @@
         out_state = out_iter.next(out_state)
         obj_state = obj_iter.next(obj_state)
 
+accumulate_driver = jit.JitDriver(name='numpy_accumulate_flat',
+                                  greens=['shapelen', 'func', 'dtype'],
+                                  reds='auto')
+
+def do_accumulate(space, shape, func, arr, dtype, axis, out, identity, temp):
+    out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative=True)
+    out_state = out_iter.reset()
+    temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis, False)
+    temp_state = temp_iter.reset()
+    arr_iter, arr_state = arr.create_iter()
+    arr_iter.track_index = False
+    if identity is not None:
+        identity = identity.convert_to(space, dtype)
+    shapelen = len(shape)
+    while not out_iter.done(out_state):
+        accumulate_driver.jit_merge_point(shapelen=shapelen, func=func,
+                                          dtype=dtype)
+        w_val = arr_iter.getitem(arr_state).convert_to(space, dtype)
+        arr_state = arr_iter.next(arr_state)
+
+        out_indices = out_iter.indices(out_state)
+        if out_indices[axis] == 0:
+            if identity is not None:
+                w_val = func(dtype, identity, w_val)
+        else:
+            cur = temp_iter.getitem(temp_state)
+            w_val = func(dtype, cur, w_val)
+
+        out_iter.setitem(out_state, w_val)
+        out_state = out_iter.next(out_state)
+        temp_iter.setitem(temp_state, w_val)
+        temp_state = temp_iter.next(temp_state)
+    return out
+
 def fill(arr, box):
     arr_iter, arr_state = arr.create_iter()
     while not arr_iter.done(arr_state):
@@ -302,16 +336,9 @@
                                    greens=['shapelen', 'func', 'dtype'],
                                    reds='auto')
 
-def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity, cumulative,
-                   temp):
-    out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative)
+def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity):
+    out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative=False)
     out_state = out_iter.reset()
-    if cumulative:
-        temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis, False)
-        temp_state = temp_iter.reset()
-    else:
-        temp_iter = out_iter  # hack
-        temp_state = out_state
     arr_iter, arr_state = arr.create_iter()
     arr_iter.track_index = False
     if identity is not None:
@@ -328,16 +355,11 @@
             if identity is not None:
                 w_val = func(dtype, identity, w_val)
         else:
-            cur = temp_iter.getitem(temp_state)
+            cur = out_iter.getitem(out_state)
             w_val = func(dtype, cur, w_val)
 
         out_iter.setitem(out_state, w_val)
         out_state = out_iter.next(out_state)
-        if cumulative:
-            temp_iter.setitem(temp_state, w_val)
-            temp_state = temp_iter.next(temp_state)
-        else:
-            temp_state = out_state
     return out
 
 
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -345,9 +345,12 @@
                 if self.identity is not None:
                     out.fill(space, self.identity.convert_to(space, dtype))
                 return out
-            loop.do_axis_reduce(space, shape, self.func, obj, dtype, axis,
-                                out, self.identity, (variant == ACCUMULATE),
-                                temp)
+            if variant == REDUCE:
+                loop.do_axis_reduce(space, shape, self.func, obj, dtype, axis,
+                                    out, self.identity)
+            elif variant == ACCUMULATE:
+                loop.do_accumulate(space, shape, self.func, obj, dtype, axis,
+                                    out, self.identity, temp)
             if call__array_wrap__:
                 out = space.call_method(obj, '__array_wrap__', out)
             return out

From noreply at buildbot.pypy.org  Fri Jul 24 04:24:36 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 24 Jul 2015 04:24:36 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: Split accumulate and reduce code
 paths in W_Ufunc.reduce()
Message-ID: <20150724022436.BE4BA1C037F@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78646:4db3dee7c611
Date: 2015-07-24 03:24 +0100
http://bitbucket.org/pypy/pypy/changeset/4db3dee7c611/

Log:	Split accumulate and reduce code paths in W_Ufunc.reduce()

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -300,20 +300,66 @@
 
         if variant == ACCUMULATE:
             dtype = self.find_binop_type(space, dtype)
-        else:
-            _, dtype, _ = self.find_specialization(space, dtype, dtype, out,
-                                                   casting='unsafe')
-        call__array_wrap__ = True
-        if shapelen > 1 and axis < shapelen:
-            temp = None
-            if variant == ACCUMULATE:
+            call__array_wrap__ = True
+            if shapelen > 1 and axis < shapelen:
+                temp = None
                 shape = obj_shape[:]
                 temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]
                 if out:
                     dtype = out.get_dtype()
                 temp = W_NDimArray.from_shape(space, temp_shape, dtype,
-                                              w_instance=obj)
-            elif keepdims:
+                                            w_instance=obj)
+                if out:
+                    # Test for shape agreement
+                    # XXX maybe we need to do broadcasting here, although I must
+                    #     say I don't understand the details for axis reduce
+                    if out.ndims() > len(shape):
+                        raise oefmt(space.w_ValueError,
+                                    "output parameter for reduction operation %s "
+                                    "has too many dimensions", self.name)
+                    elif out.ndims() < len(shape):
+                        raise oefmt(space.w_ValueError,
+                                    "output parameter for reduction operation %s "
+                                    "does not have enough dimensions", self.name)
+                    elif out.get_shape() != shape:
+                        raise oefmt(space.w_ValueError,
+                                    "output parameter shape mismatch, expecting "
+                                    "[%s], got [%s]",
+                                    ",".join([str(x) for x in shape]),
+                                    ",".join([str(x) for x in out.get_shape()]),
+                                    )
+                    call__array_wrap__ = False
+                    dtype = out.get_dtype()
+                else:
+                    out = W_NDimArray.from_shape(space, shape, dtype,
+                                                w_instance=obj)
+                if obj.get_size() == 0:
+                    if self.identity is not None:
+                        out.fill(space, self.identity.convert_to(space, dtype))
+                    return out
+                loop.do_accumulate(space, shape, self.func, obj, dtype, axis,
+                                    out, self.identity, temp)
+            else:
+                if out:
+                    call__array_wrap__ = False
+                    if out.get_shape() != [obj.get_size()]:
+                        raise OperationError(space.w_ValueError, space.wrap(
+                            "out of incompatible size"))
+                else:
+                    out = W_NDimArray.from_shape(space, [obj.get_size()], dtype,
+                                                w_instance=obj)
+                loop.compute_reduce_cumulative(space, obj, out, dtype, self.func,
+                                            self.identity)
+            if call__array_wrap__:
+                out = space.call_method(obj, '__array_wrap__', out)
+            return out
+
+        _, dtype, _ = self.find_specialization(space, dtype, dtype, out,
+                                                   casting='unsafe')
+        call__array_wrap__ = True
+        if shapelen > 1 and axis < shapelen:
+            temp = None
+            if keepdims:
                 shape = obj_shape[:axis] + [1] + obj_shape[axis + 1:]
             else:
                 shape = obj_shape[:axis] + obj_shape[axis + 1:]
@@ -345,26 +391,8 @@
                 if self.identity is not None:
                     out.fill(space, self.identity.convert_to(space, dtype))
                 return out
-            if variant == REDUCE:
-                loop.do_axis_reduce(space, shape, self.func, obj, dtype, axis,
-                                    out, self.identity)
-            elif variant == ACCUMULATE:
-                loop.do_accumulate(space, shape, self.func, obj, dtype, axis,
-                                    out, self.identity, temp)
-            if call__array_wrap__:
-                out = space.call_method(obj, '__array_wrap__', out)
-            return out
-        if variant == ACCUMULATE:
-            if out:
-                call__array_wrap__ = False
-                if out.get_shape() != [obj.get_size()]:
-                    raise OperationError(space.w_ValueError, space.wrap(
-                        "out of incompatible size"))
-            else:
-                out = W_NDimArray.from_shape(space, [obj.get_size()], dtype,
-                                             w_instance=obj)
-            loop.compute_reduce_cumulative(space, obj, out, dtype, self.func,
-                                           self.identity)
+            loop.do_axis_reduce(space, shape, self.func, obj, dtype, axis,
+                                out, self.identity)
             if call__array_wrap__:
                 out = space.call_method(obj, '__array_wrap__', out)
             return out

From noreply at buildbot.pypy.org  Fri Jul 24 08:43:21 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 24 Jul 2015 08:43:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: more conditions for fix in cb76e099cd71
Message-ID: <20150724064321.7C59B1C0433@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78647:18a12b7bba21
Date: 2015-07-24 08:55 +0300
http://bitbucket.org/pypy/pypy/changeset/18a12b7bba21/

Log:	more conditions for fix in cb76e099cd71

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -713,7 +713,8 @@
         if use_min_scalar:
             w_arg1 = convert_to_array(space, w_arg1)
             w_arg2 = convert_to_array(space, w_arg2)
-        elif in_casting == 'safe' and l_dtype.num == 7 and r_dtype.num == 7:
+        elif (in_casting == 'safe' and l_dtype.num == 7 and r_dtype.num == 7 and
+              out is None and not self.promote_to_float):
             # while long (7) can be cast to int32 (5) on 32 bit, don't do it
             return l_dtype, l_dtype
         for dt_in, dt_out in self.dtypes:

From noreply at buildbot.pypy.org  Fri Jul 24 08:43:22 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 24 Jul 2015 08:43:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: do not raise exceptions in __init__
Message-ID: <20150724064322.A7D381C0433@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78648:614e58595142
Date: 2015-07-24 09:43 +0300
http://bitbucket.org/pypy/pypy/changeset/614e58595142/

Log:	do not raise exceptions in __init__

diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -2,6 +2,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.tool.pairtype import extendabletype
 from pypy.module.micronumpy import support
+from pypy.module.micronumpy import constants as NPY
 
 def wrap_impl(space, w_cls, w_instance, impl):
     if w_cls is None or space.is_w(w_cls, space.gettypefor(W_NDimArray)):
@@ -39,6 +40,13 @@
     def from_shape(space, shape, dtype, order='C', w_instance=None, zero=True):
         from pypy.module.micronumpy import concrete, descriptor, boxes
         from pypy.module.micronumpy.strides import calc_strides
+        if len(shape) > NPY.MAXDIMS:
+            raise oefmt(space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        try:
+            support.product(shape) * dtype.elsize
+        except OverflowError as e:
+            raise oefmt(space.w_ValueError, "array is too big")
         strides, backstrides = calc_strides(shape, dtype.base, order)
         impl = concrete.ConcreteArray(shape, dtype.base, order, strides,
                                       backstrides, zero=zero)
@@ -56,13 +64,19 @@
         from pypy.module.micronumpy.strides import (calc_strides,
                                                     calc_backstrides)
         isize = dtype.elsize
+        if len(shape) > NPY.MAXDIMS:
+            raise oefmt(space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        try:
+            totalsize = support.product(shape) * isize
+        except OverflowError as e:
+            raise oefmt(space.w_ValueError, "array is too big")
         if storage_bytes > 0 :
-            totalsize = support.product(shape) * isize
             if totalsize > storage_bytes:
                 raise OperationError(space.w_TypeError, space.wrap(
                     "buffer is too small for requested array"))
         else:
-            storage_bytes = support.product(shape) * isize
+            storage_bytes = totalsize
         if strides is None:
             strides, backstrides = calc_strides(shape, dtype, order)
         else:
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -400,17 +400,11 @@
 
 class ConcreteArrayNotOwning(BaseConcreteArray):
     def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0):
-        if len(shape) > NPY.MAXDIMS:
-            raise oefmt(dtype.itemtype.space.w_ValueError,
-                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
         make_sure_not_resized(shape)
         make_sure_not_resized(strides)
         make_sure_not_resized(backstrides)
         self.shape = shape
-        try:
-            self.size = support.product(shape) * dtype.elsize
-        except OverflowError as e:
-            raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big")
+        self.size = support.product(shape) * dtype.elsize
         self.order = order
         self.dtype = dtype
         self.strides = strides
@@ -425,6 +419,13 @@
             box, 0, self.size, 0, self.gcstruct)
 
     def set_shape(self, space, orig_array, new_shape):
+        if len(new_shape) > NPY.MAXDIMS:
+            raise oefmt(space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        try:
+            support.product(new_shape) * self.dtype.elsize
+        except OverflowError as e:
+            raise oefmt(space.w_ValueError, "array is too big")
         strides, backstrides = calc_strides(new_shape, self.dtype,
                                                     self.order)
         return SliceArray(self.start, strides, backstrides, new_shape, self,
@@ -451,14 +452,9 @@
                  storage=lltype.nullptr(RAW_STORAGE), zero=True):
         gcstruct = V_OBJECTSTORE
         flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
-        if len(shape) > NPY.MAXDIMS:
-            raise oefmt(dtype.itemtype.space.w_ValueError,
-                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        length = support.product(shape)
+        self.size = length * dtype.elsize
         if storage == lltype.nullptr(RAW_STORAGE):
-            try:
-                length = support.product(shape)
-            except OverflowError as e:
-                raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big")
             if dtype.num == NPY.OBJECT:
                 storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True)
                 gcstruct = _create_objectstore(storage, length, dtype.elsize)
@@ -559,6 +555,13 @@
         loop.fill(self, box.convert_to(space, self.dtype))
 
     def set_shape(self, space, orig_array, new_shape):
+        if len(new_shape) > NPY.MAXDIMS:
+            raise oefmt(space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        try:
+            support.product(new_shape) * self.dtype.elsize
+        except OverflowError as e:
+            raise oefmt(space.w_ValueError, "array is too big")
         if len(self.get_shape()) < 2 or self.size == 0:
             # TODO: this code could be refactored into calc_strides
             # but then calc_strides would have to accept a stepping factor
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1335,7 +1335,9 @@
     dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
         space.gettypefor(descriptor.W_Dtype), w_dtype))
     shape = shape_converter(space, w_shape, dtype)
-
+    if len(shape) > NPY.MAXDIMS:
+        raise oefmt(space.w_ValueError,
+            "sequence too large; must be smaller than %d", NPY.MAXDIMS)
     if not space.is_none(w_buffer):
         if (not space.is_none(w_strides)):
             strides = [space.int_w(w_i) for w_i in
@@ -1372,6 +1374,10 @@
     if space.is_w(w_subtype, space.gettypefor(W_NDimArray)):
         return W_NDimArray.from_shape(space, shape, dtype, order)
     strides, backstrides = calc_strides(shape, dtype.base, order)
+    try:
+        totalsize = support.product(shape) * dtype.base.elsize
+    except OverflowError as e:
+        raise oefmt(space.w_ValueError, "array is too big")
     impl = ConcreteArray(shape, dtype.base, order, strides, backstrides)
     w_ret = space.allocate_instance(W_NDimArray, w_subtype)
     W_NDimArray.__init__(w_ret, impl)

From noreply at buildbot.pypy.org  Fri Jul 24 11:11:21 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 24 Jul 2015 11:11:21 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: add stm_leave_transactional_zone_final
 for use before freeing thread resources
Message-ID: <20150724091121.BB25A1C0433@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: use-gcc
Changeset: r1896:c0e7e64a894d
Date: 2015-07-24 11:13 +0200
http://bitbucket.org/pypy/stmgc/changeset/c0e7e64a894d/

Log:	add stm_leave_transactional_zone_final for use before freeing thread
	resources

diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -14,7 +14,8 @@
         pthread_mutex_t global_mutex;
         pthread_cond_t cond[_C_TOTAL];
         /* some additional pieces of global state follow */
-        uint8_t in_use1[NB_SEGMENTS];   /* 1 if running a pthread, idx=0 unused */
+        uint8_t in_use1[NB_SEGMENTS];   /* 1 if running a pthread, idx=0 unused,
+                                           2 if soon_finished_or_inevitable_thread_segment */
     };
     char reserved[192];
 } sync_ctl __attribute__((aligned(64)));
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -437,6 +437,10 @@
 
    stm_enter_transactional_zone() and stm_leave_transactional_zone()
    preserve the value of errno.
+
+   stm_leave_transactional_zone_final() commits the transaction
+   unconditionally and allows the caller to teardown the whole
+   thread (unregister thread local, etc.).
 */
 #ifdef STM_DEBUGPRINT
 #include <stdio.h>
@@ -468,6 +472,11 @@
         _stm_leave_noninevitable_transactional_zone();
     }
 }
+static inline void stm_leave_transactional_zone_final(stm_thread_local_t *tl) {
+    assert(STM_SEGMENT->running_thread == tl);
+    _stm_commit_transaction();
+}
+
 
 /* stm_force_transaction_break() is in theory equivalent to
    stm_leave_transactional_zone() immediately followed by

From noreply at buildbot.pypy.org  Fri Jul 24 11:14:41 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 24 Jul 2015 11:14:41 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: import stmgc and hopefully fix the
 issue of tearing down a thread before really committing the running
 transaction
Message-ID: <20150724091441.B30C01C0433@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78649:188629c10d4a
Date: 2015-07-24 11:16 +0200
http://bitbucket.org/pypy/pypy/changeset/188629c10d4a/

Log:	import stmgc and hopefully fix the issue of tearing down a thread
	before really committing the running transaction

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-088f807586c2
+a89f21f5670b+
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -14,7 +14,8 @@
         pthread_mutex_t global_mutex;
         pthread_cond_t cond[_C_TOTAL];
         /* some additional pieces of global state follow */
-        uint8_t in_use1[NB_SEGMENTS];   /* 1 if running a pthread, idx=0 unused */
+        uint8_t in_use1[NB_SEGMENTS];   /* 1 if running a pthread, idx=0 unused,
+                                           2 if soon_finished_or_inevitable_thread_segment */
     };
     char reserved[192];
 } sync_ctl __attribute__((aligned(64)));
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -437,6 +437,10 @@
 
    stm_enter_transactional_zone() and stm_leave_transactional_zone()
    preserve the value of errno.
+
+   stm_leave_transactional_zone_final() commits the transaction
+   unconditionally and allows the caller to teardown the whole
+   thread (unregister thread local, etc.).
 */
 #ifdef STM_DEBUGPRINT
 #include <stdio.h>
@@ -468,6 +472,11 @@
         _stm_leave_noninevitable_transactional_zone();
     }
 }
+static inline void stm_leave_transactional_zone_final(stm_thread_local_t *tl) {
+    assert(STM_SEGMENT->running_thread == tl);
+    _stm_commit_transaction();
+}
+
 
 /* stm_force_transaction_break() is in theory equivalent to
    stm_leave_transactional_zone() immediately followed by
diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -98,16 +98,19 @@
 
 void pypy_stm_leave_callback_call(void *rjbuf, long token)
 {
-    stm_leave_transactional_zone(&stm_thread_local);
-    stm_rewind_jmp_leaveframe(&stm_thread_local, (rewind_jmp_buf *)rjbuf);
-
     if (token == 1) {
         /* if we're returning into foreign C code that was not itself
            called from Python code, then we're ignoring the atomic
            status and committing anyway. */
+        stm_leave_transactional_zone_final(&stm_thread_local);
+        stm_rewind_jmp_leaveframe(&stm_thread_local, (rewind_jmp_buf *)rjbuf);
+
         int e = errno;
         pypy_stm_unregister_thread_local();
         errno = e;
+    } else {
+        stm_leave_transactional_zone(&stm_thread_local);
+        stm_rewind_jmp_leaveframe(&stm_thread_local, (rewind_jmp_buf *)rjbuf);
     }
 }
 

From noreply at buildbot.pypy.org  Fri Jul 24 11:51:38 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 24 Jul 2015 11:51:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: trivial jit-test opcode fix
Message-ID: <20150724095138.DF0EF1C1230@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78650:dde9255e376c
Date: 2015-07-24 12:49 +0300
http://bitbucket.org/pypy/pypy/changeset/dde9255e376c/

Log:	trivial jit-test opcode fix

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -29,7 +29,7 @@
             guard_true(i15, descr=...)
             guard_not_invalidated(descr=...)
             i17 = cast_float_to_int(f16)
-            i19 = int_and(i17, 255)
+            i19 = int_is_true(i17)
             guard_true(i19, descr=...)
             i20 = getfield_gc_pure(p2, descr=<FieldU pypy.module.micronumpy.boxes.W_BoolBox.inst_value \d+>)
             i21 = int_is_true(i20)

From noreply at buildbot.pypy.org  Fri Jul 24 11:51:40 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 24 Jul 2015 11:51:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: test fix after 7ebfda5c3881,
 now test is cpython compatible
Message-ID: <20150724095140.2AF711C1230@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78651:c176f08d7abd
Date: 2015-07-24 12:51 +0300
http://bitbucket.org/pypy/pypy/changeset/c176f08d7abd/

Log:	test fix after 7ebfda5c3881, now test is cpython compatible

diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -639,10 +639,10 @@
             pass
         a = A()
         b = B()
-        assert isinstance(a, A)
+        assert isinstance(a, A) # "shortcut" does not go through metaclass
         assert not isinstance(a, B)
         assert isinstance(b, A)
-        assert not isinstance(b, B)
+        assert isinstance(b, B) # "shortcut" does not go through metaclass
         assert isinstance(4, A)
         assert not issubclass(A, A)
         assert not issubclass(B, A)

From noreply at buildbot.pypy.org  Fri Jul 24 12:05:12 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 24 Jul 2015 12:05:12 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: failing test
Message-ID: <20150724100512.5B4DA1C1230@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fix-strbuf
Changeset: r78652:7b73c5d11ce0
Date: 2015-07-24 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/7b73c5d11ce0/

Log:	failing test

diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -67,7 +67,6 @@
         return self
 
 
-delegation_dict = {}
 for key, value in W_BytesObject.typedef.rawdict.iteritems():
     if not isinstance(value, interp2app):
         continue
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ b/pypy/objspace/std/test/test_strbufobject.py
@@ -83,3 +83,9 @@
         a = 'a'
         a += 'b'
         raises(TypeError, "a += 5")
+
+    def test_mix_strings_format(self):
+        a = 'a'
+        a += 'b'
+        assert 'foo%s' % a == 'fooab'
+        assert (a + '%s') % ('foo',) == 'bfoo'

From noreply at buildbot.pypy.org  Fri Jul 24 12:32:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 24 Jul 2015 12:32:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a test for operator.methodcaller()
Message-ID: <20150724103210.E2C461C0987@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78653:e7ee41dab3bd
Date: 2015-07-24 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/e7ee41dab3bd/

Log:	Add a test for operator.methodcaller()

diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -10,13 +10,16 @@
         class A(object):
             getx = operator.attrgetter('x')
             get3 = operator.itemgetter(3)
+            callx = operator.methodcaller("append", "x")
         a = A()
         a.x = 5
         assert a.getx(a) == 5
         assert a.get3("foobar") == "b"
         assert a.getx(*(a,)) == 5
         assert a.get3(obj="foobar") == "b"
-
+        l = []
+        a.callx(l)
+        assert l == ["x"]
 
     def test_getter_multiple_gest(self):
         import operator

From noreply at buildbot.pypy.org  Fri Jul 24 13:37:26 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 24 Jul 2015 13:37:26 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: Backed out changeset: c0e7e64a894d
Message-ID: <20150724113726.4E2071C0987@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: use-gcc
Changeset: r1897:ce143e28d5ee
Date: 2015-07-24 13:38 +0200
http://bitbucket.org/pypy/stmgc/changeset/ce143e28d5ee/

Log:	Backed out changeset: c0e7e64a894d (was actually done already in a
	different way)

diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -14,8 +14,7 @@
         pthread_mutex_t global_mutex;
         pthread_cond_t cond[_C_TOTAL];
         /* some additional pieces of global state follow */
-        uint8_t in_use1[NB_SEGMENTS];   /* 1 if running a pthread, idx=0 unused,
-                                           2 if soon_finished_or_inevitable_thread_segment */
+        uint8_t in_use1[NB_SEGMENTS];   /* 1 if running a pthread, idx=0 unused */
     };
     char reserved[192];
 } sync_ctl __attribute__((aligned(64)));
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -437,10 +437,6 @@
 
    stm_enter_transactional_zone() and stm_leave_transactional_zone()
    preserve the value of errno.
-
-   stm_leave_transactional_zone_final() commits the transaction
-   unconditionally and allows the caller to teardown the whole
-   thread (unregister thread local, etc.).
 */
 #ifdef STM_DEBUGPRINT
 #include <stdio.h>
@@ -472,11 +468,6 @@
         _stm_leave_noninevitable_transactional_zone();
     }
 }
-static inline void stm_leave_transactional_zone_final(stm_thread_local_t *tl) {
-    assert(STM_SEGMENT->running_thread == tl);
-    _stm_commit_transaction();
-}
-
 
 /* stm_force_transaction_break() is in theory equivalent to
    stm_leave_transactional_zone() immediately followed by

From noreply at buildbot.pypy.org  Fri Jul 24 13:37:40 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 24 Jul 2015 13:37:40 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: Backed out changeset: 188629c10d4a
Message-ID: <20150724113740.5A8741C0987@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78654:6c2f7b61c774
Date: 2015-07-24 13:39 +0200
http://bitbucket.org/pypy/pypy/changeset/6c2f7b61c774/

Log:	Backed out changeset: 188629c10d4a (was actually done already in a
	different way)

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-a89f21f5670b+
+088f807586c2
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -14,8 +14,7 @@
         pthread_mutex_t global_mutex;
         pthread_cond_t cond[_C_TOTAL];
         /* some additional pieces of global state follow */
-        uint8_t in_use1[NB_SEGMENTS];   /* 1 if running a pthread, idx=0 unused,
-                                           2 if soon_finished_or_inevitable_thread_segment */
+        uint8_t in_use1[NB_SEGMENTS];   /* 1 if running a pthread, idx=0 unused */
     };
     char reserved[192];
 } sync_ctl __attribute__((aligned(64)));
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -437,10 +437,6 @@
 
    stm_enter_transactional_zone() and stm_leave_transactional_zone()
    preserve the value of errno.
-
-   stm_leave_transactional_zone_final() commits the transaction
-   unconditionally and allows the caller to teardown the whole
-   thread (unregister thread local, etc.).
 */
 #ifdef STM_DEBUGPRINT
 #include <stdio.h>
@@ -472,11 +468,6 @@
         _stm_leave_noninevitable_transactional_zone();
     }
 }
-static inline void stm_leave_transactional_zone_final(stm_thread_local_t *tl) {
-    assert(STM_SEGMENT->running_thread == tl);
-    _stm_commit_transaction();
-}
-
 
 /* stm_force_transaction_break() is in theory equivalent to
    stm_leave_transactional_zone() immediately followed by
diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -98,19 +98,16 @@
 
 void pypy_stm_leave_callback_call(void *rjbuf, long token)
 {
+    stm_leave_transactional_zone(&stm_thread_local);
+    stm_rewind_jmp_leaveframe(&stm_thread_local, (rewind_jmp_buf *)rjbuf);
+
     if (token == 1) {
         /* if we're returning into foreign C code that was not itself
            called from Python code, then we're ignoring the atomic
            status and committing anyway. */
-        stm_leave_transactional_zone_final(&stm_thread_local);
-        stm_rewind_jmp_leaveframe(&stm_thread_local, (rewind_jmp_buf *)rjbuf);
-
         int e = errno;
         pypy_stm_unregister_thread_local();
         errno = e;
-    } else {
-        stm_leave_transactional_zone(&stm_thread_local);
-        stm_rewind_jmp_leaveframe(&stm_thread_local, (rewind_jmp_buf *)rjbuf);
     }
 }
 

From noreply at buildbot.pypy.org  Fri Jul 24 14:59:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 24 Jul 2015 14:59:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: since we're read only,
 don't do a copy when we don't have to
Message-ID: <20150724125913.5EE851C1F82@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78655:1c1bafcc37b9
Date: 2015-07-24 14:58 +0200
http://bitbucket.org/pypy/pypy/changeset/1c1bafcc37b9/

Log:	since we're read only, don't do a copy when we don't have to

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -74,6 +74,8 @@
             return ""
         if step == 1:
             assert 0 <= start <= stop
+            if start == 0 and stop == len(self.value):
+                return self.value
             return self.value[start:stop]
         return Buffer.getslice(self, start, stop, step, size)
 

From noreply at buildbot.pypy.org  Fri Jul 24 14:59:14 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 24 Jul 2015 14:59:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150724125914.8CD1F1C1F82@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r78656:1caecf593f7c
Date: 2015-07-24 14:59 +0200
http://bitbucket.org/pypy/pypy/changeset/1caecf593f7c/

Log:	merge

diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -10,13 +10,16 @@
         class A(object):
             getx = operator.attrgetter('x')
             get3 = operator.itemgetter(3)
+            callx = operator.methodcaller("append", "x")
         a = A()
         a.x = 5
         assert a.getx(a) == 5
         assert a.get3("foobar") == "b"
         assert a.getx(*(a,)) == 5
         assert a.get3(obj="foobar") == "b"
-
+        l = []
+        a.callx(l)
+        assert l == ["x"]
 
     def test_getter_multiple_gest(self):
         import operator

From noreply at buildbot.pypy.org  Fri Jul 24 17:36:45 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 24 Jul 2015 17:36:45 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: another attempt to fix things
	(unsuccessfully)
Message-ID: <20150724153645.B7CA71C1FA7@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78657:bd074f735670
Date: 2015-07-24 17:38 +0200
http://bitbucket.org/pypy/pypy/changeset/bd074f735670/

Log:	another attempt to fix things (unsuccessfully)

diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -212,6 +212,30 @@
         self.default(hop)
         self.pop_roots(hop, livevars)
 
+
+    # sync with lloperation.py:
+    # These operations need roots pushed around their execution.
+    # stm_allocate_* should never be seen here and are handled by
+    # the super class through gct_malloc_** and similar.
+    gct_stm_unregister_thread_local  = _gct_with_roots_pushed
+    gct_stm_collect                  = _gct_with_roots_pushed
+    gct_stm_become_inevitable        = _gct_with_roots_pushed
+    gct_stm_enter_transactional_zone = _gct_with_roots_pushed
+    gct_stm_leave_transactional_zone = _gct_with_roots_pushed
+    gct_stm_abort_and_retry          = _gct_with_roots_pushed
+    gct_stm_enter_callback_call      = _gct_with_roots_pushed
+    gct_stm_leave_callback_call      = _gct_with_roots_pushed
+    gct_stm_transaction_break        = _gct_with_roots_pushed
+    gct_stm_stop_all_other_threads   = _gct_with_roots_pushed
+    gct_stm_hint_commit_soon         = _gct_with_roots_pushed
+    gct_stm_hashtable_read           = _gct_with_roots_pushed
+    gct_stm_hashtable_write          = _gct_with_roots_pushed
+    gct_stm_hashtable_lookup         = _gct_with_roots_pushed
+    gct_stm_queue_get                = _gct_with_roots_pushed
+    gct_stm_queue_put                = _gct_with_roots_pushed
+    gct_stm_queue_join               = _gct_with_roots_pushed
+
+
     def gct_stm_malloc_nonmovable(self, hop):
         op = hop.spaceop
         PTRTYPE = op.result.concretetype
@@ -229,14 +253,7 @@
         self.pop_roots(hop, livevars)
         hop.genop("cast_opaque_ptr", [v_result], resultvar=op.result)
 
-    # sync with lloperation.py
-    gct_stm_become_inevitable                       = _gct_with_roots_pushed
-    gct_stm_hint_commit_soon                        = _gct_with_roots_pushed
-    gct_stm_stop_all_other_threads                  = _gct_with_roots_pushed
-    gct_stm_transaction_break                       = _gct_with_roots_pushed
-    gct_stm_collect                                 = _gct_with_roots_pushed
-    gct_stm_queue_get                               = _gct_with_roots_pushed
-    gct_stm_queue_join                              = _gct_with_roots_pushed
+
 
 
 class StmRootWalker(BaseRootWalker):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -414,7 +414,7 @@
     # (some ops like stm_commit_transaction don't need it because there
     #  must be no gc-var access afterwards anyway)
     'stm_register_thread_local': LLOp(),
-    'stm_unregister_thread_local': LLOp(),
+    'stm_unregister_thread_local': LLOp(canmallocgc=True),
     'stm_read':               LLOp(),
     'stm_write':              LLOp(),
     'stm_can_move':           LLOp(),
@@ -438,7 +438,7 @@
     'stm_leave_transactional_zone':       LLOp(canmallocgc=True),
     'stm_abort_and_retry':                LLOp(canmallocgc=True),
     'stm_enter_callback_call':            LLOp(canmallocgc=True),
-    'stm_leave_callback_call':            LLOp(),
+    'stm_leave_callback_call':            LLOp(canmallocgc=True),
     'stm_transaction_break':              LLOp(canmallocgc=True, canrun=True),
     'stm_should_break_transaction':       LLOp(sideeffects=False),
     'stm_rewind_jmp_frame':               LLOp(canrun=True),
@@ -474,9 +474,9 @@
 
     'stm_hashtable_create':   LLOp(),
     'stm_hashtable_free':     LLOp(),
-    'stm_hashtable_read':     LLOp(),
-    'stm_hashtable_write':    LLOp(),
-    'stm_hashtable_lookup':   LLOp(),
+    'stm_hashtable_read':     LLOp(canmallocgc=True),
+    'stm_hashtable_write':    LLOp(canmallocgc=True),
+    'stm_hashtable_lookup':   LLOp(canmallocgc=True),
     'stm_hashtable_write_entry':        LLOp(),
     'stm_hashtable_length_upper_bound': LLOp(),
     'stm_hashtable_list'  :   LLOp(),
@@ -485,7 +485,7 @@
     'stm_queue_create':       LLOp(),
     'stm_queue_free':         LLOp(),
     'stm_queue_get':          LLOp(canmallocgc=True),   # push roots!
-    'stm_queue_put':          LLOp(),
+    'stm_queue_put':          LLOp(canmallocgc=True),
     'stm_queue_task_done':    LLOp(),
     'stm_queue_join':         LLOp(canmallocgc=True),   # push roots!
     'stm_queue_tracefn':      LLOp(),

From noreply at buildbot.pypy.org  Sat Jul 25 12:33:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 25 Jul 2015 12:33:40 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: stupid fix
Message-ID: <20150725103340.5EDB71C08F3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fix-strbuf
Changeset: r78658:933b019bffdc
Date: 2015-07-25 12:33 +0200
http://bitbucket.org/pypy/pypy/changeset/933b019bffdc/

Log:	stupid fix

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -942,9 +942,9 @@
     upper = interpindirect2app(W_AbstractBytesObject.descr_upper),
     zfill = interpindirect2app(W_AbstractBytesObject.descr_zfill),
 
-    format = interpindirect2app(W_BytesObject.descr_format),
-    __format__ = interpindirect2app(W_BytesObject.descr__format__),
-    __mod__ = interpindirect2app(W_BytesObject.descr_mod),
+    format = interpindirect2app(W_AbstractBytesObject.descr_format),
+    __format__ = interpindirect2app(W_AbstractBytesObject.descr__format__),
+    __mod__ = interpindirect2app(W_AbstractBytesObject.descr_mod),
     __getnewargs__ = interpindirect2app(
         W_AbstractBytesObject.descr_getnewargs),
     _formatter_parser = interp2app(W_BytesObject.descr_formatter_parser),
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ b/pypy/objspace/std/test/test_strbufobject.py
@@ -88,4 +88,4 @@
         a = 'a'
         a += 'b'
         assert 'foo%s' % a == 'fooab'
-        assert (a + '%s') % ('foo',) == 'bfoo'
+        assert (a + '%s') % ('foo',) == 'abfoo'

From noreply at buildbot.pypy.org  Sat Jul 25 15:04:31 2015
From: noreply at buildbot.pypy.org (nip3o)
Date: Sat, 25 Jul 2015 15:04:31 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3-exceptions: Fixed time.strptime test on
	Darwin.
Message-ID: <20150725130431.9C9F91C12F4@cobra.cs.uni-duesseldorf.de>

Author: Niclas Olofsson <n at niclasolofsson.se>
Branch: py3.3-exceptions
Changeset: r78659:3cc1d9bd955c
Date: 2015-07-25 13:58 +0200
http://bitbucket.org/pypy/pypy/changeset/3cc1d9bd955c/

Log:	Fixed time.strptime test on Darwin.

diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -248,8 +248,6 @@
         raises(TypeError, time.strftime, ())
         raises(TypeError, time.strftime, (1,))
         raises(TypeError, time.strftime, range(8))
-        exp = '0 01 01 00 00 00 1 001'
-        assert time.strftime("%Y %m %d %H %M %S %w %j", (0,)*9) == exp
 
         # Guard against invalid/non-supported format string
         # so that Python don't crash (Windows crashes when the format string
@@ -260,8 +258,15 @@
             # darwin strips % of unknown format codes
             # http://bugs.python.org/issue9811
             assert time.strftime('%f') == 'f'
+
+            # Darwin always use four digits for %Y, Linux uses as many as needed.
+            expected_year = '0000'
         else:
             assert time.strftime('%f') == '%f'
+            expected_year = '0'
+
+        expected_formatted_date = expected_year + ' 01 01 00 00 00 1 001'
+        assert time.strftime("%Y %m %d %H %M %S %w %j", (0,) * 9) == expected_formatted_date
 
     def test_strftime_ext(self):
         import time

From noreply at buildbot.pypy.org  Sat Jul 25 15:04:32 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 25 Jul 2015 15:04:32 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Merged in nip3o/pypy/py3.3-exceptions
	(pull request #328)
Message-ID: <20150725130432.E77F31C12F4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: py3.3
Changeset: r78660:42968b84b6df
Date: 2015-07-25 15:04 +0200
http://bitbucket.org/pypy/pypy/changeset/42968b84b6df/

Log:	Merged in nip3o/pypy/py3.3-exceptions (pull request #328)

	Fixed strftime test on darwin

diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -248,8 +248,6 @@
         raises(TypeError, time.strftime, ())
         raises(TypeError, time.strftime, (1,))
         raises(TypeError, time.strftime, range(8))
-        exp = '0 01 01 00 00 00 1 001'
-        assert time.strftime("%Y %m %d %H %M %S %w %j", (0,)*9) == exp
 
         # Guard against invalid/non-supported format string
         # so that Python don't crash (Windows crashes when the format string
@@ -260,8 +258,15 @@
             # darwin strips % of unknown format codes
             # http://bugs.python.org/issue9811
             assert time.strftime('%f') == 'f'
+
+            # Darwin always use four digits for %Y, Linux uses as many as needed.
+            expected_year = '0000'
         else:
             assert time.strftime('%f') == '%f'
+            expected_year = '0'
+
+        expected_formatted_date = expected_year + ' 01 01 00 00 00 1 001'
+        assert time.strftime("%Y %m %d %H %M %S %w %j", (0,) * 9) == expected_formatted_date
 
     def test_strftime_ext(self):
         import time

From noreply at buildbot.pypy.org  Sat Jul 25 18:37:54 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 25 Jul 2015 18:37:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Create out_converter() to factor out
	some code
Message-ID: <20150725163754.3D6611C02A3@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r78661:07616c948f75
Date: 2015-07-25 17:37 +0100
http://bitbucket.org/pypy/pypy/changeset/07616c948f75/

Log:	Create out_converter() to factor out some code

diff --git a/pypy/module/micronumpy/converters.py b/pypy/module/micronumpy/converters.py
--- a/pypy/module/micronumpy/converters.py
+++ b/pypy/module/micronumpy/converters.py
@@ -113,3 +113,12 @@
         shape.append(space.int_w(w_item))
     shape += dtype.shape
     return shape[:]
+
+def out_converter(space, w_out):
+    from .ndarray import W_NDimArray
+    if space.is_none(w_out):
+        return None
+    elif not isinstance(w_out, W_NDimArray):
+        raise oefmt(space.w_TypeError, 'output must be an array')
+    else:
+        return w_out
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -15,8 +15,9 @@
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array, \
     ArrayArgumentException, wrap_impl
 from pypy.module.micronumpy.concrete import BaseConcreteArray
-from pypy.module.micronumpy.converters import multi_axis_converter, \
-    order_converter, shape_converter, searchside_converter
+from pypy.module.micronumpy.converters import (
+    multi_axis_converter, order_converter, shape_converter,
+    searchside_converter, out_converter)
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import (
     get_shape_from_iterable, shape_agreement, shape_agreement_multiple,
@@ -1090,12 +1091,7 @@
 
     def descr_dot(self, space, w_other, w_out=None):
         from .casting import find_result_type
-        if space.is_none(w_out):
-            out = None
-        elif not isinstance(w_out, W_NDimArray):
-            raise oefmt(space.w_TypeError, 'output must be an array')
-        else:
-            out = w_out
+        out = out_converter(space, w_out)
         other = convert_to_array(space, w_other)
         if other.is_scalar():
             #Note: w_out is not modified, this is numpy compliant.
@@ -1152,12 +1148,7 @@
     def _reduce_ufunc_impl(ufunc_name, cumulative=False, bool_result=False):
         @unwrap_spec(keepdims=bool)
         def impl(self, space, w_axis=None, w_dtype=None, w_out=None, keepdims=False):
-            if space.is_none(w_out):
-                out = None
-            elif not isinstance(w_out, W_NDimArray):
-                raise oefmt(space.w_TypeError, 'output must be an array')
-            else:
-                out = w_out
+            out = out_converter(space, w_out)
             if bool_result:
                 w_dtype = descriptor.get_dtype_cache(space).w_booldtype
             return getattr(ufuncs.get(space), ufunc_name).reduce(
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -20,6 +20,7 @@
 from pypy.module.micronumpy.strides import shape_agreement
 from pypy.module.micronumpy.support import (_parse_signature, product,
         get_storage_as_int, is_rhs_priority_higher)
+from .converters import out_converter
 from .casting import (
     can_cast_type, can_cast_array, can_cast_to,
     find_result_type, promote_types)
@@ -124,10 +125,7 @@
         args_w, kwds_w = __args__.unpack()
         # sig, extobj are used in generic ufuncs
         w_subok, w_out, sig, w_casting, extobj = self.parse_kwargs(space, kwds_w)
-        if space.is_w(w_out, space.w_None):
-            out = None
-        else:
-            out = w_out
+        out = out_converter(space, w_out)
         if (w_subok is not None and space.is_true(w_subok)):
             raise oefmt(space.w_NotImplementedError, "parameter subok unsupported")
         if kwds_w:
@@ -149,9 +147,6 @@
             out = args_w[-1]
         else:
             args_w = args_w + [out]
-        if out is not None and not isinstance(out, W_NDimArray):
-            raise OperationError(space.w_TypeError, space.wrap(
-                                            'output must be an array'))
         if w_casting is None:
             casting = 'unsafe'
         else:
@@ -163,13 +158,7 @@
     def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None):
         if space.is_none(w_axis):
             w_axis = space.wrap(0)
-        if space.is_none(w_out):
-            out = None
-        elif not isinstance(w_out, W_NDimArray):
-            raise OperationError(space.w_TypeError, space.wrap(
-                                                'output must be an array'))
-        else:
-            out = w_out
+        out = out_converter(space, w_out)
         return self.reduce(space, w_obj, w_axis, True, #keepdims must be true
                            out, w_dtype, cumulative=True)
 
@@ -231,13 +220,7 @@
         from pypy.module.micronumpy.ndarray import W_NDimArray
         if w_axis is None:
             w_axis = space.wrap(0)
-        if space.is_none(w_out):
-            out = None
-        elif not isinstance(w_out, W_NDimArray):
-            raise OperationError(space.w_TypeError, space.wrap(
-                'output must be an array'))
-        else:
-            out = w_out
+        out = out_converter(space, w_out)
         return self.reduce(space, w_obj, w_axis, keepdims, out, w_dtype)
 
     def reduce(self, space, w_obj, w_axis, keepdims=False, out=None, dtype=None,
@@ -460,11 +443,7 @@
         w_obj = args_w[0]
         out = None
         if len(args_w) > 1:
-            out = args_w[1]
-            if space.is_w(out, space.w_None):
-                out = None
-            elif out is not None and not isinstance(out, W_NDimArray):
-                raise oefmt(space.w_TypeError, 'output must be an array')
+            out = out_converter(space, args_w[1])
         w_obj = numpify(space, w_obj)
         dtype = w_obj.get_dtype(space)
         calc_dtype, dt_out, func = self.find_specialization(space, dtype, out, casting)
@@ -562,10 +541,7 @@
     def call(self, space, args_w, sig, casting, extobj):
         if len(args_w) > 2:
             [w_lhs, w_rhs, out] = args_w
-            if space.is_none(out):
-                out = None
-            elif not isinstance(out, W_NDimArray):
-                raise oefmt(space.w_TypeError, 'output must be an array')
+            out = out_converter(space, out)
         else:
             [w_lhs, w_rhs] = args_w
             out = None

From noreply at buildbot.pypy.org  Sat Jul 25 19:22:29 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 25 Jul 2015 19:22:29 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: hg merge default
Message-ID: <20150725172229.D30DA1C0400@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78662:dbe051dac229
Date: 2015-07-25 18:14 +0100
http://bitbucket.org/pypy/pypy/changeset/dbe051dac229/

Log:	hg merge default

diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -2,6 +2,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.tool.pairtype import extendabletype
 from pypy.module.micronumpy import support
+from pypy.module.micronumpy import constants as NPY
 
 def wrap_impl(space, w_cls, w_instance, impl):
     if w_cls is None or space.is_w(w_cls, space.gettypefor(W_NDimArray)):
@@ -39,6 +40,13 @@
     def from_shape(space, shape, dtype, order='C', w_instance=None, zero=True):
         from pypy.module.micronumpy import concrete, descriptor, boxes
         from pypy.module.micronumpy.strides import calc_strides
+        if len(shape) > NPY.MAXDIMS:
+            raise oefmt(space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        try:
+            support.product(shape) * dtype.elsize
+        except OverflowError as e:
+            raise oefmt(space.w_ValueError, "array is too big")
         strides, backstrides = calc_strides(shape, dtype.base, order)
         impl = concrete.ConcreteArray(shape, dtype.base, order, strides,
                                       backstrides, zero=zero)
@@ -56,13 +64,19 @@
         from pypy.module.micronumpy.strides import (calc_strides,
                                                     calc_backstrides)
         isize = dtype.elsize
+        if len(shape) > NPY.MAXDIMS:
+            raise oefmt(space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        try:
+            totalsize = support.product(shape) * isize
+        except OverflowError as e:
+            raise oefmt(space.w_ValueError, "array is too big")
         if storage_bytes > 0 :
-            totalsize = support.product(shape) * isize
             if totalsize > storage_bytes:
                 raise OperationError(space.w_TypeError, space.wrap(
                     "buffer is too small for requested array"))
         else:
-            storage_bytes = support.product(shape) * isize
+            storage_bytes = totalsize
         if strides is None:
             strides, backstrides = calc_strides(shape, dtype, order)
         else:
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -400,17 +400,11 @@
 
 class ConcreteArrayNotOwning(BaseConcreteArray):
     def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0):
-        if len(shape) > NPY.MAXDIMS:
-            raise oefmt(dtype.itemtype.space.w_ValueError,
-                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
         make_sure_not_resized(shape)
         make_sure_not_resized(strides)
         make_sure_not_resized(backstrides)
         self.shape = shape
-        try:
-            self.size = support.product(shape) * dtype.elsize
-        except OverflowError as e:
-            raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big")
+        self.size = support.product(shape) * dtype.elsize
         self.order = order
         self.dtype = dtype
         self.strides = strides
@@ -425,6 +419,13 @@
             box, 0, self.size, 0, self.gcstruct)
 
     def set_shape(self, space, orig_array, new_shape):
+        if len(new_shape) > NPY.MAXDIMS:
+            raise oefmt(space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        try:
+            support.product(new_shape) * self.dtype.elsize
+        except OverflowError as e:
+            raise oefmt(space.w_ValueError, "array is too big")
         strides, backstrides = calc_strides(new_shape, self.dtype,
                                                     self.order)
         return SliceArray(self.start, strides, backstrides, new_shape, self,
@@ -451,14 +452,9 @@
                  storage=lltype.nullptr(RAW_STORAGE), zero=True):
         gcstruct = V_OBJECTSTORE
         flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
-        if len(shape) > NPY.MAXDIMS:
-            raise oefmt(dtype.itemtype.space.w_ValueError,
-                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        length = support.product(shape)
+        self.size = length * dtype.elsize
         if storage == lltype.nullptr(RAW_STORAGE):
-            try:
-                length = support.product(shape)
-            except OverflowError as e:
-                raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big")
             if dtype.num == NPY.OBJECT:
                 storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True)
                 gcstruct = _create_objectstore(storage, length, dtype.elsize)
@@ -559,6 +555,13 @@
         loop.fill(self, box.convert_to(space, self.dtype))
 
     def set_shape(self, space, orig_array, new_shape):
+        if len(new_shape) > NPY.MAXDIMS:
+            raise oefmt(space.w_ValueError,
+                "sequence too large; must be smaller than %d", NPY.MAXDIMS)
+        try:
+            support.product(new_shape) * self.dtype.elsize
+        except OverflowError as e:
+            raise oefmt(space.w_ValueError, "array is too big")
         if len(self.get_shape()) < 2 or self.size == 0:
             # TODO: this code could be refactored into calc_strides
             # but then calc_strides would have to accept a stepping factor
diff --git a/pypy/module/micronumpy/converters.py b/pypy/module/micronumpy/converters.py
--- a/pypy/module/micronumpy/converters.py
+++ b/pypy/module/micronumpy/converters.py
@@ -113,3 +113,12 @@
         shape.append(space.int_w(w_item))
     shape += dtype.shape
     return shape[:]
+
+def out_converter(space, w_out):
+    from .ndarray import W_NDimArray
+    if space.is_none(w_out):
+        return None
+    elif not isinstance(w_out, W_NDimArray):
+        raise oefmt(space.w_TypeError, 'output must be an array')
+    else:
+        return w_out
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -15,8 +15,9 @@
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array, \
     ArrayArgumentException, wrap_impl
 from pypy.module.micronumpy.concrete import BaseConcreteArray
-from pypy.module.micronumpy.converters import multi_axis_converter, \
-    order_converter, shape_converter, searchside_converter
+from pypy.module.micronumpy.converters import (
+    multi_axis_converter, order_converter, shape_converter,
+    searchside_converter, out_converter)
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import (
     get_shape_from_iterable, shape_agreement, shape_agreement_multiple,
@@ -1090,12 +1091,7 @@
 
     def descr_dot(self, space, w_other, w_out=None):
         from .casting import find_result_type
-        if space.is_none(w_out):
-            out = None
-        elif not isinstance(w_out, W_NDimArray):
-            raise oefmt(space.w_TypeError, 'output must be an array')
-        else:
-            out = w_out
+        out = out_converter(space, w_out)
         other = convert_to_array(space, w_other)
         if other.is_scalar():
             #Note: w_out is not modified, this is numpy compliant.
@@ -1152,12 +1148,7 @@
     def _reduce_ufunc_impl(ufunc_name, name, variant=ufuncs.REDUCE, bool_result=False):
         @unwrap_spec(keepdims=bool)
         def impl(self, space, w_axis=None, w_dtype=None, w_out=None, keepdims=False):
-            if space.is_none(w_out):
-                out = None
-            elif not isinstance(w_out, W_NDimArray):
-                raise oefmt(space.w_TypeError, 'output must be an array')
-            else:
-                out = w_out
+            out = out_converter(space, w_out)
             if bool_result:
                 w_dtype = descriptor.get_dtype_cache(space).w_booldtype
             return getattr(ufuncs.get(space), ufunc_name).reduce(
@@ -1336,7 +1327,9 @@
     dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
         space.gettypefor(descriptor.W_Dtype), w_dtype))
     shape = shape_converter(space, w_shape, dtype)
-
+    if len(shape) > NPY.MAXDIMS:
+        raise oefmt(space.w_ValueError,
+            "sequence too large; must be smaller than %d", NPY.MAXDIMS)
     if not space.is_none(w_buffer):
         if (not space.is_none(w_strides)):
             strides = [space.int_w(w_i) for w_i in
@@ -1373,6 +1366,10 @@
     if space.is_w(w_subtype, space.gettypefor(W_NDimArray)):
         return W_NDimArray.from_shape(space, shape, dtype, order)
     strides, backstrides = calc_strides(shape, dtype.base, order)
+    try:
+        totalsize = support.product(shape) * dtype.base.elsize
+    except OverflowError as e:
+        raise oefmt(space.w_ValueError, "array is too big")
     impl = ConcreteArray(shape, dtype.base, order, strides, backstrides)
     w_ret = space.allocate_instance(W_NDimArray, w_subtype)
     W_NDimArray.__init__(w_ret, impl)
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -20,6 +20,7 @@
 from pypy.module.micronumpy.strides import shape_agreement
 from pypy.module.micronumpy.support import (_parse_signature, product,
         get_storage_as_int, is_rhs_priority_higher)
+from .converters import out_converter
 from .casting import (
     can_cast_type, can_cast_array, can_cast_to,
     find_result_type, promote_types)
@@ -127,10 +128,7 @@
         args_w, kwds_w = __args__.unpack()
         # sig, extobj are used in generic ufuncs
         w_subok, w_out, sig, w_casting, extobj = self.parse_kwargs(space, kwds_w)
-        if space.is_w(w_out, space.w_None):
-            out = None
-        else:
-            out = w_out
+        out = out_converter(space, w_out)
         if (w_subok is not None and space.is_true(w_subok)):
             raise oefmt(space.w_NotImplementedError, "parameter subok unsupported")
         if kwds_w:
@@ -152,9 +150,6 @@
             out = args_w[-1]
         else:
             args_w = args_w + [out]
-        if out is not None and not isinstance(out, W_NDimArray):
-            raise OperationError(space.w_TypeError, space.wrap(
-                                            'output must be an array'))
         if w_casting is None:
             casting = 'unsafe'
         else:
@@ -166,13 +161,7 @@
     def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None):
         if space.is_none(w_axis):
             w_axis = space.wrap(0)
-        if space.is_none(w_out):
-            out = None
-        elif not isinstance(w_out, W_NDimArray):
-            raise OperationError(space.w_TypeError, space.wrap(
-                                                'output must be an array'))
-        else:
-            out = w_out
+        out = out_converter(space, w_out)
         return self.reduce(space, w_obj, w_axis, True, #keepdims must be true
                            out, w_dtype, variant=ACCUMULATE)
 
@@ -234,13 +223,7 @@
         from pypy.module.micronumpy.ndarray import W_NDimArray
         if w_axis is None:
             w_axis = space.wrap(0)
-        if space.is_none(w_out):
-            out = None
-        elif not isinstance(w_out, W_NDimArray):
-            raise OperationError(space.w_TypeError, space.wrap(
-                'output must be an array'))
-        else:
-            out = w_out
+        out = out_converter(space, w_out)
         return self.reduce(space, w_obj, w_axis, keepdims, out, w_dtype)
 
     @specialize.arg(7)
@@ -497,11 +480,7 @@
         w_obj = args_w[0]
         out = None
         if len(args_w) > 1:
-            out = args_w[1]
-            if space.is_w(out, space.w_None):
-                out = None
-            elif out is not None and not isinstance(out, W_NDimArray):
-                raise oefmt(space.w_TypeError, 'output must be an array')
+            out = out_converter(space, args_w[1])
         w_obj = numpify(space, w_obj)
         dtype = w_obj.get_dtype(space)
         calc_dtype, dt_out, func = self.find_specialization(space, dtype, out, casting)
@@ -599,10 +578,7 @@
     def call(self, space, args_w, sig, casting, extobj):
         if len(args_w) > 2:
             [w_lhs, w_rhs, out] = args_w
-            if space.is_none(out):
-                out = None
-            elif not isinstance(out, W_NDimArray):
-                raise oefmt(space.w_TypeError, 'output must be an array')
+            out = out_converter(space, out)
         else:
             [w_lhs, w_rhs] = args_w
             out = None
@@ -750,7 +726,8 @@
         if use_min_scalar:
             w_arg1 = convert_to_array(space, w_arg1)
             w_arg2 = convert_to_array(space, w_arg2)
-        elif in_casting == 'safe' and l_dtype.num == 7 and r_dtype.num == 7:
+        elif (in_casting == 'safe' and l_dtype.num == 7 and r_dtype.num == 7 and
+              out is None and not self.promote_to_float):
             # while long (7) can be cast to int32 (5) on 32 bit, don't do it
             return l_dtype, l_dtype
         for dt_in, dt_out in self.dtypes:
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -10,13 +10,16 @@
         class A(object):
             getx = operator.attrgetter('x')
             get3 = operator.itemgetter(3)
+            callx = operator.methodcaller("append", "x")
         a = A()
         a.x = 5
         assert a.getx(a) == 5
         assert a.get3("foobar") == "b"
         assert a.getx(*(a,)) == 5
         assert a.get3(obj="foobar") == "b"
-
+        l = []
+        a.callx(l)
+        assert l == ["x"]
 
     def test_getter_multiple_gest(self):
         import operator
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -29,7 +29,7 @@
             guard_true(i15, descr=...)
             guard_not_invalidated(descr=...)
             i17 = cast_float_to_int(f16)
-            i19 = int_and(i17, 255)
+            i19 = int_is_true(i17)
             guard_true(i19, descr=...)
             i20 = getfield_gc_pure(p2, descr=<FieldU pypy.module.micronumpy.boxes.W_BoolBox.inst_value \d+>)
             i21 = int_is_true(i20)
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -639,10 +639,10 @@
             pass
         a = A()
         b = B()
-        assert isinstance(a, A)
+        assert isinstance(a, A) # "shortcut" does not go through metaclass
         assert not isinstance(a, B)
         assert isinstance(b, A)
-        assert not isinstance(b, B)
+        assert isinstance(b, B) # "shortcut" does not go through metaclass
         assert isinstance(4, A)
         assert not issubclass(A, A)
         assert not issubclass(B, A)
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -74,6 +74,8 @@
             return ""
         if step == 1:
             assert 0 <= start <= stop
+            if start == 0 and stop == len(self.value):
+                return self.value
             return self.value[start:stop]
         return Buffer.getslice(self, start, stop, step, size)
 

From noreply at buildbot.pypy.org  Sat Jul 25 19:32:42 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 25 Jul 2015 19:32:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix for 32 bit
Message-ID: <20150725173242.68BE81C0400@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78663:c2af38ee8b4f
Date: 2015-07-25 20:11 +0300
http://bitbucket.org/pypy/pypy/changeset/c2af38ee8b4f/

Log:	fix for 32 bit

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -9,7 +9,7 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import (
         specialize, compute_hash, we_are_translated, enforceargs)
-from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
+from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, ovfcheck
 from pypy.module.micronumpy import types, boxes, support, constants as NPY
 from .base import W_NDimArray
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
@@ -975,8 +975,12 @@
     if shape is not None:
         subdtype = make_new_dtype(space, w_subtype, w_dtype, alignment, copy, w_metadata=w_metadata)
         assert isinstance(subdtype, W_Dtype)
-        size = support.product(shape)
-        size *= subdtype.elsize
+        try:
+            size = support.product(shape)
+            size = ovfcheck(size * subdtype.elsize)
+        except OverflowError:
+            raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
+                  "dtype size in bytes must fit into a C int.")
         if size > 0x7fffffff:
             raise oefmt(space.w_ValueError, "invalid shape in fixed-type tuple: "
                   "dtype size in bytes must fit into a C int.")

From noreply at buildbot.pypy.org  Sun Jul 26 11:50:19 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 26 Jul 2015 11:50:19 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: (fijal, arigo) push push push
Message-ID: <20150726095019.C876F1C072C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-strbuf
Changeset: r78664:586fe659736b
Date: 2015-07-26 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/586fe659736b/

Log:	(fijal, arigo) push push push

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -77,11 +77,13 @@
     def _op_val(space, w_other):
         return space.buffer_w(w_other, space.BUF_SIMPLE).as_str()
 
-    def _chr(self, char):
+    @staticmethod
+    def _chr(char):
         assert len(char) == 1
-        return str(char)[0]
+        return char[0]
 
-    def _multi_chr(self, char):
+    @staticmethod
+    def _multi_chr(char):
         return [char]
 
     @staticmethod
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -483,7 +483,7 @@
     @staticmethod
     def _use_rstr_ops(space, w_other):
         from pypy.objspace.std.unicodeobject import W_UnicodeObject
-        return (isinstance(w_other, W_BytesObject) or
+        return (isinstance(w_other, W_AbstractBytesObject) or
                 isinstance(w_other, W_UnicodeObject))
 
     @staticmethod
@@ -495,9 +495,10 @@
                 raise
         return space.charbuf_w(w_other)
 
-    def _chr(self, char):
+    @staticmethod
+    def _chr(char):
         assert len(char) == 1
-        return str(char)[0]
+        return char[0]
 
     _builder = StringBuilder
 
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -28,7 +28,8 @@
         start, end = unwrap_start_stop(space, lenself, w_start, w_end)
         return (value, start, end)
 
-    def _multi_chr(self, c):
+    @staticmethod
+    def _multi_chr(c):
         return c
 
     def descr_len(self, space):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -117,9 +117,10 @@
         return unicode_from_encoded_object(
             space, w_other, None, "strict")._value
 
-    def _chr(self, char):
+    @staticmethod
+    def _chr(char):
         assert len(char) == 1
-        return unicode(char)[0]
+        return unichr(ord(char[0]))
 
     _builder = UnicodeBuilder
 

From noreply at buildbot.pypy.org  Sun Jul 26 11:50:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 26 Jul 2015 11:50:21 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: Add this option in that branch
Message-ID: <20150726095021.06B001C072C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-strbuf
Changeset: r78665:d1bf61de76a1
Date: 2015-07-26 11:50 +0200
http://bitbucket.org/pypy/pypy/changeset/d1bf61de76a1/

Log:	Add this option in that branch

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -297,6 +297,7 @@
         #config.objspace.std.suggest(newshortcut=True)
         config.objspace.std.suggest(withspecialisedtuple=True)
         config.objspace.std.suggest(withidentitydict=True)
+        config.objspace.std.suggest(withstrbuf=True)
         #if not IS_64_BITS:
         #    config.objspace.std.suggest(withsmalllong=True)
 

From noreply at buildbot.pypy.org  Sun Jul 26 12:32:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 26 Jul 2015 12:32:02 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: (fijal, arigo)
Message-ID: <20150726103202.617F71C061F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-strbuf
Changeset: r78666:f1ee691d79af
Date: 2015-07-26 12:32 +0200
http://bitbucket.org/pypy/pypy/changeset/f1ee691d79af/

Log:	(fijal, arigo)

	Move these three methods to W_AbstractBytesObject, with test

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -424,6 +424,21 @@
         of the specified width. The string S is never truncated.
         """
 
+    def writebuf_w(self, space):
+        raise OperationError(space.w_TypeError, space.wrap(
+            "Cannot use string as modifiable buffer"))
+
+    def charbuf_w(self, space):
+        return self.str_w(space)
+
+    def ord(self, space):
+        value = self.str_w(space)
+        if len(value) != 1:
+            raise oefmt(space.w_TypeError,
+                        "ord() expected a character, but string of length %d "
+                        "found", len(value))
+        return space.wrap(ord(value[0]))
+
 
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
@@ -450,22 +465,9 @@
     def readbuf_w(self, space):
         return StringBuffer(self._value)
 
-    def writebuf_w(self, space):
-        raise OperationError(space.w_TypeError, space.wrap(
-            "Cannot use string as modifiable buffer"))
-
-    charbuf_w = str_w
-
     def listview_bytes(self):
         return _create_list_from_bytes(self._value)
 
-    def ord(self, space):
-        if len(self._value) != 1:
-            raise oefmt(space.w_TypeError,
-                        "ord() expected a character, but string of length %d "
-                        "found", len(self._value))
-        return space.wrap(ord(self._value[0]))
-
     def _new(self, value):
         return W_BytesObject(value)
 
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ b/pypy/objspace/std/test/test_strbufobject.py
@@ -89,3 +89,8 @@
         a += 'b'
         assert 'foo%s' % a == 'fooab'
         assert (a + '%s') % ('foo',) == 'abfoo'
+
+    def test_print(self):
+        a = 'abc'
+        a += 'bc'
+        print a

From noreply at buildbot.pypy.org  Sun Jul 26 13:42:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 26 Jul 2015 13:42:01 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: (fijal, arigo)
Message-ID: <20150726114201.961691C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-strbuf
Changeset: r78667:b2013723e0ec
Date: 2015-07-26 13:42 +0200
http://bitbucket.org/pypy/pypy/changeset/b2013723e0ec/

Log:	(fijal, arigo)

	Don't use TypeError in RPython please

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -200,7 +200,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
@@ -208,7 +208,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
@@ -216,7 +216,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
@@ -224,7 +224,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def str_w(self, space):
         self._typed_unwrap_error(space, "string")
@@ -354,6 +354,9 @@
 class DescrMismatch(Exception):
     pass
 
+class BufferInterfaceNotFound(Exception):
+    pass
+
 def wrappable_class_name(Class):
     try:
         return Class.typedef.name
@@ -1403,7 +1406,7 @@
         # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
         try:
             return w_obj.buffer_w(self, flags)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "'%T' does not have the buffer interface", w_obj)
 
@@ -1411,7 +1414,7 @@
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
             return w_obj.readbuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "expected a readable buffer object")
 
@@ -1419,7 +1422,7 @@
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
             return w_obj.writebuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "expected a writeable buffer object")
 
@@ -1427,7 +1430,7 @@
         # Old buffer interface, returns a character buffer (PyObject_AsCharBuffer)
         try:
             return w_obj.charbuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "expected a character buffer object")
 
@@ -1451,11 +1454,11 @@
                 return self.str(w_obj).readbuf_w(self)
             try:
                 return w_obj.buffer_w(self, 0)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 pass
             try:
                 return w_obj.readbuf_w(self)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("string or buffer", w_obj)
         elif code == 's#':
             if self.isinstance_w(w_obj, self.w_str):
@@ -1464,24 +1467,23 @@
                 return self.str(w_obj).str_w(self)
             try:
                 return w_obj.readbuf_w(self).as_str()
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only buffer", w_obj)
         elif code == 'w*':
             try:
-                try:
-                    return w_obj.buffer_w(self, self.BUF_WRITABLE)
-                except OperationError:
-                    self._getarg_error("read-write buffer", w_obj)
-            except TypeError:
+                return w_obj.buffer_w(self, self.BUF_WRITABLE)
+            except OperationError:
+                self._getarg_error("read-write buffer", w_obj)
+            except BufferInterfaceNotFound:
                 pass
             try:
                 return w_obj.writebuf_w(self)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("read-write buffer", w_obj)
         elif code == 't#':
             try:
                 return w_obj.charbuf_w(self)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only character buffer", w_obj)
         else:
             assert False
@@ -1503,13 +1505,13 @@
                 raise
         try:
             buf = w_obj.buffer_w(self, 0)
-        except TypeError:
+        except BufferInterfaceNotFound:
             pass
         else:
             return buf.as_str()
         try:
             buf = w_obj.readbuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             self._getarg_error("string or buffer", w_obj)
         else:
             return buf.as_str()
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -12,6 +12,7 @@
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
     interp_attrproperty, make_weakref_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.interpreter.streamutil import wrap_streamerror, wrap_oserror_as_ioerror
 
 
@@ -499,7 +500,7 @@
                         line = w_line.readbuf_w(space).as_str()
                     else:
                         line = w_line.charbuf_w(space)
-                except TypeError:
+                except BufferInterfaceNotFound:
                     raise OperationError(space.w_TypeError, space.wrap(
                         "writelines() argument must be a sequence of strings"))
                 else:
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.baseobjspace import W_Root, BufferInterfaceNotFound
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.error import OperationError, wrap_windowserror, oefmt
@@ -335,7 +335,7 @@
         else:
             try:
                 value = w_value.readbuf_w(space)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 raise oefmt(space.w_TypeError,
                             "Objects of type '%T' can not be used as binary "
                             "registry values", w_value)
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -9,6 +9,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
@@ -60,7 +61,7 @@
     try:
         a_buf = w_a.buffer_w(space, space.BUF_SIMPLE)
         b_buf = w_b.buffer_w(space, space.BUF_SIMPLE)
-    except TypeError:
+    except BufferInterfaceNotFound:
         raise oefmt(space.w_TypeError,
                     "unsupported operand types(s) or combination of types: "
                     "'%T' and '%T'", w_a, w_b)

From noreply at buildbot.pypy.org  Sun Jul 26 14:32:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 26 Jul 2015 14:32:37 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: (fijal, arigo)
Message-ID: <20150726123237.C382D1C20D6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-strbuf
Changeset: r78668:b21a6e8e8f70
Date: 2015-07-26 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/b21a6e8e8f70/

Log:	(fijal, arigo)

	Argh, bug and fix

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -687,12 +687,12 @@
 class __extend__(pairtype(SomeStringBuilder, SomeStringBuilder)):
 
     def union((obj1, obj2)):
-        return obj1
+        return SomeStringBuilder()
 
 class __extend__(pairtype(SomeUnicodeBuilder, SomeUnicodeBuilder)):
 
     def union((obj1, obj2)):
-        return obj1
+        return SomeUnicodeBuilder()
 
 class PrebuiltStringBuilderEntry(ExtRegistryEntry):
     _type_ = StringBuilder
diff --git a/rpython/rtyper/test/test_rbuilder.py b/rpython/rtyper/test/test_rbuilder.py
--- a/rpython/rtyper/test/test_rbuilder.py
+++ b/rpython/rtyper/test/test_rbuilder.py
@@ -217,12 +217,17 @@
 
     def test_string_builder_union(self):
         s = StringBuilder()
+        s.append('foo')
 
         def f(i):
             if i % 2:
                 s2 = StringBuilder()
+                s2.append('x')
             else:
                 s2 = s
-            return s2.build()
+            return len(s2.build())
 
-        self.interpret(f, [3])
+        res = self.interpret(f, [33])
+        assert res == 1
+        res = self.interpret(f, [34])
+        assert res == 3

From noreply at buildbot.pypy.org  Sun Jul 26 16:32:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 26 Jul 2015 16:32:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Workaround: don't accept "--no-jit" as
 the useless option "--no-jittest"
Message-ID: <20150726143214.D31F81C1236@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78669:e6a046f8d969
Date: 2015-07-26 15:34 +0100
http://bitbucket.org/pypy/pypy/changeset/e6a046f8d969/

Log:	Workaround: don't accept "--no-jit" as the useless option "--no-
	jittest"

diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -24,7 +24,7 @@
     ("annotate", "do type inference", "-a --annotate", ""),
     ("rtype", "do rtyping", "-t --rtype", ""),
     ("pyjitpl", "JIT generation step", "--pyjitpl", ""),
-    ("jittest", "JIT test with llgraph backend", "--jittest", ""),
+    ("jittest", "JIT test with llgraph backend", "--pyjittest", ""),
     ("backendopt", "do backend optimizations", "--backendopt", ""),
     ("source", "create source", "-s --source", ""),
     ("compile", "compile", "-c --compile", " (default goal)"),

From noreply at buildbot.pypy.org  Sun Jul 26 17:04:46 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 26 Jul 2015 17:04:46 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: refactor .cumsum() and .cumprod()
Message-ID: <20150726150446.2D8601C13F7@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78670:985e46cb47d8
Date: 2015-07-26 16:04 +0100
http://bitbucket.org/pypy/pypy/changeset/985e46cb47d8/

Log:	refactor .cumsum() and .cumprod()

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1145,14 +1145,14 @@
 
     # ----------------------- reduce -------------------------------
 
-    def _reduce_ufunc_impl(ufunc_name, name, variant=ufuncs.REDUCE, bool_result=False):
+    def _reduce_ufunc_impl(ufunc_name, name, bool_result=False):
         @unwrap_spec(keepdims=bool)
         def impl(self, space, w_axis=None, w_dtype=None, w_out=None, keepdims=False):
             out = out_converter(space, w_out)
             if bool_result:
                 w_dtype = descriptor.get_dtype_cache(space).w_booldtype
             return getattr(ufuncs.get(space), ufunc_name).reduce(
-                space, self, w_axis, keepdims, out, w_dtype, variant=variant)
+                space, self, w_axis, keepdims, out, w_dtype)
         impl.__name__ = name
         return impl
 
@@ -1163,8 +1163,23 @@
     descr_all = _reduce_ufunc_impl('logical_and', "descr_all", bool_result=True)
     descr_any = _reduce_ufunc_impl('logical_or', "descr_any", bool_result=True)
 
-    descr_cumsum = _reduce_ufunc_impl('add', "descr_cumsum", variant=ufuncs.ACCUMULATE)
-    descr_cumprod = _reduce_ufunc_impl('multiply', "descr_cumprod", variant=ufuncs.ACCUMULATE)
+
+    def _accumulate_method(ufunc_name, name):
+        def method(self, space, w_axis=None, w_dtype=None, w_out=None):
+            out = out_converter(space, w_out)
+            if space.is_none(w_axis):
+                w_axis = space.wrap(0)
+                arr = self.reshape(space, space.wrap(-1))
+            else:
+                arr = self
+            ufunc = getattr(ufuncs.get(space), ufunc_name)
+            return ufunc.reduce(space, arr, w_axis, False, out, w_dtype,
+                                variant=ufuncs.ACCUMULATE)
+        method.__name__ = name
+        return method
+
+    descr_cumsum = _accumulate_method('add', 'descr_cumsum')
+    descr_cumprod = _accumulate_method('multiply', 'descr_cumprod')
 
     def _reduce_argmax_argmin_impl(raw_name):
         op_name = "arg%s" % raw_name
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -159,7 +159,7 @@
         return retval
 
     def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None):
-        if space.is_none(w_axis):
+        if w_axis is None:
             w_axis = space.wrap(0)
         out = out_converter(space, w_out)
         return self.reduce(space, w_obj, w_axis, True, #keepdims must be true
@@ -243,7 +243,9 @@
         if obj.is_scalar():
             return obj.get_scalar_value()
         shapelen = len(obj_shape)
+
         if space.is_none(w_axis):
+            axes = range(shapelen)
             axis = maxint
         else:
             if space.isinstance_w(w_axis, space.w_tuple) and space.len_w(w_axis) == 1:
@@ -253,6 +255,7 @@
                 raise oefmt(space.w_ValueError, "'axis' entry is out of bounds")
             if axis < 0:
                 axis += shapelen
+            axes = [axis]
         assert axis >= 0
         dtype = decode_w_dtype(space, dtype)
 
@@ -282,9 +285,14 @@
                             "which has no identity", self.name)
 
         if variant == ACCUMULATE:
+            if len(axes) != 1:
+                raise oefmt(space.w_ValueError,
+                    "accumulate does not allow multiple axes")
+            axis = axes[0]
+            assert axis >= 0
             dtype = self.find_binop_type(space, dtype)
             call__array_wrap__ = True
-            if shapelen > 1 and axis < shapelen:
+            if shapelen > 1:
                 temp = None
                 shape = obj_shape[:]
                 temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]

From noreply at buildbot.pypy.org  Sun Jul 26 17:22:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 26 Jul 2015 17:22:02 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: A large mostly-no-op commit
 extracting W_AbstractUnicodeObject
Message-ID: <20150726152202.583F21C0622@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-strbuf
Changeset: r78671:cf9f0f7412c0
Date: 2015-07-26 17:22 +0200
http://bitbucket.org/pypy/pypy/changeset/cf9f0f7412c0/

Log:	A large mostly-no-op commit extracting W_AbstractUnicodeObject

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -49,69 +49,89 @@
 
     def descr_add(self, space, w_other):
         """x.__add__(y) <==> x+y"""
+        raise NotImplementedError
 
     def descr_contains(self, space, w_sub):
         """x.__contains__(y) <==> y in x"""
+        raise NotImplementedError
 
     def descr_eq(self, space, w_other):
         """x.__eq__(y) <==> x==y"""
+        raise NotImplementedError
 
     def descr__format__(self, space, w_format_spec):
         """S.__format__(format_spec) -> string
 
         Return a formatted version of S as described by format_spec.
         """
+        raise NotImplementedError
 
     def descr_ge(self, space, w_other):
         """x.__ge__(y) <==> x>=y"""
+        raise NotImplementedError
 
     def descr_getitem(self, space, w_index):
         """x.__getitem__(y) <==> x[y]"""
+        raise NotImplementedError
 
     def descr_getnewargs(self, space):
         ""
+        raise NotImplementedError
 
     def descr_getslice(self, space, w_start, w_stop):
         """x.__getslice__(i, j) <==> x[i:j]
 
         Use of negative indices is not supported.
         """
+        raise NotImplementedError
 
     def descr_gt(self, space, w_other):
         """x.__gt__(y) <==> x>y"""
+        raise NotImplementedError
 
     def descr_hash(self, space):
         """x.__hash__() <==> hash(x)"""
+        raise NotImplementedError
 
     def descr_le(self, space, w_other):
         """x.__le__(y) <==> x<=y"""
+        raise NotImplementedError
 
     def descr_len(self, space):
         """x.__len__() <==> len(x)"""
+        raise NotImplementedError
 
     def descr_lt(self, space, w_other):
         """x.__lt__(y) <==> x<y"""
+        raise NotImplementedError
 
     def descr_mod(self, space, w_values):
         """x.__mod__(y) <==> x%y"""
+        raise NotImplementedError
 
     def descr_mul(self, space, w_times):
         """x.__mul__(n) <==> x*n"""
+        raise NotImplementedError
 
     def descr_ne(self, space, w_other):
         """x.__ne__(y) <==> x!=y"""
+        raise NotImplementedError
 
     def descr_repr(self, space):
         """x.__repr__() <==> repr(x)"""
+        raise NotImplementedError
 
     def descr_rmod(self, space, w_values):
         """x.__rmod__(y) <==> y%x"""
+        raise NotImplementedError
 
     def descr_rmul(self, space, w_times):
         """x.__rmul__(n) <==> n*x"""
+        raise NotImplementedError
 
     def descr_str(self, space):
         """x.__str__() <==> str(x)"""
+        raise NotImplementedError
 
     def descr_capitalize(self, space):
         """S.capitalize() -> string
@@ -119,6 +139,7 @@
         Return a capitalized version of S, i.e. make the first character
         have upper case and the rest lower case.
         """
+        raise NotImplementedError
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_center(self, space, width, w_fillchar):
@@ -127,6 +148,7 @@
         Return S centered in a string of length width. Padding is
         done using the specified fill character (default is a space).
         """
+        raise NotImplementedError
 
     def descr_count(self, space, w_sub, w_start=None, w_end=None):
         """S.count(sub[, start[, end]]) -> int
@@ -135,6 +157,7 @@
         string S[start:end].  Optional arguments start and end are interpreted
         as in slice notation.
         """
+        raise NotImplementedError
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
         """S.decode(encoding=None, errors='strict') -> object
@@ -146,6 +169,7 @@
         as well as any other name registered with codecs.register_error that is
         able to handle UnicodeDecodeErrors.
         """
+        raise NotImplementedError
 
     def descr_encode(self, space, w_encoding=None, w_errors=None):
         """S.encode(encoding=None, errors='strict') -> object
@@ -157,6 +181,7 @@
         'xmlcharrefreplace' as well as any other name registered with
         codecs.register_error that is able to handle UnicodeEncodeErrors.
         """
+        raise NotImplementedError
 
     def descr_endswith(self, space, w_suffix, w_start=None, w_end=None):
         """S.endswith(suffix[, start[, end]]) -> bool
@@ -166,6 +191,7 @@
         With optional end, stop comparing S at that position.
         suffix can also be a tuple of strings to try.
         """
+        raise NotImplementedError
 
     @unwrap_spec(tabsize=int)
     def descr_expandtabs(self, space, tabsize=8):
@@ -174,6 +200,7 @@
         Return a copy of S where all tab characters are expanded using spaces.
         If tabsize is not given, a tab size of 8 characters is assumed.
         """
+        raise NotImplementedError
 
     def descr_find(self, space, w_sub, w_start=None, w_end=None):
         """S.find(sub[, start[, end]]) -> int
@@ -184,6 +211,7 @@
 
         Return -1 on failure.
         """
+        raise NotImplementedError
 
     def descr_format(self, space, __args__):
         """S.format(*args, **kwargs) -> string
@@ -191,12 +219,14 @@
         Return a formatted version of S, using substitutions from args and
         kwargs.  The substitutions are identified by braces ('{' and '}').
         """
+        raise NotImplementedError
 
     def descr_index(self, space, w_sub, w_start=None, w_end=None):
         """S.index(sub[, start[, end]]) -> int
 
         Like S.find() but raise ValueError when the substring is not found.
         """
+        raise NotImplementedError
 
     def descr_isalnum(self, space):
         """S.isalnum() -> bool
@@ -204,6 +234,7 @@
         Return True if all characters in S are alphanumeric
         and there is at least one character in S, False otherwise.
         """
+        raise NotImplementedError
 
     def descr_isalpha(self, space):
         """S.isalpha() -> bool
@@ -211,6 +242,7 @@
         Return True if all characters in S are alphabetic
         and there is at least one character in S, False otherwise.
         """
+        raise NotImplementedError
 
     def descr_isdigit(self, space):
         """S.isdigit() -> bool
@@ -218,6 +250,7 @@
         Return True if all characters in S are digits
         and there is at least one character in S, False otherwise.
         """
+        raise NotImplementedError
 
     def descr_islower(self, space):
         """S.islower() -> bool
@@ -225,6 +258,7 @@
         Return True if all cased characters in S are lowercase and there is
         at least one cased character in S, False otherwise.
         """
+        raise NotImplementedError
 
     def descr_isspace(self, space):
         """S.isspace() -> bool
@@ -232,6 +266,7 @@
         Return True if all characters in S are whitespace
         and there is at least one character in S, False otherwise.
         """
+        raise NotImplementedError
 
     def descr_istitle(self, space):
         """S.istitle() -> bool
@@ -241,6 +276,7 @@
         characters and lowercase characters only cased ones. Return False
         otherwise.
         """
+        raise NotImplementedError
 
     def descr_isupper(self, space):
         """S.isupper() -> bool
@@ -248,6 +284,7 @@
         Return True if all cased characters in S are uppercase and there is
         at least one cased character in S, False otherwise.
         """
+        raise NotImplementedError
 
     def descr_join(self, space, w_list):
         """S.join(iterable) -> string
@@ -255,6 +292,7 @@
         Return a string which is the concatenation of the strings in the
         iterable.  The separator between elements is S.
         """
+        raise NotImplementedError
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_ljust(self, space, width, w_fillchar):
@@ -263,12 +301,14 @@
         Return S left-justified in a string of length width. Padding is
         done using the specified fill character (default is a space).
         """
+        raise NotImplementedError
 
     def descr_lower(self, space):
         """S.lower() -> string
 
         Return a copy of the string S converted to lowercase.
         """
+        raise NotImplementedError
 
     def descr_lstrip(self, space, w_chars=None):
         """S.lstrip([chars]) -> string or unicode
@@ -277,6 +317,7 @@
         If chars is given and not None, remove characters in chars instead.
         If chars is unicode, S will be converted to unicode before stripping
         """
+        raise NotImplementedError
 
     def descr_partition(self, space, w_sub):
         """S.partition(sep) -> (head, sep, tail)
@@ -285,6 +326,7 @@
         the separator itself, and the part after it.  If the separator is not
         found, return S and two empty strings.
         """
+        raise NotImplementedError
 
     @unwrap_spec(count=int)
     def descr_replace(self, space, w_old, w_new, count=-1):
@@ -294,6 +336,7 @@
         old replaced by new.  If the optional argument count is
         given, only the first count occurrences are replaced.
         """
+        raise NotImplementedError
 
     def descr_rfind(self, space, w_sub, w_start=None, w_end=None):
         """S.rfind(sub[, start[, end]]) -> int
@@ -304,12 +347,14 @@
 
         Return -1 on failure.
         """
+        raise NotImplementedError
 
     def descr_rindex(self, space, w_sub, w_start=None, w_end=None):
         """S.rindex(sub[, start[, end]]) -> int
 
         Like S.rfind() but raise ValueError when the substring is not found.
         """
+        raise NotImplementedError
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_rjust(self, space, width, w_fillchar):
@@ -318,6 +363,7 @@
         Return S right-justified in a string of length width. Padding is
         done using the specified fill character (default is a space).
         """
+        raise NotImplementedError
 
     def descr_rpartition(self, space, w_sub):
         """S.rpartition(sep) -> (head, sep, tail)
@@ -326,6 +372,7 @@
         the part before it, the separator itself, and the part after it.  If
         the separator is not found, return two empty strings and S.
         """
+        raise NotImplementedError
 
     @unwrap_spec(maxsplit=int)
     def descr_rsplit(self, space, w_sep=None, maxsplit=-1):
@@ -337,6 +384,7 @@
         done. If sep is not specified or is None, any whitespace string
         is a separator.
         """
+        raise NotImplementedError
 
     def descr_rstrip(self, space, w_chars=None):
         """S.rstrip([chars]) -> string or unicode
@@ -345,6 +393,7 @@
         If chars is given and not None, remove characters in chars instead.
         If chars is unicode, S will be converted to unicode before stripping
         """
+        raise NotImplementedError
 
     @unwrap_spec(maxsplit=int)
     def descr_split(self, space, w_sep=None, maxsplit=-1):
@@ -356,6 +405,7 @@
         whitespace string is a separator and empty strings are removed
         from the result.
         """
+        raise NotImplementedError
 
     @unwrap_spec(keepends=bool)
     def descr_splitlines(self, space, keepends=False):
@@ -365,6 +415,7 @@
         Line breaks are not included in the resulting list unless keepends
         is given and true.
         """
+        raise NotImplementedError
 
     def descr_startswith(self, space, w_prefix, w_start=None, w_end=None):
         """S.startswith(prefix[, start[, end]]) -> bool
@@ -374,6 +425,7 @@
         With optional end, stop comparing S at that position.
         prefix can also be a tuple of strings to try.
         """
+        raise NotImplementedError
 
     def descr_strip(self, space, w_chars=None):
         """S.strip([chars]) -> string or unicode
@@ -383,6 +435,7 @@
         If chars is given and not None, remove characters in chars instead.
         If chars is unicode, S will be converted to unicode before stripping
         """
+        raise NotImplementedError
 
     def descr_swapcase(self, space):
         """S.swapcase() -> string
@@ -390,6 +443,7 @@
         Return a copy of the string S with uppercase characters
         converted to lowercase and vice versa.
         """
+        raise NotImplementedError
 
     def descr_title(self, space):
         """S.title() -> string
@@ -397,6 +451,7 @@
         Return a titlecased version of S, i.e. words start with uppercase
         characters, all remaining cased characters have lowercase.
         """
+        raise NotImplementedError
 
     @unwrap_spec(w_deletechars=WrappedDefault(''))
     def descr_translate(self, space, w_table, w_deletechars):
@@ -409,12 +464,14 @@
         If the table argument is None, no translation is applied and
         the operation simply removes the characters in deletechars.
         """
+        raise NotImplementedError
 
     def descr_upper(self, space):
         """S.upper() -> string
 
         Return a copy of the string S converted to uppercase.
         """
+        raise NotImplementedError
 
     @unwrap_spec(width=int)
     def descr_zfill(self, space, width):
@@ -423,6 +480,7 @@
         Pad a numeric string S with zeros on the left, to fill a field
         of the specified width. The string S is never truncated.
         """
+        raise NotImplementedError
 
     def writebuf_w(self, space):
         raise OperationError(space.w_TypeError, space.wrap(
@@ -439,6 +497,16 @@
                         "found", len(value))
         return space.wrap(ord(value[0]))
 
+    def descr_formatter_parser(self, space):
+        from pypy.objspace.std.newformat import str_template_formatter
+        tformat = str_template_formatter(space, space.str_w(self))
+        return tformat.formatter_parser()
+
+    def descr_formatter_field_name_split(self, space):
+        from pypy.objspace.std.newformat import str_template_formatter
+        tformat = str_template_formatter(space, space.str_w(self))
+        return tformat.formatter_field_name_split()
+
 
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
@@ -832,16 +900,6 @@
     def descr_upper(self, space):
         return W_BytesObject(self._value.upper())
 
-    def descr_formatter_parser(self, space):
-        from pypy.objspace.std.newformat import str_template_formatter
-        tformat = str_template_formatter(space, space.str_w(self))
-        return tformat.formatter_parser()
-
-    def descr_formatter_field_name_split(self, space):
-        from pypy.objspace.std.newformat import str_template_formatter
-        tformat = str_template_formatter(space, space.str_w(self))
-        return tformat.formatter_field_name_split()
-
 
 def _create_list_from_bytes(value):
     # need this helper function to allow the jit to look inside and inline
@@ -950,9 +1008,10 @@
     __mod__ = interpindirect2app(W_AbstractBytesObject.descr_mod),
     __getnewargs__ = interpindirect2app(
         W_AbstractBytesObject.descr_getnewargs),
-    _formatter_parser = interp2app(W_BytesObject.descr_formatter_parser),
+    _formatter_parser =
+        interp2app(W_AbstractBytesObject.descr_formatter_parser),
     _formatter_field_name_split =
-        interp2app(W_BytesObject.descr_formatter_field_name_split),
+        interp2app(W_AbstractBytesObject.descr_formatter_field_name_split),
 )
 W_BytesObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ b/pypy/objspace/std/test/test_strbufobject.py
@@ -94,3 +94,8 @@
         a = 'abc'
         a += 'bc'
         print a
+
+    def test_formatter_parser(self):
+        a = 'abc'
+        a += 'bc'
+        assert list(a._formatter_parser()) == [('abcbc', None, None, None)]
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -11,7 +11,8 @@
 from pypy.interpreter import unicodehelper
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
+from pypy.interpreter.gateway import (
+    WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
 from pypy.interpreter.typedef import TypeDef
 from pypy.module.unicodedata import unicodedb
 from pypy.objspace.std import newformat
@@ -24,7 +25,488 @@
            'unicode_from_string', 'unicode_to_decimal_w']
 
 
-class W_UnicodeObject(W_Root):
+class W_AbstractUnicodeObject(W_Root):
+    __slots__ = ()
+
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_AbstractUnicodeObject):
+            return False
+        if self is w_other:
+            return True
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return False
+        return space.unicode_w(self) is space.unicode_w(w_other)
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class:
+            return None
+        return space.wrap(compute_unique_id(space.unicode_w(self)))
+
+    def str_w(self, space):
+        return space.str_w(space.str(self))
+
+    charbuf_w = str_w
+
+    def descr_add(self, space, w_other):
+        """x.__add__(y) <==> x+y"""
+        raise NotImplementedError
+
+    def descr_contains(self, space, w_sub):
+        """x.__contains__(y) <==> y in x"""
+        raise NotImplementedError
+
+    def descr_eq(self, space, w_other):
+        """x.__eq__(y) <==> x==y"""
+        raise NotImplementedError
+
+    def descr__format__(self, space, w_format_spec):
+        """S.__format__(format_spec) -> string
+
+        Return a formatted version of S as described by format_spec.
+        """
+        raise NotImplementedError
+
+    def descr_ge(self, space, w_other):
+        """x.__ge__(y) <==> x>=y"""
+        raise NotImplementedError
+
+    def descr_getitem(self, space, w_index):
+        """x.__getitem__(y) <==> x[y]"""
+        raise NotImplementedError
+
+    def descr_getnewargs(self, space):
+        ""
+        raise NotImplementedError
+
+    def descr_getslice(self, space, w_start, w_stop):
+        """x.__getslice__(i, j) <==> x[i:j]
+
+        Use of negative indices is not supported.
+        """
+        raise NotImplementedError
+
+    def descr_gt(self, space, w_other):
+        """x.__gt__(y) <==> x>y"""
+        raise NotImplementedError
+
+    def descr_hash(self, space):
+        """x.__hash__() <==> hash(x)"""
+        raise NotImplementedError
+
+    def descr_le(self, space, w_other):
+        """x.__le__(y) <==> x<=y"""
+        raise NotImplementedError
+
+    def descr_len(self, space):
+        """x.__len__() <==> len(x)"""
+        raise NotImplementedError
+
+    def descr_lt(self, space, w_other):
+        """x.__lt__(y) <==> x<y"""
+        raise NotImplementedError
+
+    def descr_mod(self, space, w_values):
+        """x.__mod__(y) <==> x%y"""
+        raise NotImplementedError
+
+    def descr_mul(self, space, w_times):
+        """x.__mul__(n) <==> x*n"""
+        raise NotImplementedError
+
+    def descr_ne(self, space, w_other):
+        """x.__ne__(y) <==> x!=y"""
+        raise NotImplementedError
+
+    def descr_repr(self, space):
+        """x.__repr__() <==> repr(x)"""
+        raise NotImplementedError
+
+    def descr_rmod(self, space, w_values):
+        """x.__rmod__(y) <==> y%x"""
+        raise NotImplementedError
+
+    def descr_rmul(self, space, w_times):
+        """x.__rmul__(n) <==> n*x"""
+        raise NotImplementedError
+
+    def descr_str(self, space):
+        """x.__str__() <==> str(x)"""
+        raise NotImplementedError
+
+    def descr_capitalize(self, space):
+        """S.capitalize() -> unicode
+
+        Return a capitalized version of S, i.e. make the first character
+        have upper case and the rest lower case.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
+    def descr_center(self, space, width, w_fillchar):
+        """S.center(width[, fillchar]) -> unicode
+
+        Return S centered in a Unicode string of length width. Padding is
+        done using the specified fill character (default is a space).
+        """
+        raise NotImplementedError
+
+    def descr_count(self, space, w_sub, w_start=None, w_end=None):
+        """S.count(sub[, start[, end]]) -> int
+
+        Return the number of non-overlapping occurrences of substring sub in
+        Unicode string S[start:end].  Optional arguments start and end are
+        interpreted as in slice notation.
+        """
+        raise NotImplementedError
+
+    def descr_decode(self, space, w_encoding=None, w_errors=None):
+        """S.decode(encoding=None, errors='strict') -> string or unicode
+
+        Decode S using the codec registered for encoding. encoding defaults
+        to the default encoding. errors may be given to set a different error
+        handling scheme. Default is 'strict' meaning that encoding errors raise
+        a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'
+        as well as any other name registered with codecs.register_error that is
+        able to handle UnicodeDecodeErrors.
+        """
+        raise NotImplementedError
+
+    def descr_encode(self, space, w_encoding=None, w_errors=None):
+        """S.encode(encoding=None, errors='strict') -> string or unicode
+
+        Encode S using the codec registered for encoding. encoding defaults
+        to the default encoding. errors may be given to set a different error
+        handling scheme. Default is 'strict' meaning that encoding errors raise
+        a UnicodeEncodeError. Other possible values are 'ignore', 'replace' and
+        'xmlcharrefreplace' as well as any other name registered with
+        codecs.register_error that can handle UnicodeEncodeErrors.
+        """
+        raise NotImplementedError
+
+    def descr_endswith(self, space, w_suffix, w_start=None, w_end=None):
+        """S.endswith(suffix[, start[, end]]) -> bool
+
+        Return True if S ends with the specified suffix, False otherwise.
+        With optional start, test S beginning at that position.
+        With optional end, stop comparing S at that position.
+        suffix can also be a tuple of strings to try.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(tabsize=int)
+    def descr_expandtabs(self, space, tabsize=8):
+        """S.expandtabs([tabsize]) -> unicode
+
+        Return a copy of S where all tab characters are expanded using spaces.
+        If tabsize is not given, a tab size of 8 characters is assumed.
+        """
+        raise NotImplementedError
+
+    def descr_find(self, space, w_sub, w_start=None, w_end=None):
+        """S.find(sub[, start[, end]]) -> int
+
+        Return the lowest index in S where substring sub is found,
+        such that sub is contained within S[start:end].  Optional
+        arguments start and end are interpreted as in slice notation.
+
+        Return -1 on failure.
+        """
+        raise NotImplementedError
+
+    def descr_format(self, space, __args__):
+        """S.format(*args, **kwargs) -> unicode
+
+        Return a formatted version of S, using substitutions from args and
+        kwargs.  The substitutions are identified by braces ('{' and '}').
+        """
+        raise NotImplementedError
+
+    def descr_index(self, space, w_sub, w_start=None, w_end=None):
+        """S.index(sub[, start[, end]]) -> int
+
+        Like S.find() but raise ValueError when the substring is not found.
+        """
+        raise NotImplementedError
+
+    def descr_isalnum(self, space):
+        """S.isalnum() -> bool
+
+        Return True if all characters in S are alphanumeric
+        and there is at least one character in S, False otherwise.
+        """
+        raise NotImplementedError
+
+    def descr_isalpha(self, space):
+        """S.isalpha() -> bool
+
+        Return True if all characters in S are alphabetic
+        and there is at least one character in S, False otherwise.
+        """
+        raise NotImplementedError
+
+    def descr_isdecimal(self, space):
+        """S.isdecimal() -> bool
+
+        Return True if there are only decimal characters in S,
+        False otherwise.
+        """
+        raise NotImplementedError
+
+    def descr_isdigit(self, space):
+        """S.isdigit() -> bool
+
+        Return True if all characters in S are digits
+        and there is at least one character in S, False otherwise.
+        """
+        raise NotImplementedError
+
+    def descr_islower(self, space):
+        """S.islower() -> bool
+
+        Return True if all cased characters in S are lowercase and there is
+        at least one cased character in S, False otherwise.
+        """
+        raise NotImplementedError
+
+    def descr_isnumeric(self, space):
+        """S.isnumeric() -> bool
+
+        Return True if there are only numeric characters in S,
+        False otherwise.
+        """
+        raise NotImplementedError
+
+    def descr_isspace(self, space):
+        """S.isspace() -> bool
+
+        Return True if all characters in S are whitespace
+        and there is at least one character in S, False otherwise.
+        """
+        raise NotImplementedError
+
+    def descr_istitle(self, space):
+        """S.istitle() -> bool
+
+        Return True if S is a titlecased string and there is at least one
+        character in S, i.e. upper- and titlecase characters may only
+        follow uncased characters and lowercase characters only cased ones.
+        Return False otherwise.
+        """
+        raise NotImplementedError
+
+    def descr_isupper(self, space):
+        """S.isupper() -> bool
+
+        Return True if all cased characters in S are uppercase and there is
+        at least one cased character in S, False otherwise.
+        """
+        raise NotImplementedError
+
+    def descr_join(self, space, w_list):
+        """S.join(iterable) -> unicode
+
+        Return a string which is the concatenation of the strings in the
+        iterable.  The separator between elements is S.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
+    def descr_ljust(self, space, width, w_fillchar):
+        """S.ljust(width[, fillchar]) -> int
+
+        Return S left-justified in a Unicode string of length width. Padding is
+        done using the specified fill character (default is a space).
+        """
+        raise NotImplementedError
+
+    def descr_lower(self, space):
+        """S.lower() -> unicode
+
+        Return a copy of the string S converted to lowercase.
+        """
+        raise NotImplementedError
+
+    def descr_lstrip(self, space, w_chars=None):
+        """S.lstrip([chars]) -> unicode
+
+        Return a copy of the string S with leading whitespace removed.
+        If chars is given and not None, remove characters in chars instead.
+        If chars is a str, it will be converted to unicode before stripping
+        """
+        raise NotImplementedError
+
+    def descr_partition(self, space, w_sub):
+        """S.partition(sep) -> (head, sep, tail)
+
+        Search for the separator sep in S, and return the part before it,
+        the separator itself, and the part after it.  If the separator is not
+        found, return S and two empty strings.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(count=int)
+    def descr_replace(self, space, w_old, w_new, count=-1):
+        """S.replace(old, new[, count]) -> unicode
+
+        Return a copy of S with all occurrences of substring
+        old replaced by new.  If the optional argument count is
+        given, only the first count occurrences are replaced.
+        """
+        raise NotImplementedError
+
+    def descr_rfind(self, space, w_sub, w_start=None, w_end=None):
+        """S.rfind(sub[, start[, end]]) -> int
+
+        Return the highest index in S where substring sub is found,
+        such that sub is contained within S[start:end].  Optional
+        arguments start and end are interpreted as in slice notation.
+
+        Return -1 on failure.
+        """
+        raise NotImplementedError
+
+    def descr_rindex(self, space, w_sub, w_start=None, w_end=None):
+        """S.rindex(sub[, start[, end]]) -> int
+
+        Like S.rfind() but raise ValueError when the substring is not found.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
+    def descr_rjust(self, space, width, w_fillchar):
+        """S.rjust(width[, fillchar]) -> unicode
+
+        Return S right-justified in a Unicode string of length width. Padding
+        is done using the specified fill character (default is a space).
+        """
+        raise NotImplementedError
+
+    def descr_rpartition(self, space, w_sub):
+        """S.rpartition(sep) -> (head, sep, tail)
+
+        Search for the separator sep in S, starting at the end of S, and return
+        the part before it, the separator itself, and the part after it.  If
+        the separator is not found, return two empty strings and S.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(maxsplit=int)
+    def descr_rsplit(self, space, w_sep=None, maxsplit=-1):
+        """S.rsplit(sep=None, maxsplit=-1) -> list of strings
+
+        Return a list of the words in S, using sep as the
+        delimiter string, starting at the end of the string and
+        working to the front.  If maxsplit is given, at most maxsplit
+        splits are done. If sep is not specified, any whitespace string
+        is a separator.
+        """
+        raise NotImplementedError
+
+    def descr_rstrip(self, space, w_chars=None):
+        """S.rstrip([chars]) -> unicode
+
+        Return a copy of the string S with trailing whitespace removed.
+        If chars is given and not None, remove characters in chars instead.
+        If chars is a str, it will be converted to unicode before stripping
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(maxsplit=int)
+    def descr_split(self, space, w_sep=None, maxsplit=-1):
+        """S.split(sep=None, maxsplit=-1) -> list of strings
+
+        Return a list of the words in S, using sep as the
+        delimiter string.  If maxsplit is given, at most maxsplit
+        splits are done. If sep is not specified or is None, any
+        whitespace string is a separator and empty strings are
+        removed from the result.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(keepends=bool)
+    def descr_splitlines(self, space, keepends=False):
+        """S.splitlines(keepends=False) -> list of strings
+
+        Return a list of the lines in S, breaking at line boundaries.
+        Line breaks are not included in the resulting list unless keepends
+        is given and true.
+        """
+        raise NotImplementedError
+
+    def descr_startswith(self, space, w_prefix, w_start=None, w_end=None):
+        """S.startswith(prefix[, start[, end]]) -> bool
+
+        Return True if S starts with the specified prefix, False otherwise.
+        With optional start, test S beginning at that position.
+        With optional end, stop comparing S at that position.
+        prefix can also be a tuple of strings to try.
+        """
+        raise NotImplementedError
+
+    def descr_strip(self, space, w_chars=None):
+        """S.strip([chars]) -> unicode
+
+        Return a copy of the string S with leading and trailing
+        whitespace removed.
+        If chars is given and not None, remove characters in chars instead.
+        If chars is a str, it will be converted to unicode before stripping
+        """
+        raise NotImplementedError
+
+    def descr_swapcase(self, space):
+        """S.swapcase() -> unicode
+
+        Return a copy of S with uppercase characters converted to lowercase
+        and vice versa.
+        """
+        raise NotImplementedError
+
+    def descr_title(self, space):
+        """S.title() -> unicode
+
+        Return a titlecased version of S, i.e. words start with title case
+        characters, all remaining cased characters have lower case.
+        """
+        raise NotImplementedError
+
+    def descr_translate(self, space, w_table):
+        """S.translate(table) -> unicode
+
+        Return a copy of the string S, where all characters have been mapped
+        through the given translation table, which must be a mapping of
+        Unicode ordinals to Unicode ordinals, Unicode strings or None.
+        Unmapped characters are left untouched. Characters mapped to None
+        are deleted.
+        """
+        raise NotImplementedError
+
+    def descr_upper(self, space):
+        """S.upper() -> unicode
+
+        Return a copy of S converted to uppercase.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(width=int)
+    def descr_zfill(self, space, width):
+        """S.zfill(width) -> unicode
+
+        Pad a numeric string S with zeros on the left, to fill a field
+        of the specified width. The string S is never truncated.
+        """
+
+    def descr_formatter_parser(self, space):
+        from pypy.objspace.std.newformat import unicode_template_formatter
+        tformat = unicode_template_formatter(space, space.unicode_w(self))
+        return tformat.formatter_parser()
+
+    def descr_formatter_field_name_split(self, space):
+        from pypy.objspace.std.newformat import unicode_template_formatter
+        tformat = unicode_template_formatter(space, space.unicode_w(self))
+        return tformat.formatter_field_name_split()
+
+
+class W_UnicodeObject(W_AbstractUnicodeObject):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
 
@@ -45,23 +527,6 @@
             return w_self
         return W_UnicodeObject(w_self._value)
 
-    def is_w(self, space, w_other):
-        if not isinstance(w_other, W_UnicodeObject):
-            return False
-        if self is w_other:
-            return True
-        if self.user_overridden_class or w_other.user_overridden_class:
-            return False
-        return space.unicode_w(self) is space.unicode_w(w_other)
-
-    def immutable_unique_id(self, space):
-        if self.user_overridden_class:
-            return None
-        return space.wrap(compute_unique_id(space.unicode_w(self)))
-
-    def str_w(self, space):
-        return space.str_w(space.str(self))
-
     def unicode_w(self, space):
         return self._value
 
@@ -76,8 +541,6 @@
         raise OperationError(space.w_TypeError, space.wrap(
             "cannot use unicode as modifiable buffer"))
 
-    charbuf_w = str_w
-
     def listview_unicode(w_self):
         return _create_list_from_unicode(w_self._value)
 
@@ -357,16 +820,6 @@
             return 0
         return 1
 
-    def descr_formatter_parser(self, space):
-        from pypy.objspace.std.newformat import unicode_template_formatter
-        tformat = unicode_template_formatter(space, space.unicode_w(self))
-        return tformat.formatter_parser()
-
-    def descr_formatter_field_name_split(self, space):
-        from pypy.objspace.std.newformat import unicode_template_formatter
-        tformat = unicode_template_formatter(space, space.unicode_w(self))
-        return tformat.formatter_field_name_split()
-
     def descr_isdecimal(self, space):
         return self._is_generic(space, '_isdecimal')
 
@@ -555,535 +1008,86 @@
         return unicode_from_encoded_object(space, w_str, "ascii", "strict")
 
 
-class UnicodeDocstrings:
-    """unicode(object='') -> unicode object
+W_UnicodeObject.typedef = TypeDef(
+    "unicode", basestring_typedef,
+    __new__ = interp2app(W_UnicodeObject.descr_new),
+    __doc__ = """unicode(object='') -> unicode object
     unicode(string[, encoding[, errors]]) -> unicode object
 
     Create a new Unicode object from the given encoded string.
     encoding defaults to the current default string encoding.
     errors can be 'strict', 'replace' or 'ignore' and defaults to 'strict'.
+    """,
 
-    """
+    __repr__ = interpindirect2app(W_AbstractUnicodeObject.descr_repr),
+    __str__ = interpindirect2app(W_AbstractUnicodeObject.descr_str),
+    __hash__ = interpindirect2app(W_AbstractUnicodeObject.descr_hash),
 
-    def __add__():
-        """x.__add__(y) <==> x+y"""
+    __eq__ = interpindirect2app(W_AbstractUnicodeObject.descr_eq),
+    __ne__ = interpindirect2app(W_AbstractUnicodeObject.descr_ne),
+    __lt__ = interpindirect2app(W_AbstractUnicodeObject.descr_lt),
+    __le__ = interpindirect2app(W_AbstractUnicodeObject.descr_le),
+    __gt__ = interpindirect2app(W_AbstractUnicodeObject.descr_gt),
+    __ge__ = interpindirect2app(W_AbstractUnicodeObject.descr_ge),
 
-    def __contains__():
-        """x.__contains__(y) <==> y in x"""
+    __len__ = interpindirect2app(W_AbstractUnicodeObject.descr_len),
+    __contains__ = interpindirect2app(W_AbstractUnicodeObject.descr_contains),
 
-    def __eq__():
-        """x.__eq__(y) <==> x==y"""
+    __add__ = interpindirect2app(W_AbstractUnicodeObject.descr_add),
+    __mul__ = interpindirect2app(W_AbstractUnicodeObject.descr_mul),
+    __rmul__ = interpindirect2app(W_AbstractUnicodeObject.descr_mul),
 
-    def __format__():
-        """S.__format__(format_spec) -> unicode
+    __getitem__ = interpindirect2app(W_AbstractUnicodeObject.descr_getitem),
+    __getslice__ = interpindirect2app(W_AbstractUnicodeObject.descr_getslice),
 
-        Return a formatted version of S as described by format_spec.
-        """
+    capitalize = interpindirect2app(W_AbstractUnicodeObject.descr_capitalize),
+    center = interpindirect2app(W_AbstractUnicodeObject.descr_center),
+    count = interpindirect2app(W_AbstractUnicodeObject.descr_count),
+    decode = interpindirect2app(W_AbstractUnicodeObject.descr_decode),
+    encode = interpindirect2app(W_AbstractUnicodeObject.descr_encode),
+    expandtabs = interpindirect2app(W_AbstractUnicodeObject.descr_expandtabs),
+    find = interpindirect2app(W_AbstractUnicodeObject.descr_find),
+    rfind = interpindirect2app(W_AbstractUnicodeObject.descr_rfind),
+    index = interpindirect2app(W_AbstractUnicodeObject.descr_index),
+    rindex = interpindirect2app(W_AbstractUnicodeObject.descr_rindex),
+    isalnum = interpindirect2app(W_AbstractUnicodeObject.descr_isalnum),
+    isalpha = interpindirect2app(W_AbstractUnicodeObject.descr_isalpha),
+    isdecimal = interpindirect2app(W_AbstractUnicodeObject.descr_isdecimal),
+    isdigit = interpindirect2app(W_AbstractUnicodeObject.descr_isdigit),
+    islower = interpindirect2app(W_AbstractUnicodeObject.descr_islower),
+    isnumeric = interpindirect2app(W_AbstractUnicodeObject.descr_isnumeric),
+    isspace = interpindirect2app(W_AbstractUnicodeObject.descr_isspace),
+    istitle = interpindirect2app(W_AbstractUnicodeObject.descr_istitle),
+    isupper = interpindirect2app(W_AbstractUnicodeObject.descr_isupper),
+    join = interpindirect2app(W_AbstractUnicodeObject.descr_join),
+    ljust = interpindirect2app(W_AbstractUnicodeObject.descr_ljust),
+    rjust = interpindirect2app(W_AbstractUnicodeObject.descr_rjust),
+    lower = interpindirect2app(W_AbstractUnicodeObject.descr_lower),
+    partition = interpindirect2app(W_AbstractUnicodeObject.descr_partition),
+    rpartition = interpindirect2app(W_AbstractUnicodeObject.descr_rpartition),
+    replace = interpindirect2app(W_AbstractUnicodeObject.descr_replace),
+    split = interpindirect2app(W_AbstractUnicodeObject.descr_split),
+    rsplit = interpindirect2app(W_AbstractUnicodeObject.descr_rsplit),
+    splitlines = interpindirect2app(W_AbstractUnicodeObject.descr_splitlines),
+    startswith = interpindirect2app(W_AbstractUnicodeObject.descr_startswith),
+    endswith = interpindirect2app(W_AbstractUnicodeObject.descr_endswith),
+    strip = interpindirect2app(W_AbstractUnicodeObject.descr_strip),
+    lstrip = interpindirect2app(W_AbstractUnicodeObject.descr_lstrip),
+    rstrip = interpindirect2app(W_AbstractUnicodeObject.descr_rstrip),
+    swapcase = interpindirect2app(W_AbstractUnicodeObject.descr_swapcase),
+    title = interpindirect2app(W_AbstractUnicodeObject.descr_title),
+    translate = interpindirect2app(W_AbstractUnicodeObject.descr_translate),
+    upper = interpindirect2app(W_AbstractUnicodeObject.descr_upper),
+    zfill = interpindirect2app(W_AbstractUnicodeObject.descr_zfill),
 
-    def __ge__():
-        """x.__ge__(y) <==> x>=y"""
-
-    def __getattribute__():
-        """x.__getattribute__('name') <==> x.name"""
-
-    def __getitem__():
-        """x.__getitem__(y) <==> x[y]"""
-
-    def __getnewargs__():
-        ""
-
-    def __getslice__():
-        """x.__getslice__(i, j) <==> x[i:j]
-
-        Use of negative indices is not supported.
-        """
-
-    def __gt__():
-        """x.__gt__(y) <==> x>y"""
-
-    def __hash__():
-        """x.__hash__() <==> hash(x)"""
-
-    def __le__():
-        """x.__le__(y) <==> x<=y"""
-
-    def __len__():
-        """x.__len__() <==> len(x)"""
-
-    def __lt__():
-        """x.__lt__(y) <==> x<y"""
-
-    def __mod__():
-        """x.__mod__(y) <==> x%y"""
-
-    def __mul__():
-        """x.__mul__(n) <==> x*n"""
-
-    def __ne__():
-        """x.__ne__(y) <==> x!=y"""
-
-    def __repr__():
-        """x.__repr__() <==> repr(x)"""
-
-    def __rmod__():
-        """x.__rmod__(y) <==> y%x"""
-
-    def __rmul__():
-        """x.__rmul__(n) <==> n*x"""
-
-    def __sizeof__():
-        """S.__sizeof__() -> size of S in memory, in bytes"""
-
-    def __str__():
-        """x.__str__() <==> str(x)"""
-
-    def capitalize():
-        """S.capitalize() -> unicode
-
-        Return a capitalized version of S, i.e. make the first character
-        have upper case and the rest lower case.
-        """
-
-    def center():
-        """S.center(width[, fillchar]) -> unicode
-
-        Return S centered in a Unicode string of length width. Padding is
-        done using the specified fill character (default is a space).
-        """
-
-    def count():
-        """S.count(sub[, start[, end]]) -> int
-
-        Return the number of non-overlapping occurrences of substring sub in
-        Unicode string S[start:end].  Optional arguments start and end are
-        interpreted as in slice notation.
-        """
-
-    def decode():
-        """S.decode(encoding=None, errors='strict') -> string or unicode
-
-        Decode S using the codec registered for encoding. encoding defaults
-        to the default encoding. errors may be given to set a different error
-        handling scheme. Default is 'strict' meaning that encoding errors raise
-        a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'
-        as well as any other name registered with codecs.register_error that is
-        able to handle UnicodeDecodeErrors.
-        """
-
-    def encode():
-        """S.encode(encoding=None, errors='strict') -> string or unicode
-
-        Encode S using the codec registered for encoding. encoding defaults
-        to the default encoding. errors may be given to set a different error
-        handling scheme. Default is 'strict' meaning that encoding errors raise
-        a UnicodeEncodeError. Other possible values are 'ignore', 'replace' and
-        'xmlcharrefreplace' as well as any other name registered with
-        codecs.register_error that can handle UnicodeEncodeErrors.
-        """
-
-    def endswith():
-        """S.endswith(suffix[, start[, end]]) -> bool
-
-        Return True if S ends with the specified suffix, False otherwise.
-        With optional start, test S beginning at that position.
-        With optional end, stop comparing S at that position.
-        suffix can also be a tuple of strings to try.
-        """
-
-    def expandtabs():
-        """S.expandtabs([tabsize]) -> unicode
-
-        Return a copy of S where all tab characters are expanded using spaces.
-        If tabsize is not given, a tab size of 8 characters is assumed.
-        """
-
-    def find():
-        """S.find(sub[, start[, end]]) -> int
-
-        Return the lowest index in S where substring sub is found,
-        such that sub is contained within S[start:end].  Optional
-        arguments start and end are interpreted as in slice notation.
-
-        Return -1 on failure.
-        """
-
-    def format():
-        """S.format(*args, **kwargs) -> unicode
-
-        Return a formatted version of S, using substitutions from args and
-        kwargs.  The substitutions are identified by braces ('{' and '}').
-        """
-
-    def index():
-        """S.index(sub[, start[, end]]) -> int
-
-        Like S.find() but raise ValueError when the substring is not found.
-        """
-
-    def isalnum():
-        """S.isalnum() -> bool
-
-        Return True if all characters in S are alphanumeric
-        and there is at least one character in S, False otherwise.
-        """
-
-    def isalpha():
-        """S.isalpha() -> bool
-
-        Return True if all characters in S are alphabetic
-        and there is at least one character in S, False otherwise.
-        """
-
-    def isdecimal():
-        """S.isdecimal() -> bool
-
-        Return True if there are only decimal characters in S,
-        False otherwise.
-        """
-
-    def isdigit():
-        """S.isdigit() -> bool
-
-        Return True if all characters in S are digits
-        and there is at least one character in S, False otherwise.
-        """
-
-    def islower():
-        """S.islower() -> bool
-
-        Return True if all cased characters in S are lowercase and there is
-        at least one cased character in S, False otherwise.
-        """
-
-    def isnumeric():
-        """S.isnumeric() -> bool
-
-        Return True if there are only numeric characters in S,
-        False otherwise.
-        """
-
-    def isspace():
-        """S.isspace() -> bool
-
-        Return True if all characters in S are whitespace
-        and there is at least one character in S, False otherwise.
-        """
-
-    def istitle():
-        """S.istitle() -> bool
-
-        Return True if S is a titlecased string and there is at least one
-        character in S, i.e. upper- and titlecase characters may only
-        follow uncased characters and lowercase characters only cased ones.
-        Return False otherwise.
-        """
-
-    def isupper():
-        """S.isupper() -> bool
-
-        Return True if all cased characters in S are uppercase and there is
-        at least one cased character in S, False otherwise.
-        """
-
-    def join():
-        """S.join(iterable) -> unicode
-
-        Return a string which is the concatenation of the strings in the
-        iterable.  The separator between elements is S.
-        """
-
-    def ljust():
-        """S.ljust(width[, fillchar]) -> int
-
-        Return S left-justified in a Unicode string of length width. Padding is
-        done using the specified fill character (default is a space).
-        """
-
-    def lower():
-        """S.lower() -> unicode
-
-        Return a copy of the string S converted to lowercase.
-        """
-
-    def lstrip():
-        """S.lstrip([chars]) -> unicode
-
-        Return a copy of the string S with leading whitespace removed.
-        If chars is given and not None, remove characters in chars instead.
-        If chars is a str, it will be converted to unicode before stripping
-        """
-
-    def partition():
-        """S.partition(sep) -> (head, sep, tail)
-
-        Search for the separator sep in S, and return the part before it,
-        the separator itself, and the part after it.  If the separator is not
-        found, return S and two empty strings.
-        """
-
-    def replace():
-        """S.replace(old, new[, count]) -> unicode
-
-        Return a copy of S with all occurrences of substring
-        old replaced by new.  If the optional argument count is
-        given, only the first count occurrences are replaced.
-        """
-
-    def rfind():
-        """S.rfind(sub[, start[, end]]) -> int
-
-        Return the highest index in S where substring sub is found,
-        such that sub is contained within S[start:end].  Optional
-        arguments start and end are interpreted as in slice notation.
-
-        Return -1 on failure.
-        """
-
-    def rindex():
-        """S.rindex(sub[, start[, end]]) -> int
-
-        Like S.rfind() but raise ValueError when the substring is not found.
-        """
-
-    def rjust():
-        """S.rjust(width[, fillchar]) -> unicode
-
-        Return S right-justified in a Unicode string of length width. Padding
-        is done using the specified fill character (default is a space).
-        """
-
-    def rpartition():
-        """S.rpartition(sep) -> (head, sep, tail)
-
-        Search for the separator sep in S, starting at the end of S, and return
-        the part before it, the separator itself, and the part after it.  If
-        the separator is not found, return two empty strings and S.
-        """
-
-    def rsplit():
-        """S.rsplit(sep=None, maxsplit=-1) -> list of strings
-
-        Return a list of the words in S, using sep as the
-        delimiter string, starting at the end of the string and
-        working to the front.  If maxsplit is given, at most maxsplit
-        splits are done. If sep is not specified, any whitespace string
-        is a separator.
-        """
-
-    def rstrip():
-        """S.rstrip([chars]) -> unicode
-
-        Return a copy of the string S with trailing whitespace removed.
-        If chars is given and not None, remove characters in chars instead.
-        If chars is a str, it will be converted to unicode before stripping
-        """
-
-    def split():
-        """S.split(sep=None, maxsplit=-1) -> list of strings
-
-        Return a list of the words in S, using sep as the
-        delimiter string.  If maxsplit is given, at most maxsplit
-        splits are done. If sep is not specified or is None, any
-        whitespace string is a separator and empty strings are
-        removed from the result.
-        """
-
-    def splitlines():
-        """S.splitlines(keepends=False) -> list of strings
-
-        Return a list of the lines in S, breaking at line boundaries.
-        Line breaks are not included in the resulting list unless keepends
-        is given and true.
-        """
-
-    def startswith():
-        """S.startswith(prefix[, start[, end]]) -> bool
-
-        Return True if S starts with the specified prefix, False otherwise.
-        With optional start, test S beginning at that position.
-        With optional end, stop comparing S at that position.
-        prefix can also be a tuple of strings to try.
-        """
-
-    def strip():
-        """S.strip([chars]) -> unicode
-
-        Return a copy of the string S with leading and trailing
-        whitespace removed.
-        If chars is given and not None, remove characters in chars instead.
-        If chars is a str, it will be converted to unicode before stripping
-        """
-
-    def swapcase():
-        """S.swapcase() -> unicode
-
-        Return a copy of S with uppercase characters converted to lowercase
-        and vice versa.
-        """
-
-    def title():
-        """S.title() -> unicode
-
-        Return a titlecased version of S, i.e. words start with title case
-        characters, all remaining cased characters have lower case.
-        """
-
-    def translate():
-        """S.translate(table) -> unicode
-
-        Return a copy of the string S, where all characters have been mapped
-        through the given translation table, which must be a mapping of
-        Unicode ordinals to Unicode ordinals, Unicode strings or None.
-        Unmapped characters are left untouched. Characters mapped to None
-        are deleted.
-        """
-
-    def upper():
-        """S.upper() -> unicode
-
-        Return a copy of S converted to uppercase.
-        """
-
-    def zfill():
-        """S.zfill(width) -> unicode
-
-        Pad a numeric string S with zeros on the left, to fill a field
-        of the specified width. The string S is never truncated.
-        """
-
-
-W_UnicodeObject.typedef = TypeDef(
-    "unicode", basestring_typedef,
-    __new__ = interp2app(W_UnicodeObject.descr_new),
-    __doc__ = UnicodeDocstrings.__doc__,
-
-    __repr__ = interp2app(W_UnicodeObject.descr_repr,
-                          doc=UnicodeDocstrings.__repr__.__doc__),
-    __str__ = interp2app(W_UnicodeObject.descr_str,
-                         doc=UnicodeDocstrings.__str__.__doc__),
-    __hash__ = interp2app(W_UnicodeObject.descr_hash,
-                          doc=UnicodeDocstrings.__hash__.__doc__),
-
-    __eq__ = interp2app(W_UnicodeObject.descr_eq,
-                        doc=UnicodeDocstrings.__eq__.__doc__),
-    __ne__ = interp2app(W_UnicodeObject.descr_ne,
-                        doc=UnicodeDocstrings.__ne__.__doc__),
-    __lt__ = interp2app(W_UnicodeObject.descr_lt,
-                        doc=UnicodeDocstrings.__lt__.__doc__),
-    __le__ = interp2app(W_UnicodeObject.descr_le,
-                        doc=UnicodeDocstrings.__le__.__doc__),
-    __gt__ = interp2app(W_UnicodeObject.descr_gt,
-                        doc=UnicodeDocstrings.__gt__.__doc__),
-    __ge__ = interp2app(W_UnicodeObject.descr_ge,
-                        doc=UnicodeDocstrings.__ge__.__doc__),
-
-    __len__ = interp2app(W_UnicodeObject.descr_len,
-                         doc=UnicodeDocstrings.__len__.__doc__),
-    __contains__ = interp2app(W_UnicodeObject.descr_contains,
-                              doc=UnicodeDocstrings.__contains__.__doc__),
-
-    __add__ = interp2app(W_UnicodeObject.descr_add,
-                         doc=UnicodeDocstrings.__add__.__doc__),
-    __mul__ = interp2app(W_UnicodeObject.descr_mul,
-                         doc=UnicodeDocstrings.__mul__.__doc__),
-    __rmul__ = interp2app(W_UnicodeObject.descr_mul,
-                          doc=UnicodeDocstrings.__rmul__.__doc__),
-
-    __getitem__ = interp2app(W_UnicodeObject.descr_getitem,
-                             doc=UnicodeDocstrings.__getitem__.__doc__),
-    __getslice__ = interp2app(W_UnicodeObject.descr_getslice,
-                              doc=UnicodeDocstrings.__getslice__.__doc__),
-
-    capitalize = interp2app(W_UnicodeObject.descr_capitalize,
-                            doc=UnicodeDocstrings.capitalize.__doc__),
-    center = interp2app(W_UnicodeObject.descr_center,
-                        doc=UnicodeDocstrings.center.__doc__),
-    count = interp2app(W_UnicodeObject.descr_count,
-                       doc=UnicodeDocstrings.count.__doc__),
-    decode = interp2app(W_UnicodeObject.descr_decode,
-                        doc=UnicodeDocstrings.decode.__doc__),
-    encode = interp2app(W_UnicodeObject.descr_encode,
-                        doc=UnicodeDocstrings.encode.__doc__),
-    expandtabs = interp2app(W_UnicodeObject.descr_expandtabs,
-                            doc=UnicodeDocstrings.expandtabs.__doc__),
-    find = interp2app(W_UnicodeObject.descr_find,
-                      doc=UnicodeDocstrings.find.__doc__),
-    rfind = interp2app(W_UnicodeObject.descr_rfind,
-                       doc=UnicodeDocstrings.rfind.__doc__),
-    index = interp2app(W_UnicodeObject.descr_index,
-                       doc=UnicodeDocstrings.index.__doc__),
-    rindex = interp2app(W_UnicodeObject.descr_rindex,
-                        doc=UnicodeDocstrings.rindex.__doc__),
-    isalnum = interp2app(W_UnicodeObject.descr_isalnum,
-                         doc=UnicodeDocstrings.isalnum.__doc__),
-    isalpha = interp2app(W_UnicodeObject.descr_isalpha,
-                         doc=UnicodeDocstrings.isalpha.__doc__),
-    isdecimal = interp2app(W_UnicodeObject.descr_isdecimal,
-                           doc=UnicodeDocstrings.isdecimal.__doc__),
-    isdigit = interp2app(W_UnicodeObject.descr_isdigit,
-                         doc=UnicodeDocstrings.isdigit.__doc__),
-    islower = interp2app(W_UnicodeObject.descr_islower,
-                         doc=UnicodeDocstrings.islower.__doc__),
-    isnumeric = interp2app(W_UnicodeObject.descr_isnumeric,
-                           doc=UnicodeDocstrings.isnumeric.__doc__),
-    isspace = interp2app(W_UnicodeObject.descr_isspace,
-                         doc=UnicodeDocstrings.isspace.__doc__),
-    istitle = interp2app(W_UnicodeObject.descr_istitle,
-                         doc=UnicodeDocstrings.istitle.__doc__),
-    isupper = interp2app(W_UnicodeObject.descr_isupper,
-                         doc=UnicodeDocstrings.isupper.__doc__),
-    join = interp2app(W_UnicodeObject.descr_join,
-                      doc=UnicodeDocstrings.join.__doc__),
-    ljust = interp2app(W_UnicodeObject.descr_ljust,
-                       doc=UnicodeDocstrings.ljust.__doc__),
-    rjust = interp2app(W_UnicodeObject.descr_rjust,
-                       doc=UnicodeDocstrings.rjust.__doc__),
-    lower = interp2app(W_UnicodeObject.descr_lower,
-                       doc=UnicodeDocstrings.lower.__doc__),
-    partition = interp2app(W_UnicodeObject.descr_partition,
-                           doc=UnicodeDocstrings.partition.__doc__),
-    rpartition = interp2app(W_UnicodeObject.descr_rpartition,
-                            doc=UnicodeDocstrings.rpartition.__doc__),
-    replace = interp2app(W_UnicodeObject.descr_replace,
-                         doc=UnicodeDocstrings.replace.__doc__),
-    split = interp2app(W_UnicodeObject.descr_split,
-                       doc=UnicodeDocstrings.split.__doc__),
-    rsplit = interp2app(W_UnicodeObject.descr_rsplit,
-                        doc=UnicodeDocstrings.rsplit.__doc__),
-    splitlines = interp2app(W_UnicodeObject.descr_splitlines,
-                            doc=UnicodeDocstrings.splitlines.__doc__),
-    startswith = interp2app(W_UnicodeObject.descr_startswith,
-                            doc=UnicodeDocstrings.startswith.__doc__),
-    endswith = interp2app(W_UnicodeObject.descr_endswith,
-                          doc=UnicodeDocstrings.endswith.__doc__),
-    strip = interp2app(W_UnicodeObject.descr_strip,
-                       doc=UnicodeDocstrings.strip.__doc__),
-    lstrip = interp2app(W_UnicodeObject.descr_lstrip,
-                        doc=UnicodeDocstrings.lstrip.__doc__),
-    rstrip = interp2app(W_UnicodeObject.descr_rstrip,
-                        doc=UnicodeDocstrings.rstrip.__doc__),
-    swapcase = interp2app(W_UnicodeObject.descr_swapcase,
-                          doc=UnicodeDocstrings.swapcase.__doc__),
-    title = interp2app(W_UnicodeObject.descr_title,
-                       doc=UnicodeDocstrings.title.__doc__),
-    translate = interp2app(W_UnicodeObject.descr_translate,
-                           doc=UnicodeDocstrings.translate.__doc__),
-    upper = interp2app(W_UnicodeObject.descr_upper,
-                       doc=UnicodeDocstrings.upper.__doc__),
-    zfill = interp2app(W_UnicodeObject.descr_zfill,
-                       doc=UnicodeDocstrings.zfill.__doc__),
-
-    format = interp2app(W_UnicodeObject.descr_format,
-                        doc=UnicodeDocstrings.format.__doc__),
-    __format__ = interp2app(W_UnicodeObject.descr__format__,
-                            doc=UnicodeDocstrings.__format__.__doc__),
-    __mod__ = interp2app(W_UnicodeObject.descr_mod,
-                         doc=UnicodeDocstrings.__mod__.__doc__),
-    __getnewargs__ = interp2app(W_UnicodeObject.descr_getnewargs,
-                                doc=UnicodeDocstrings.__getnewargs__.__doc__),
-    _formatter_parser = interp2app(W_UnicodeObject.descr_formatter_parser),
+    format = interpindirect2app(W_AbstractUnicodeObject.descr_format),
+    __format__ = interpindirect2app(W_AbstractUnicodeObject.descr__format__),
+    __mod__ = interpindirect2app(W_AbstractUnicodeObject.descr_mod),
+    __getnewargs__ = interpindirect2app(W_AbstractUnicodeObject.descr_getnewargs),
+    _formatter_parser =
+        interp2app(W_AbstractUnicodeObject.descr_formatter_parser),
     _formatter_field_name_split =
-        interp2app(W_UnicodeObject.descr_formatter_field_name_split),
+        interp2app(W_AbstractUnicodeObject.descr_formatter_field_name_split),
 )
 W_UnicodeObject.typedef.flag_sequence_bug_compat = True
 

From noreply at buildbot.pypy.org  Sun Jul 26 17:53:14 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 26 Jul 2015 17:53:14 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: move temp array creation inside
 loop.do_accumulate()
Message-ID: <20150726155314.E0F501C02A3@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78672:80032f7aad54
Date: 2015-07-26 16:53 +0100
http://bitbucket.org/pypy/pypy/changeset/80032f7aad54/

Log:	move temp array creation inside loop.do_accumulate()

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -246,9 +246,12 @@
                                   greens=['shapelen', 'func', 'dtype'],
                                   reds='auto')
 
-def do_accumulate(space, shape, func, arr, dtype, axis, out, identity, temp):
+def do_accumulate(space, shape, func, arr, dtype, axis, out, identity):
     out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative=True)
     out_state = out_iter.reset()
+    obj_shape = arr.get_shape()
+    temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]
+    temp = W_NDimArray.from_shape(space, temp_shape, dtype, w_instance=arr)
     temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis, False)
     temp_state = temp_iter.reset()
     arr_iter, arr_state = arr.create_iter()
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -293,13 +293,7 @@
             dtype = self.find_binop_type(space, dtype)
             call__array_wrap__ = True
             if shapelen > 1:
-                temp = None
                 shape = obj_shape[:]
-                temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]
-                if out:
-                    dtype = out.get_dtype()
-                temp = W_NDimArray.from_shape(space, temp_shape, dtype,
-                                            w_instance=obj)
                 if out:
                     # Test for shape agreement
                     # XXX maybe we need to do broadcasting here, although I must
@@ -329,7 +323,7 @@
                         out.fill(space, self.identity.convert_to(space, dtype))
                     return out
                 loop.do_accumulate(space, shape, self.func, obj, dtype, axis,
-                                    out, self.identity, temp)
+                                    out, self.identity)
             else:
                 if out:
                     call__array_wrap__ = False

From noreply at buildbot.pypy.org  Sun Jul 26 22:38:26 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 26 Jul 2015 22:38:26 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-buffered: abandon wrong approach
Message-ID: <20150726203826.95BC71C149C@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-buffered
Changeset: r78673:6d6688e4c3af
Date: 2015-07-24 12:53 +0300
http://bitbucket.org/pypy/pypy/changeset/6d6688e4c3af/

Log:	abandon wrong approach


From noreply at buildbot.pypy.org  Sun Jul 26 22:38:27 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 26 Jul 2015 22:38:27 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: merge closed-branch
Message-ID: <20150726203827.C2BD51C14D3@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r78674:818c46d1e9ca
Date: 2015-07-24 12:55 +0300
http://bitbucket.org/pypy/pypy/changeset/818c46d1e9ca/

Log:	merge closed-branch


From noreply at buildbot.pypy.org  Sun Jul 26 22:38:29 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 26 Jul 2015 22:38:29 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-revisited: improve test to show
 interaction of buffered, external_loop
Message-ID: <20150726203829.059141C14D9@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-revisited
Changeset: r78675:f4f65673cc96
Date: 2015-07-25 23:04 +0300
http://bitbucket.org/pypy/pypy/changeset/f4f65673cc96/

Log:	improve test to show interaction of buffered, external_loop

diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -149,19 +149,39 @@
         # assert str(exc.value).startswith("Iterator flag EXTERNAL_LOOP cannot")
 
     def test_buffered(self):
-        from numpy import arange, nditer, array
-        a = arange(6).reshape(2,3)
-        import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['buffered'])
-            skip('nditer buffered not implmented')
+        from numpy import arange, nditer, array, isscalar
+        a = arange(24).reshape(2, 3, 4)
+        r = []
+        for x in nditer(a, flags=['external_loop'], order='F'):
+            r.append(x)
+        array_r = array(r)
+        assert len(array_r.shape) == 2
+        assert array_r.shape == (12, 2)
+        assert (array_r == [[0, 12], [4, 16], [8, 20], [1, 13], [5, 17], [9, 21],
+                      [2, 14], [6, 18], [10, 22], [3, 15], [7, 19], [11, 23]]).all
+        assert (a == arange(24).reshape(2, 3, 4)).all()
+
+        r = []
+        for x in nditer(a, flags=['buffered'], order='F'):
+            r.append(x)
+        array_r = array(r)
+        assert len(array_r.shape) == 1
+        assert array_r.shape == (24,)
+        assert r[0].shape == ()
+        assert not isscalar(r[0])
+        assert (array_r == [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21,
+                      2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23]).all
+        assert a.shape == (2, 3, 4)
+        assert (a == arange(24).reshape(2, 3, 4)).all()
+
         r = []
         for x in nditer(a, flags=['external_loop', 'buffered'], order='F'):
             r.append(x)
-        array_r = array(r)
-        assert len(array_r.shape) == 2
-        assert array_r.shape == (1, 6)
-        assert (array_r == [0, 3, 1, 4, 2, 5]).all()
+        assert r[0].shape == (24,)
+        assert (array_r == [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21,
+                      2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23]).all
+        assert a.shape == (2, 3, 4)
+        assert (a == arange(24).reshape(2, 3, 4)).all()
 
     def test_op_dtype(self):
         from numpy import arange, nditer, sqrt, array

From noreply at buildbot.pypy.org  Sun Jul 26 22:38:30 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 26 Jul 2015 22:38:30 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-revisited: start to handle buffering,
 implement nditer casting
Message-ID: <20150726203830.29C141C1507@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-revisited
Changeset: r78676:b7128a40d13d
Date: 2015-07-26 23:16 +0300
http://bitbucket.org/pypy/pypy/changeset/b7128a40d13d/

Log:	start to handle buffering, implement nditer casting

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -9,7 +9,8 @@
 from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
                                             shape_agreement, shape_agreement_multiple)
-from pypy.module.micronumpy.casting import find_binop_result_dtype
+from pypy.module.micronumpy.casting import (find_binop_result_dtype, 
+                    can_cast_array, can_cast_type)
 
 
 def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
@@ -108,9 +109,7 @@
         if item == 'external_loop':
             nditer.external_loop = True
         elif item == 'buffered':
-            raise oefmt(space.w_NotImplementedError,
-                'nditer buffered not implemented yet')
-            # For numpy compatability
+            # Each iterator should be 1d
             nditer.buffered = True
         elif item == 'c_index':
             nditer.tracked_index = 'C'
@@ -213,30 +212,6 @@
         return arr
 
 
-def get_iter(space, order, arr, shape, dtype, op_flags, base):
-    imp = arr.implementation
-    backward = is_backward(imp, order)
-    if arr.is_scalar():
-        return ConcreteIter(imp, 1, [], [], [], op_flags, base)
-    if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \
-       (abs(imp.strides[0]) > abs(imp.strides[-1]) and backward):
-        # flip the strides. Is this always true for multidimension?
-        strides = imp.strides[:]
-        backstrides = imp.backstrides[:]
-        shape = imp.shape[:]
-        strides.reverse()
-        backstrides.reverse()
-        shape.reverse()
-    else:
-        strides = imp.strides
-        backstrides = imp.backstrides
-    r = calculate_broadcast_strides(strides, backstrides, imp.shape,
-                                    shape, backward)
-    if len(shape) != len(r[0]):
-        # shape can be shorter when using an external loop, just return a view
-        return ConcreteIter(imp, imp.get_size(), imp.shape, r[0], r[1], op_flags, base)
-    return ConcreteIter(imp, imp.get_size(), shape, r[0], r[1], op_flags, base)
-
 def calculate_ndim(op_in, oa_ndim):
     if oa_ndim >=0:
         return oa_ndim
@@ -383,6 +358,10 @@
         self.done = False
         self.first_next = True
         self.op_axes = []
+        if not space.is_w(w_casting, space.w_None):
+            self.casting = space.str_w(w_casting)
+        else:
+            self.casting = 'safe'
         # convert w_seq operands to a list of W_NDimArray
         if space.isinstance_w(w_seq, space.w_tuple) or \
            space.isinstance_w(w_seq, space.w_list):
@@ -465,14 +444,38 @@
                 if not self_d:
                     self.dtypes[i] = seq_d
                 elif self_d != seq_d:
-                    if not 'r' in self.op_flags[i].tmp_copy:
-                        raise oefmt(space.w_TypeError,
-                                    "Iterator operand required copying or "
-                                    "buffering for operand %d", i)
-                    impl = self.seq[i].implementation
-                    order = support.get_order_as_CF(impl.order, self.order)
-                    new_impl = impl.astype(space, self_d, order)
-                    self.seq[i] = W_NDimArray(new_impl)
+                        impl = self.seq[i].implementation
+                        order = support.get_order_as_CF(impl.order, self.order)
+                        if self.buffered or 'r' in self.op_flags[i].tmp_copy:
+                            if not can_cast_array(
+                                    space, self.seq[i], self_d, self.casting):
+                                raise oefmt(space.w_TypeError, "Iterator operand %d"
+                                    " dtype could not be cast from %s to %s"
+                                    " according to the rule '%s'", i, 
+                                    space.str_w(seq_d.descr_repr(space)),
+                                    space.str_w(self_d.descr_repr(space)),
+                                    self.casting)
+ 
+                            new_impl = impl.astype(space, self_d, order).copy(space)
+                            self.seq[i] = W_NDimArray(new_impl)
+                        else:
+                            raise oefmt(space.w_TypeError, "Iterator "
+                                "operand required copying or buffering, "
+                                "but neither copying nor buffering was "
+                                "enabled")
+                        if 'w' in self.op_flags[i].rw:
+                            if not can_cast_type(
+                                    space, self_d, seq_d, self.casting):
+                                raise oefmt(space.w_TypeError, "Iterator"
+                                    " requested dtype could not be cast from "
+                                    " %s to %s, the operand %d dtype, accord"
+                                    "ing to the rule '%s'", 
+                                    space.str_w(self_d.descr_repr(space)),
+                                    space.str_w(seq_d.descr_repr(space)),
+                                    i, self.casting)
+        elif self.buffered:
+            self.seq = [s.descr_copy(space, w_order=space.wrap(self.order)) for s in self.seq]
+            self.dtypes = [s.get_dtype() for s in self.seq]
         else:
             #copy them from seq
             self.dtypes = [s.get_dtype() for s in self.seq]
@@ -480,14 +483,43 @@
         # create an iterator for each operand
         self.iters = []
         for i in range(len(self.seq)):
-            it = get_iter(space, self.order, self.seq[i], self.shape,
-                          self.dtypes[i], self.op_flags[i], self)
+            it = self.get_iter(space, i)
             it.contiguous = False
             self.iters.append((it, it.reset()))
 
         if self.external_loop:
             coalesce_axes(self, space)
 
+    def get_iter(self, space, i):
+        arr = self.seq[i]
+        dtype = self.dtypes[i]
+        shape = self.shape
+        imp = arr.implementation
+        backward = is_backward(imp, self.order)
+        if arr.is_scalar():
+            return ConcreteIter(imp, 1, [], [], [], self.op_flags[i], self)
+        if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \
+           (abs(imp.strides[0]) > abs(imp.strides[-1]) and backward):
+            # flip the strides. Is this always true for multidimension?
+            strides = imp.strides[:]
+            backstrides = imp.backstrides[:]
+            shape = imp.shape[:]
+            strides.reverse()
+            backstrides.reverse()
+            shape.reverse()
+        else:
+            strides = imp.strides
+            backstrides = imp.backstrides
+        r = calculate_broadcast_strides(strides, backstrides, imp.shape,
+                                        shape, backward)
+        iter_shape = shape
+        if len(shape) != len(r[0]):
+            # shape can be shorter when using an external loop, just return a view
+            iter_shape = imp.shape
+        return ConcreteIter(imp, imp.get_size(), iter_shape, r[0], r[1],
+                            self.op_flags[i], self)
+
+
     def set_op_axes(self, space, w_op_axes):
         if space.len_w(w_op_axes) != len(self.seq):
             raise oefmt(space.w_ValueError,
@@ -520,8 +552,8 @@
         return space.wrap(self)
 
     def getitem(self, it, st):
-        res = it.getoperand(st)
-        return W_NDimArray(res)
+        w_res = W_NDimArray(it.getoperand(st))
+        return w_res
 
     def descr_getitem(self, space, w_idx):
         idx = space.int_w(w_idx)
diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -38,3 +38,6 @@
     a = zeros(*args, **kwargs)
     a.fill(1)
     return a
+
+def isscalar(a):
+    return type(a) in [typeinfo[t] for t in types]
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -64,18 +64,15 @@
         a = arange(24).reshape(2, 3, 4)
         import sys
         r = []
-        n = 0
         for x in nditer(a, flags=['external_loop']):
             r.append(x)
-            n += 1
-        assert n == 1
+        assert len(r) == 1
+        assert r[0].shape == (24,)
         assert (array(r) == range(24)).all()
         r = []
-        n = 0
         for x in nditer(a, flags=['external_loop'], order='F'):
             r.append(x)
-            n += 1
-        assert n == 12
+        assert len(r) == 12
         assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21],
                              [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23],
                             ]).all()
@@ -160,9 +157,16 @@
         assert (array_r == [[0, 12], [4, 16], [8, 20], [1, 13], [5, 17], [9, 21],
                       [2, 14], [6, 18], [10, 22], [3, 15], [7, 19], [11, 23]]).all
         assert (a == arange(24).reshape(2, 3, 4)).all()
+        a[0,0,0] = 100
+        assert r[0][0] == 100
 
         r = []
-        for x in nditer(a, flags=['buffered'], order='F'):
+        try:
+            it = nditer(a, flags=['buffered'], order='F')
+        except NotImplementedError as e:
+            assert 'unsupported value for order' in str(e)
+            skip('buffered with order="F" requires fortran tmp array creation')
+        for x in it:
             r.append(x)
         array_r = array(r)
         assert len(array_r.shape) == 1
@@ -172,6 +176,9 @@
         assert (array_r == [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21,
                       2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23]).all
         assert a.shape == (2, 3, 4)
+        a[0,0,0] = 0
+        # buffered copies the data into a tmp array
+        assert r[0] == 100
         assert (a == arange(24).reshape(2, 3, 4)).all()
 
         r = []
@@ -208,11 +215,10 @@
         from numpy import arange, nditer
         import sys
         a = arange(6.)
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['buffered'], op_dtypes=['float32'])
-            skip('nditer casting not implemented yet')
         exc = raises(TypeError, nditer, a, flags=['buffered'], op_dtypes=['float32'])
-        assert str(exc.value).startswith("Iterator operand 0 dtype could not be cast")
+        assert str(exc.value) == "Iterator operand 0 dtype could not be " + \
+            "cast from dtype('float64') to dtype('float32') according to the" +\
+            " rule 'safe'"
         r = []
         for x in nditer(a, flags=['buffered'], op_dtypes=['float32'],
                                 casting='same_kind'):
@@ -223,6 +229,7 @@
         assert str(exc.value).startswith("Iterator operand 0 dtype could not be cast")
         r = []
         b = arange(6)
+        print 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
         exc = raises(TypeError, nditer, b, flags=['buffered'], op_dtypes=['float64'],
                                 op_flags=['readwrite'], casting='same_kind')
         assert str(exc.value).startswith("Iterator requested dtype could not be cast")

From noreply at buildbot.pypy.org  Sun Jul 26 22:38:31 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 26 Jul 2015 22:38:31 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-revisited: do not buffer output arrays,
 fix exception formatting
Message-ID: <20150726203831.4C07A1C1545@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-revisited
Changeset: r78677:bfac28c76c83
Date: 2015-07-26 23:32 +0300
http://bitbucket.org/pypy/pypy/changeset/bfac28c76c83/

Log:	do not buffer output arrays, fix exception formatting

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -424,8 +424,15 @@
                     self.seq[i] = W_NDimArray.from_shape(space, self.shape, out_dtype)
                 else:
                     if not self.op_flags[i].broadcast:
-                        # Raises if ooutput cannot be broadcast
-                        shape_agreement(space, self.shape, self.seq[i], False)
+                        # Raises if output cannot be broadcast
+                        try:
+                            shape_agreement(space, self.shape, self.seq[i], False)
+                        except OperationError as e:
+                            raise oefmt(space.w_ValueError, "non-broadcastable"
+                                " output operand with shape %s doesn't match "
+                                "the broadcast shape %s", 
+                                str(self.seq[i].get_shape()),
+                                str(self.shape)) 
 
         if self.tracked_index != "":
             if self.order == "K":
@@ -474,7 +481,10 @@
                                     space.str_w(seq_d.descr_repr(space)),
                                     i, self.casting)
         elif self.buffered:
-            self.seq = [s.descr_copy(space, w_order=space.wrap(self.order)) for s in self.seq]
+            for i in range(len(self.seq)):
+                if i not in outargs:
+                    self.seq[i] = self.seq[i].descr_copy(space,
+                                     w_order=space.wrap(self.order))
             self.dtypes = [s.get_dtype() for s in self.seq]
         else:
             #copy them from seq
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -229,7 +229,6 @@
         assert str(exc.value).startswith("Iterator operand 0 dtype could not be cast")
         r = []
         b = arange(6)
-        print 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
         exc = raises(TypeError, nditer, b, flags=['buffered'], op_dtypes=['float64'],
                                 op_flags=['readwrite'], casting='same_kind')
         assert str(exc.value).startswith("Iterator requested dtype could not be cast")
@@ -259,9 +258,6 @@
             return it.operands[1]
         assert (square1([1, 2, 3]) == [1, 4, 9]).all()
 
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, [1, 2], flags=['buffered'])
-            skip('nditer buffered not implmented')
         def square2(a, out=None):
             it = nditer([a, out], flags=['external_loop', 'buffered'],
                         op_flags=[['readonly'],

From noreply at buildbot.pypy.org  Mon Jul 27 11:47:58 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 27 Jul 2015 11:47:58 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: minor refactoring
Message-ID: <20150727094758.9A6891C0EEA@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78678:bede2e3b7ce5
Date: 2015-07-25 10:57 +0200
http://bitbucket.org/pypy/pypy/changeset/bede2e3b7ce5/

Log:	minor refactoring

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -138,7 +138,7 @@
         self.clear_newoperations();
 
         # vectorize
-        self.build_dependency_graph()
+        self.dependency_graph = DependencyGraph(self.loop)
         self.find_adjacent_memory_refs()
         self.extend_packset()
         self.combine_packset()
@@ -193,7 +193,8 @@
             operations.append(op)
             self.emit_unrolled_operation(op)
 
-        prohibit_opnums = (rop.GUARD_FUTURE_CONDITION, rop.GUARD_EARLY_EXIT,
+        prohibit_opnums = (rop.GUARD_FUTURE_CONDITION,
+                           rop.GUARD_EARLY_EXIT,
                            rop.GUARD_NOT_INVALIDATED)
 
         orig_jump_args = jump_op.getarglist()[:]
@@ -273,9 +274,6 @@
         unroll_count = simd_vec_reg_bytes // byte_count
         return unroll_count-1 # it is already unrolled once
 
-    def build_dependency_graph(self):
-        self.dependency_graph = DependencyGraph(self.loop)
-
     def find_adjacent_memory_refs(self):
         """ the pre pass already builds a hash of memory references and the
         operations. Since it is in SSA form there are no array indices.

From noreply at buildbot.pypy.org  Mon Jul 27 11:47:59 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 27 Jul 2015 11:47:59 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: picked lltype.cast changes for cast to
 bool, implemented int_is_true as a vector operation
Message-ID: <20150727094759.E287E1C0EEA@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78679:658ac4e3b571
Date: 2015-07-27 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/658ac4e3b571/

Log:	picked lltype.cast changes for cast to bool, implemented int_is_true
	as a vector operation

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -358,7 +358,8 @@
 
     @specialize.argtype(1)
     def box(self, value):
-        if value:
+        boolean = rffi.cast(self.T, value)
+        if boolean:
             return self._True
         else:
             return self._False
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -69,7 +69,7 @@
         self.rtyper = cpu.rtyper
         self._debug = False
 
-    def stitch_bridge(self, failargs, token):
+    def stitch_bridge(self, faildescr, token):
         raise NotImplementedError
 
     def setup_once(self):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -589,7 +589,7 @@
                                                        rawstart, fullsize)
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
-    def attach_bridge(self, faildescr, token):
+    def stitch_bridge(self, faildescr, token):
         rawstart = self.materialize_loop(token)
         self.patch_jump_for_descr(faildescr, rawstart)
 
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -609,7 +609,7 @@
 
         possible_instr_unrolled = unrolling_iterable([(1,'B_xx'),(2,'W_xx'),(4,'D_xx'),(8,'Q_xx')])
 
-        def INSN(self, size, loc1, loc2):
+        def INSN(self, loc1, loc2, size):
             code1 = loc1.location_code()
             code2 = loc2.location_code()
             assert code1 == code2 == 'x'
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -52,8 +52,11 @@
             # reset to zeros
             self.mc.PXOR(temp, temp)
 
-        self.mc.PCMPEQ(size, loc, temp)
+        # cmp with zeros (in temp) creates ones at each slot where it is zero
+        self.mc.PCMPEQ(loc, temp, size)
+        # temp converted to ones
         self.mc.PCMPEQQ(temp, temp)
+        # test if all slots are zero
         self.mc.PTEST(loc, temp)
 
     # vector operations
@@ -163,6 +166,21 @@
             elif itemsize == 8:
                 self.mc.MOVUPD(dest_loc, value_loc)
 
+    def genop_vec_int_is_true(self, op, arglocs, resloc):
+        loc, size = arglocs
+        temp = X86_64_XMM_SCRATCH_REG
+        self.mc.PXOR(temp, temp)
+        # every entry that is non zero -> becomes zero
+        # zero entries become ones
+        self.mc.PCMPEQ(loc, temp, size)
+        # a second time -> every zero entry (corresponding to non zero
+        # entries before) become ones
+        self.mc.PCMPEQ(loc, temp, size)
+
+    def genop_guard_vec_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
+        self._guard_vector_true(op, arglocs[0])
+        self.implement_guard(guard_token, 'NZ')
+
     def genop_vec_int_mul(self, op, arglocs, resloc):
         loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
@@ -605,6 +623,17 @@
         tosize = result.getsize()
         self.perform(op, [resloc, imm(size), imm(tosize)], resloc)
 
+    def consider_vec_int_is_true(self, op, guard_op):
+        args = op.getarglist()
+        resloc = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
+        sizearg = op.getarg(0)
+        assert isinstance(sizearg, BoxVector)
+        size = sizearg.getsize()
+        if guard_op is not None:
+            self.perform_with_guard(op, guard_op, [resloc,imm(size)], None)
+        else:
+            self.perform(op, [resloc,imm(size)], None)
+
     def consider_vec_box(self, op):
         # pseudo instruction, needed to create a new variable
         self.xrm.force_allocate_reg(op.result)
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1273,7 +1273,9 @@
             return
 
         result = []
-        if min2:
+        if v_result.concretetype is lltype.Bool:
+            result.append(SpaceOperation('int_is_true', [v_arg], v_result))
+        elif min2:
             c_bytes = Constant(size2, lltype.Signed)
             result.append(SpaceOperation('int_signext', [v_arg, c_bytes],
                                          v_result))
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -750,6 +750,7 @@
     rop.VEC_FLOAT_ABS:   FLOAT_SINGLE_ARG_OP_TO_VOP,
     rop.VEC_FLOAT_NEG:   FLOAT_SINGLE_ARG_OP_TO_VOP,
     rop.VEC_FLOAT_EQ:    OpToVectorOp((PT_FLOAT_GENERIC,PT_FLOAT_GENERIC), INT_RES),
+    rop.VEC_INT_IS_TRUE: OpToVectorOp((PT_INT_GENERIC,PT_INT_GENERIC), PT_INT_GENERIC),
 
     rop.VEC_RAW_LOAD:         LOAD_TRANS,
     rop.VEC_GETARRAYITEM_RAW: LOAD_TRANS,
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -497,6 +497,7 @@
     'VEC_FLOAT_ABS/1',
     '_VEC_ARITHMETIC_LAST',
     'VEC_FLOAT_EQ/2',
+    'VEC_INT_IS_TRUE/1b',
 
     '_VEC_CAST_FIRST',
     'VEC_INT_SIGNEXT/2',
@@ -805,6 +806,7 @@
     rop.FLOAT_ABS: rop.VEC_FLOAT_ABS,
     rop.FLOAT_NEG: rop.VEC_FLOAT_NEG,
     rop.FLOAT_EQ:  rop.VEC_FLOAT_EQ,
+    rop.INT_IS_TRUE: rop.VEC_INT_IS_TRUE,
 
     # casts
     rop.INT_SIGNEXT: rop.VEC_INT_SIGNEXT,

From noreply at buildbot.pypy.org  Mon Jul 27 12:48:25 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 27 Jul 2015 12:48:25 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: ironed out problems indicated by
	buildbot (small batch)
Message-ID: <20150727104825.3777D1C0EEA@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78680:2a58825eefba
Date: 2015-07-27 12:09 +0200
http://bitbucket.org/pypy/pypy/changeset/2a58825eefba/

Log:	ironed out problems indicated by buildbot (small batch) llgraph now
	checks for box is equal to zero (otherwise getaccum will fail)
	renamed FloatConstant -> NumberConstant

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -579,14 +579,20 @@
 
 class NumberConstant(Node):
     def __init__(self, v):
-        assert len(v) > 0
-        c = v[-1]
-        if c == 'f':
-            self.v = float(v[:-1])
-        elif c == 'i':
-            self.v = int(v[:-1])
+        if isinstance(v, int):
+            self.v = v
+        elif isinstance(v, float):
+            self.v = v
         else:
-            self.v = float(v)
+            assert isinstance(v, str)
+            assert len(v) > 0
+            c = v[-1]
+            if c == 'f':
+                self.v = float(v[:-1])
+            elif c == 'i':
+                self.v = int(v[:-1])
+            else:
+                self.v = float(v)
 
     def __repr__(self):
         return "Const(%s)" % self.v
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -1,6 +1,6 @@
 import py
 from pypy.module.micronumpy.compile import (numpy_compile, Assignment,
-    ArrayConstant, FloatConstant, Operator, Variable, RangeConstant, Execute,
+    ArrayConstant, NumberConstant, Operator, Variable, RangeConstant, Execute,
     FunctionCall, FakeSpace, W_NDimArray)
 
 
@@ -25,30 +25,30 @@
         interp = self.compile(code)
         assert isinstance(interp.code.statements[0].expr, ArrayConstant)
         st = interp.code.statements[0]
-        assert st.expr.items == [FloatConstant(1), FloatConstant(2),
-                                 FloatConstant(3)]
+        assert st.expr.items == [NumberConstant(1), NumberConstant(2),
+                                 NumberConstant(3)]
 
     def test_array_literal2(self):
         code = "a = [[1],[2],[3]]"
         interp = self.compile(code)
         assert isinstance(interp.code.statements[0].expr, ArrayConstant)
         st = interp.code.statements[0]
-        assert st.expr.items == [ArrayConstant([FloatConstant(1)]),
-                                 ArrayConstant([FloatConstant(2)]),
-                                 ArrayConstant([FloatConstant(3)])]
+        assert st.expr.items == [ArrayConstant([NumberConstant(1)]),
+                                 ArrayConstant([NumberConstant(2)]),
+                                 ArrayConstant([NumberConstant(3)])]
 
     def test_expr_1(self):
         code = "b = a + 1"
         interp = self.compile(code)
         assert (interp.code.statements[0].expr ==
-                Operator(Variable("a"), "+", FloatConstant(1)))
+                Operator(Variable("a"), "+", NumberConstant(1)))
 
     def test_expr_2(self):
         code = "b = a + b - 3"
         interp = self.compile(code)
         assert (interp.code.statements[0].expr ==
                 Operator(Operator(Variable("a"), "+", Variable("b")), "-",
-                         FloatConstant(3)))
+                         NumberConstant(3)))
 
     def test_expr_3(self):
         # an equivalent of range
@@ -60,13 +60,13 @@
         code = "3 + a"
         interp = self.compile(code)
         assert interp.code.statements[0] == Execute(
-            Operator(FloatConstant(3), "+", Variable("a")))
+            Operator(NumberConstant(3), "+", Variable("a")))
 
     def test_array_access(self):
         code = "a -> 3"
         interp = self.compile(code)
         assert interp.code.statements[0] == Execute(
-            Operator(Variable("a"), "->", FloatConstant(3)))
+            Operator(Variable("a"), "->", NumberConstant(3)))
 
     def test_function_call(self):
         code = "sum(a)"
@@ -81,7 +81,7 @@
         """
         interp = self.compile(code)
         assert interp.code.statements[0] == Assignment(
-            'a', Operator(Variable('b'), "+", FloatConstant(3)))
+            'a', Operator(Variable('b'), "+", NumberConstant(3)))
 
 
 class TestRunner(object):
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -873,7 +873,7 @@
                 value = self.env[box]
             else:
                 value = None
-            if box.getaccum():
+            if box and box.getaccum():
                 if box.getaccum().operator == '+':
                     value = sum(value)
                 elif box.getaccum().operator == '*':
diff --git a/rpython/jit/tool/test/test_jitoutput.py b/rpython/jit/tool/test/test_jitoutput.py
--- a/rpython/jit/tool/test/test_jitoutput.py
+++ b/rpython/jit/tool/test/test_jitoutput.py
@@ -61,6 +61,8 @@
 nvirtuals:              13
 nvholes:                14
 nvreused:               15
+vecopt tried:           12
+vecopt success:         4
 Total # of loops:       100
 Total # of bridges:     300
 Freed # of loops:       99
@@ -88,3 +90,5 @@
     assert info.nvirtuals == 13
     assert info.nvholes == 14
     assert info.nvreused == 15
+    assert info.vecopt_tried == 12
+    assert info.vecopt_success == 4

From noreply at buildbot.pypy.org  Mon Jul 27 12:48:26 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 27 Jul 2015 12:48:26 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: setting right descr for guard tests
Message-ID: <20150727104826.7698A1C0EEA@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78681:b61701adde64
Date: 2015-07-27 12:19 +0200
http://bitbucket.org/pypy/pypy/changeset/b61701adde64/

Log:	setting right descr for guard tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_guard.py b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_guard.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
@@ -1,5 +1,6 @@
 import py
 
+from rpython.jit.metainterp import compile
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
 from rpython.jit.metainterp.optimizeopt.vectorize import (VecScheduleData,
@@ -32,6 +33,9 @@
 
 class GuardBaseTest(SchedulerBaseTest):
     def optguards(self, loop, user_code=False):
+        for op in loop.operations:
+            if op.is_guard():
+                op.setdescr(compile.CompileLoopVersionDescr())
         dep = DependencyGraph(loop)
         opt = GuardStrengthenOpt(dep.index_vars)
         opt.propagate_all_forward(loop, user_code)

From noreply at buildbot.pypy.org  Mon Jul 27 12:48:27 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 27 Jul 2015 12:48:27 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: using pygame viewer for the dependencies
 instead of writing it manually to the tmp folder (thx fijal for hint)
Message-ID: <20150727104827.993F71C0EEA@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78682:d3b2ba395def
Date: 2015-07-27 12:48 +0200
http://bitbucket.org/pypy/pypy/changeset/d3b2ba395def/

Log:	using pygame viewer for the dependencies instead of writing it
	manually to the tmp folder (thx fijal for hint)

diff --git a/rpython/conftest.py b/rpython/conftest.py
--- a/rpython/conftest.py
+++ b/rpython/conftest.py
@@ -40,6 +40,9 @@
     group.addoption('--viewloops', action="store_true",
            default=False, dest="viewloops",
            help="show only the compiled loops")
+    group.addoption('--viewdeps', action="store_true",
+           default=False, dest="viewdeps",
+           help="show the dependencies that have been constructed from a trace")
 
 
 def pytest_pycollect_makeitem(__multicall__,collector, name, obj):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -7,6 +7,7 @@
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph, Dependency,
         IndexVar, MemoryRef)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.conftest import option
 
 class DependencyBaseTest(BaseTest):
 
@@ -16,7 +17,7 @@
     def build_dependency(self, ops):
         loop = self.parse_loop(ops)
         self.last_graph = DependencyGraph(loop)
-        self._write_dot_and_convert_to_svg(self.last_graph, self.test_name)
+        self.show_dot_graph(self.last_graph, self.test_name)
         for node in self.last_graph.nodes:
             assert node.independent(node)
         return self.last_graph
@@ -45,7 +46,7 @@
                 node_b = graph.getnode(idx_b)
                 dependency = node_a.getedge_to(node_b)
                 if dependency is None and idx_b not in exceptions.setdefault(idx,[]):
-                    self._write_dot_and_convert_to_svg(graph, 'except')
+                    self.show_dot_graph(graph, self.test_name + '_except')
                     assert dependency is not None, \
                        " it is expected that instruction at index" + \
                        " %s depends on instr on index %s but it does not.\n%s" \
@@ -92,13 +93,13 @@
         b = self.last_graph.getnode(b)
         assert not a.independent(b), "{a} and {b} are independent!".format(a=a,b=b)
 
-    def _write_dot_and_convert_to_svg(self, graph, filename):
-        dot = graph.as_dot()
-        with open('/tmp/_'+filename+'.dot', 'w') as fd:
-            fd.write(dot)
-        with open('/tmp/'+filename+'.svg', 'w') as fd:
-            import subprocess
-            subprocess.Popen(['dot', '-Tsvg', '/tmp/_'+filename+'.dot'], stdout=fd).communicate()
+    def show_dot_graph(self, graph, name):
+        if option and option.viewdeps:
+            from rpython.translator.tool.graphpage import GraphPage
+            page = GraphPage()
+            page.source = graph.as_dot()
+            page.links = []
+            page.display()
 
     def debug_print_operations(self, loop):
         print('--- loop instr numbered ---')
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -76,7 +76,7 @@
             loop.operations.insert(idx+1, guard)
         opt.analyse_index_calculations()
         if opt.dependency_graph is not None:
-            self._write_dot_and_convert_to_svg(opt.dependency_graph, "ee" + self.test_name)
+            self.show_dot_graph(opt.dependency_graph, "early_exit_" + self.test_name)
             opt.schedule(False)
         opt.unroll_loop_iterations(loop, unroll_factor)
         opt.loop.operations = opt.get_newoperations()
@@ -84,7 +84,7 @@
         opt.clear_newoperations()
         opt.build_dependency_graph()
         self.last_graph = opt.dependency_graph
-        self._write_dot_and_convert_to_svg(self.last_graph, self.test_name)
+        self.show_dot_graph(self.last_graph, self.test_name)
         return opt
 
     def init_packset(self, loop, unroll_factor = -1):
diff --git a/rpython/translator/tool/graphpage.py b/rpython/translator/tool/graphpage.py
--- a/rpython/translator/tool/graphpage.py
+++ b/rpython/translator/tool/graphpage.py
@@ -12,7 +12,6 @@
 class GraphPage(BaseGraphPage):
     save_tmp_file = str(udir.join('graph.dot'))
 
-
 class VariableHistoryGraphPage(GraphPage):
     """ A GraphPage showing the history of variable bindings. """
 

From noreply at buildbot.pypy.org  Mon Jul 27 20:49:06 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 27 Jul 2015 20:49:06 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-revisited: test, fix division by 0
Message-ID: <20150727184906.55DB11C14D3@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-revisited
Changeset: r78683:fb5def8bce10
Date: 2015-07-27 01:51 +0300
http://bitbucket.org/pypy/pypy/changeset/fb5def8bce10/

Log:	test, fix division by 0

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -303,6 +303,8 @@
         _shape = [shape[-1]]  + old_iter.slice_shape
         _backstride = [(shape[-1] - 1) * strides[-1]] + old_iter.slice_backstride
         fastest = shape[-1]
+    if fastest == 0:
+        return old_iter
     if flat:
         _shape = [support.product(_shape)]
         if len(_stride) > 1:
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -190,6 +190,17 @@
         assert a.shape == (2, 3, 4)
         assert (a == arange(24).reshape(2, 3, 4)).all()
 
+    def test_zerosize(self):
+        from numpy import nditer, array
+        for a in [ array([]), array([1]), array([1, 2]) ]:
+            buffersize = max(16 * 1024 ** 2 // a.itemsize, 1)
+            r = []
+            for chunk in nditer(a, 
+                    flags=['external_loop', 'buffered', 'zerosize_ok'],
+                    buffersize=buffersize, order='C'):
+                r.append(chunk)
+            assert (r == a).all()
+
     def test_op_dtype(self):
         from numpy import arange, nditer, sqrt, array
         a = arange(6).reshape(2,3) - 3

From noreply at buildbot.pypy.org  Mon Jul 27 20:49:07 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 27 Jul 2015 20:49:07 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-revisited: protect from empty iterator
Message-ID: <20150727184907.A0D641C14D3@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-revisited
Changeset: r78684:8da2f287bcdf
Date: 2015-07-27 06:35 +0300
http://bitbucket.org/pypy/pypy/changeset/8da2f287bcdf/

Log:	protect from empty iterator

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -283,6 +283,8 @@
     of shape (4,3) by setting the offset to the beginning of the data at each iteration
     '''
     shape = [s+1 for s in old_iter.shape_m1]
+    if len(shape) < 1:
+        return old_iter
     strides = old_iter.strides
     backstrides = old_iter.backstrides
     if order == 'F':

From noreply at buildbot.pypy.org  Mon Jul 27 21:04:14 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 27 Jul 2015 21:04:14 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: move code around
Message-ID: <20150727190414.666631C1DE6@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78685:0501a9b82ad2
Date: 2015-07-27 19:26 +0100
http://bitbucket.org/pypy/pypy/changeset/0501a9b82ad2/

Log:	move code around

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -339,10 +339,44 @@
                 out = space.call_method(obj, '__array_wrap__', out)
             return out
 
+        axis_flags = [False] * shapelen
+        for i in axes:
+            if axis_flags[i]:
+                raise oefmt(space.w_ValueError, "duplicate value in 'axis'")
+            axis_flags[i] = True
+
+
         _, dtype, _ = self.find_specialization(space, dtype, dtype, out,
                                                    casting='unsafe')
         call__array_wrap__ = True
-        if shapelen > 1 and axis < shapelen:
+        if shapelen == len(axes):
+            if out:
+                call__array_wrap__ = False
+                if out.ndims() > 0:
+                    raise oefmt(space.w_ValueError,
+                                "output parameter for reduction operation %s has "
+                                "too many dimensions", self.name)
+                dtype = out.get_dtype()
+            res = loop.compute_reduce(space, obj, dtype, self.func, self.done_func,
+                                    self.identity)
+            if out:
+                out.set_scalar_value(res)
+                return out
+            if keepdims:
+                shape = [1] * len(obj_shape)
+                out = W_NDimArray.from_shape(space, shape, dtype, w_instance=obj)
+                out.implementation.setitem(0, res)
+                res = out
+            elif not space.is_w(space.type(w_obj), space.gettypefor(W_NDimArray)):
+                # subtypes return a ndarray subtype, not a scalar
+                out = W_NDimArray.from_shape(space, [1], dtype, w_instance=obj)
+                out.implementation.setitem(0, res)
+                res = out
+            if call__array_wrap__:
+                res = space.call_method(obj, '__array_wrap__', res)
+            return res
+
+        else:
             temp = None
             if keepdims:
                 shape = obj_shape[:axis] + [1] + obj_shape[axis + 1:]
@@ -381,31 +415,6 @@
             if call__array_wrap__:
                 out = space.call_method(obj, '__array_wrap__', out)
             return out
-        if out:
-            call__array_wrap__ = False
-            if out.ndims() > 0:
-                raise oefmt(space.w_ValueError,
-                            "output parameter for reduction operation %s has "
-                            "too many dimensions", self.name)
-            dtype = out.get_dtype()
-        res = loop.compute_reduce(space, obj, dtype, self.func, self.done_func,
-                                  self.identity)
-        if out:
-            out.set_scalar_value(res)
-            return out
-        if keepdims:
-            shape = [1] * len(obj_shape)
-            out = W_NDimArray.from_shape(space, shape, dtype, w_instance=obj)
-            out.implementation.setitem(0, res)
-            res = out
-        elif not space.is_w(space.type(w_obj), space.gettypefor(W_NDimArray)):
-            # subtypes return a ndarray subtype, not a scalar
-            out = W_NDimArray.from_shape(space, [1], dtype, w_instance=obj)
-            out.implementation.setitem(0, res)
-            res = out
-        if call__array_wrap__:
-            res = space.call_method(obj, '__array_wrap__', res)
-        return res
 
     def descr_outer(self, space, __args__):
         return self._outer(space, __args__)

From noreply at buildbot.pypy.org  Mon Jul 27 21:04:15 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 27 Jul 2015 21:04:15 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: small simplification
Message-ID: <20150727190415.7BBAC1C1DE6@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78686:ea4e0e3201e8
Date: 2015-07-27 19:53 +0100
http://bitbucket.org/pypy/pypy/changeset/ea4e0e3201e8/

Log:	small simplification

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -204,17 +204,16 @@
         self.array.setitem(state.offset, elem)
 
 
-def AxisIter(array, shape, axis, cumulative):
+def AxisIter(array, shape, axis):
     strides = array.get_strides()
     backstrides = array.get_backstrides()
-    if not cumulative:
-        if len(shape) == len(strides):
-            # keepdims = True
-            strides = strides[:axis] + [0] + strides[axis + 1:]
-            backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:]
-        else:
-            strides = strides[:axis] + [0] + strides[axis:]
-            backstrides = backstrides[:axis] + [0] + backstrides[axis:]
+    if len(shape) == len(strides):
+        # keepdims = True
+        strides = strides[:axis] + [0] + strides[axis + 1:]
+        backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:]
+    else:
+        strides = strides[:axis] + [0] + strides[axis:]
+        backstrides = backstrides[:axis] + [0] + backstrides[axis:]
     return ArrayIter(array, support.product(shape), shape, strides, backstrides)
 
 
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -9,7 +9,7 @@
 from pypy.module.micronumpy import support, constants as NPY
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.iterators import PureShapeIter, AxisIter, \
-    AllButAxisIter
+    AllButAxisIter, ArrayIter
 from pypy.interpreter.argument import Arguments
 
 
@@ -246,13 +246,13 @@
                                   greens=['shapelen', 'func', 'dtype'],
                                   reds='auto')
 
+
 def do_accumulate(space, shape, func, arr, dtype, axis, out, identity):
-    out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative=True)
-    out_state = out_iter.reset()
+    out_iter, out_state = out.create_iter()
     obj_shape = arr.get_shape()
     temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]
     temp = W_NDimArray.from_shape(space, temp_shape, dtype, w_instance=arr)
-    temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis, False)
+    temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis)
     temp_state = temp_iter.reset()
     arr_iter, arr_state = arr.create_iter()
     arr_iter.track_index = False
@@ -340,7 +340,7 @@
                                    reds='auto')
 
 def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity):
-    out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative=False)
+    out_iter = AxisIter(out.implementation, arr.get_shape(), axis)
     out_state = out_iter.reset()
     arr_iter, arr_state = arr.create_iter()
     arr_iter.track_index = False

From noreply at buildbot.pypy.org  Mon Jul 27 21:17:21 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 27 Jul 2015 21:17:21 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: another small simplification
Message-ID: <20150727191721.C9DFF1C1035@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78687:3f82d765fe9f
Date: 2015-07-27 20:17 +0100
http://bitbucket.org/pypy/pypy/changeset/3f82d765fe9f/

Log:	another small simplification

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -247,7 +247,7 @@
                                   reds='auto')
 
 
-def do_accumulate(space, shape, func, arr, dtype, axis, out, identity):
+def do_accumulate(space, func, arr, dtype, axis, out, identity):
     out_iter, out_state = out.create_iter()
     obj_shape = arr.get_shape()
     temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]
@@ -258,7 +258,7 @@
     arr_iter.track_index = False
     if identity is not None:
         identity = identity.convert_to(space, dtype)
-    shapelen = len(shape)
+    shapelen = len(obj_shape)
     while not out_iter.done(out_state):
         accumulate_driver.jit_merge_point(shapelen=shapelen, func=func,
                                           dtype=dtype)
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -322,7 +322,7 @@
                     if self.identity is not None:
                         out.fill(space, self.identity.convert_to(space, dtype))
                     return out
-                loop.do_accumulate(space, shape, self.func, obj, dtype, axis,
+                loop.do_accumulate(space, self.func, obj, dtype, axis,
                                     out, self.identity)
             else:
                 if out:

From noreply at buildbot.pypy.org  Mon Jul 27 21:50:07 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 27 Jul 2015 21:50:07 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: another small simplification
Message-ID: <20150727195007.D68651C1035@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78688:d3991d28b58a
Date: 2015-07-27 20:50 +0100
http://bitbucket.org/pypy/pypy/changeset/d3991d28b58a/

Log:	another small simplification

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -339,14 +339,14 @@
                                    greens=['shapelen', 'func', 'dtype'],
                                    reds='auto')
 
-def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity):
+def do_axis_reduce(space, func, arr, dtype, axis, out, identity):
     out_iter = AxisIter(out.implementation, arr.get_shape(), axis)
     out_state = out_iter.reset()
     arr_iter, arr_state = arr.create_iter()
     arr_iter.track_index = False
     if identity is not None:
         identity = identity.convert_to(space, dtype)
-    shapelen = len(shape)
+    shapelen = len(out.get_shape())
     while not out_iter.done(out_state):
         axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
                                            dtype=dtype)
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -410,7 +410,7 @@
                 if self.identity is not None:
                     out.fill(space, self.identity.convert_to(space, dtype))
                 return out
-            loop.do_axis_reduce(space, shape, self.func, obj, dtype, axis,
+            loop.do_axis_reduce(space, self.func, obj, dtype, axis,
                                 out, self.identity)
             if call__array_wrap__:
                 out = space.call_method(obj, '__array_wrap__', out)

From noreply at buildbot.pypy.org  Mon Jul 27 22:39:40 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 27 Jul 2015 22:39:40 +0200 (CEST)
Subject: [pypy-commit] pypy nditer-revisited: close branch to be merged
Message-ID: <20150727203940.939951C1035@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: nditer-revisited
Changeset: r78689:5ca542b8ef96
Date: 2015-07-27 23:30 +0300
http://bitbucket.org/pypy/pypy/changeset/5ca542b8ef96/

Log:	close branch to be merged


From noreply at buildbot.pypy.org  Mon Jul 27 22:39:41 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 27 Jul 2015 22:39:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge nditer-revisited which provides
 'buffered' flag and other small improvements
Message-ID: <20150727203941.C1C571C149C@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78690:c1b066339482
Date: 2015-07-27 23:32 +0300
http://bitbucket.org/pypy/pypy/changeset/c1b066339482/

Log:	merge nditer-revisited which provides 'buffered' flag and other
	small improvements

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -9,7 +9,8 @@
 from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
                                             shape_agreement, shape_agreement_multiple)
-from pypy.module.micronumpy.casting import find_binop_result_dtype
+from pypy.module.micronumpy.casting import (find_binop_result_dtype, 
+                    can_cast_array, can_cast_type)
 
 
 def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
@@ -108,9 +109,7 @@
         if item == 'external_loop':
             nditer.external_loop = True
         elif item == 'buffered':
-            raise oefmt(space.w_NotImplementedError,
-                'nditer buffered not implemented yet')
-            # For numpy compatability
+            # Each iterator should be 1d
             nditer.buffered = True
         elif item == 'c_index':
             nditer.tracked_index = 'C'
@@ -213,30 +212,6 @@
         return arr
 
 
-def get_iter(space, order, arr, shape, dtype, op_flags, base):
-    imp = arr.implementation
-    backward = is_backward(imp, order)
-    if arr.is_scalar():
-        return ConcreteIter(imp, 1, [], [], [], op_flags, base)
-    if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \
-       (abs(imp.strides[0]) > abs(imp.strides[-1]) and backward):
-        # flip the strides. Is this always true for multidimension?
-        strides = imp.strides[:]
-        backstrides = imp.backstrides[:]
-        shape = imp.shape[:]
-        strides.reverse()
-        backstrides.reverse()
-        shape.reverse()
-    else:
-        strides = imp.strides
-        backstrides = imp.backstrides
-    r = calculate_broadcast_strides(strides, backstrides, imp.shape,
-                                    shape, backward)
-    if len(shape) != len(r[0]):
-        # shape can be shorter when using an external loop, just return a view
-        return ConcreteIter(imp, imp.get_size(), imp.shape, r[0], r[1], op_flags, base)
-    return ConcreteIter(imp, imp.get_size(), shape, r[0], r[1], op_flags, base)
-
 def calculate_ndim(op_in, oa_ndim):
     if oa_ndim >=0:
         return oa_ndim
@@ -308,6 +283,8 @@
     of shape (4,3) by setting the offset to the beginning of the data at each iteration
     '''
     shape = [s+1 for s in old_iter.shape_m1]
+    if len(shape) < 1:
+        return old_iter
     strides = old_iter.strides
     backstrides = old_iter.backstrides
     if order == 'F':
@@ -328,6 +305,8 @@
         _shape = [shape[-1]]  + old_iter.slice_shape
         _backstride = [(shape[-1] - 1) * strides[-1]] + old_iter.slice_backstride
         fastest = shape[-1]
+    if fastest == 0:
+        return old_iter
     if flat:
         _shape = [support.product(_shape)]
         if len(_stride) > 1:
@@ -383,6 +362,10 @@
         self.done = False
         self.first_next = True
         self.op_axes = []
+        if not space.is_w(w_casting, space.w_None):
+            self.casting = space.str_w(w_casting)
+        else:
+            self.casting = 'safe'
         # convert w_seq operands to a list of W_NDimArray
         if space.isinstance_w(w_seq, space.w_tuple) or \
            space.isinstance_w(w_seq, space.w_list):
@@ -445,8 +428,15 @@
                     self.seq[i] = W_NDimArray.from_shape(space, self.shape, out_dtype)
                 else:
                     if not self.op_flags[i].broadcast:
-                        # Raises if ooutput cannot be broadcast
-                        shape_agreement(space, self.shape, self.seq[i], False)
+                        # Raises if output cannot be broadcast
+                        try:
+                            shape_agreement(space, self.shape, self.seq[i], False)
+                        except OperationError as e:
+                            raise oefmt(space.w_ValueError, "non-broadcastable"
+                                " output operand with shape %s doesn't match "
+                                "the broadcast shape %s", 
+                                str(self.seq[i].get_shape()),
+                                str(self.shape)) 
 
         if self.tracked_index != "":
             if self.order == "K":
@@ -465,14 +455,41 @@
                 if not self_d:
                     self.dtypes[i] = seq_d
                 elif self_d != seq_d:
-                    if not 'r' in self.op_flags[i].tmp_copy:
-                        raise oefmt(space.w_TypeError,
-                                    "Iterator operand required copying or "
-                                    "buffering for operand %d", i)
-                    impl = self.seq[i].implementation
-                    order = support.get_order_as_CF(impl.order, self.order)
-                    new_impl = impl.astype(space, self_d, order)
-                    self.seq[i] = W_NDimArray(new_impl)
+                        impl = self.seq[i].implementation
+                        order = support.get_order_as_CF(impl.order, self.order)
+                        if self.buffered or 'r' in self.op_flags[i].tmp_copy:
+                            if not can_cast_array(
+                                    space, self.seq[i], self_d, self.casting):
+                                raise oefmt(space.w_TypeError, "Iterator operand %d"
+                                    " dtype could not be cast from %s to %s"
+                                    " according to the rule '%s'", i, 
+                                    space.str_w(seq_d.descr_repr(space)),
+                                    space.str_w(self_d.descr_repr(space)),
+                                    self.casting)
+ 
+                            new_impl = impl.astype(space, self_d, order).copy(space)
+                            self.seq[i] = W_NDimArray(new_impl)
+                        else:
+                            raise oefmt(space.w_TypeError, "Iterator "
+                                "operand required copying or buffering, "
+                                "but neither copying nor buffering was "
+                                "enabled")
+                        if 'w' in self.op_flags[i].rw:
+                            if not can_cast_type(
+                                    space, self_d, seq_d, self.casting):
+                                raise oefmt(space.w_TypeError, "Iterator"
+                                    " requested dtype could not be cast from "
+                                    " %s to %s, the operand %d dtype, accord"
+                                    "ing to the rule '%s'", 
+                                    space.str_w(self_d.descr_repr(space)),
+                                    space.str_w(seq_d.descr_repr(space)),
+                                    i, self.casting)
+        elif self.buffered:
+            for i in range(len(self.seq)):
+                if i not in outargs:
+                    self.seq[i] = self.seq[i].descr_copy(space,
+                                     w_order=space.wrap(self.order))
+            self.dtypes = [s.get_dtype() for s in self.seq]
         else:
             #copy them from seq
             self.dtypes = [s.get_dtype() for s in self.seq]
@@ -480,14 +497,43 @@
         # create an iterator for each operand
         self.iters = []
         for i in range(len(self.seq)):
-            it = get_iter(space, self.order, self.seq[i], self.shape,
-                          self.dtypes[i], self.op_flags[i], self)
+            it = self.get_iter(space, i)
             it.contiguous = False
             self.iters.append((it, it.reset()))
 
         if self.external_loop:
             coalesce_axes(self, space)
 
+    def get_iter(self, space, i):
+        arr = self.seq[i]
+        dtype = self.dtypes[i]
+        shape = self.shape
+        imp = arr.implementation
+        backward = is_backward(imp, self.order)
+        if arr.is_scalar():
+            return ConcreteIter(imp, 1, [], [], [], self.op_flags[i], self)
+        if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \
+           (abs(imp.strides[0]) > abs(imp.strides[-1]) and backward):
+            # flip the strides. Is this always true for multidimension?
+            strides = imp.strides[:]
+            backstrides = imp.backstrides[:]
+            shape = imp.shape[:]
+            strides.reverse()
+            backstrides.reverse()
+            shape.reverse()
+        else:
+            strides = imp.strides
+            backstrides = imp.backstrides
+        r = calculate_broadcast_strides(strides, backstrides, imp.shape,
+                                        shape, backward)
+        iter_shape = shape
+        if len(shape) != len(r[0]):
+            # shape can be shorter when using an external loop, just return a view
+            iter_shape = imp.shape
+        return ConcreteIter(imp, imp.get_size(), iter_shape, r[0], r[1],
+                            self.op_flags[i], self)
+
+
     def set_op_axes(self, space, w_op_axes):
         if space.len_w(w_op_axes) != len(self.seq):
             raise oefmt(space.w_ValueError,
@@ -520,8 +566,8 @@
         return space.wrap(self)
 
     def getitem(self, it, st):
-        res = it.getoperand(st)
-        return W_NDimArray(res)
+        w_res = W_NDimArray(it.getoperand(st))
+        return w_res
 
     def descr_getitem(self, space, w_idx):
         idx = space.int_w(w_idx)
diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -38,3 +38,6 @@
     a = zeros(*args, **kwargs)
     a.fill(1)
     return a
+
+def isscalar(a):
+    return type(a) in [typeinfo[t] for t in types]
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -64,18 +64,15 @@
         a = arange(24).reshape(2, 3, 4)
         import sys
         r = []
-        n = 0
         for x in nditer(a, flags=['external_loop']):
             r.append(x)
-            n += 1
-        assert n == 1
+        assert len(r) == 1
+        assert r[0].shape == (24,)
         assert (array(r) == range(24)).all()
         r = []
-        n = 0
         for x in nditer(a, flags=['external_loop'], order='F'):
             r.append(x)
-            n += 1
-        assert n == 12
+        assert len(r) == 12
         assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21],
                              [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23],
                             ]).all()
@@ -149,19 +146,60 @@
         # assert str(exc.value).startswith("Iterator flag EXTERNAL_LOOP cannot")
 
     def test_buffered(self):
-        from numpy import arange, nditer, array
-        a = arange(6).reshape(2,3)
-        import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['buffered'])
-            skip('nditer buffered not implmented')
+        from numpy import arange, nditer, array, isscalar
+        a = arange(24).reshape(2, 3, 4)
+        r = []
+        for x in nditer(a, flags=['external_loop'], order='F'):
+            r.append(x)
+        array_r = array(r)
+        assert len(array_r.shape) == 2
+        assert array_r.shape == (12, 2)
+        assert (array_r == [[0, 12], [4, 16], [8, 20], [1, 13], [5, 17], [9, 21],
+                      [2, 14], [6, 18], [10, 22], [3, 15], [7, 19], [11, 23]]).all
+        assert (a == arange(24).reshape(2, 3, 4)).all()
+        a[0,0,0] = 100
+        assert r[0][0] == 100
+
+        r = []
+        try:
+            it = nditer(a, flags=['buffered'], order='F')
+        except NotImplementedError as e:
+            assert 'unsupported value for order' in str(e)
+            skip('buffered with order="F" requires fortran tmp array creation')
+        for x in it:
+            r.append(x)
+        array_r = array(r)
+        assert len(array_r.shape) == 1
+        assert array_r.shape == (24,)
+        assert r[0].shape == ()
+        assert not isscalar(r[0])
+        assert (array_r == [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21,
+                      2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23]).all
+        assert a.shape == (2, 3, 4)
+        a[0,0,0] = 0
+        # buffered copies the data into a tmp array
+        assert r[0] == 100
+        assert (a == arange(24).reshape(2, 3, 4)).all()
+
         r = []
         for x in nditer(a, flags=['external_loop', 'buffered'], order='F'):
             r.append(x)
-        array_r = array(r)
-        assert len(array_r.shape) == 2
-        assert array_r.shape == (1, 6)
-        assert (array_r == [0, 3, 1, 4, 2, 5]).all()
+        assert r[0].shape == (24,)
+        assert (array_r == [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21,
+                      2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23]).all
+        assert a.shape == (2, 3, 4)
+        assert (a == arange(24).reshape(2, 3, 4)).all()
+
+    def test_zerosize(self):
+        from numpy import nditer, array
+        for a in [ array([]), array([1]), array([1, 2]) ]:
+            buffersize = max(16 * 1024 ** 2 // a.itemsize, 1)
+            r = []
+            for chunk in nditer(a, 
+                    flags=['external_loop', 'buffered', 'zerosize_ok'],
+                    buffersize=buffersize, order='C'):
+                r.append(chunk)
+            assert (r == a).all()
 
     def test_op_dtype(self):
         from numpy import arange, nditer, sqrt, array
@@ -188,11 +226,10 @@
         from numpy import arange, nditer
         import sys
         a = arange(6.)
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['buffered'], op_dtypes=['float32'])
-            skip('nditer casting not implemented yet')
         exc = raises(TypeError, nditer, a, flags=['buffered'], op_dtypes=['float32'])
-        assert str(exc.value).startswith("Iterator operand 0 dtype could not be cast")
+        assert str(exc.value) == "Iterator operand 0 dtype could not be " + \
+            "cast from dtype('float64') to dtype('float32') according to the" +\
+            " rule 'safe'"
         r = []
         for x in nditer(a, flags=['buffered'], op_dtypes=['float32'],
                                 casting='same_kind'):
@@ -232,9 +269,6 @@
             return it.operands[1]
         assert (square1([1, 2, 3]) == [1, 4, 9]).all()
 
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, [1, 2], flags=['buffered'])
-            skip('nditer buffered not implmented')
         def square2(a, out=None):
             it = nditer([a, out], flags=['external_loop', 'buffered'],
                         op_flags=[['readonly'],

From noreply at buildbot.pypy.org  Mon Jul 27 22:39:42 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 27 Jul 2015 22:39:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <20150727203942.E6AD51C14D3@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r78691:721e9a1bc4e3
Date: 2015-07-27 23:35 +0300
http://bitbucket.org/pypy/pypy/changeset/721e9a1bc4e3/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -51,3 +51,7 @@
 .. branch: numpy-docstrings
 
 Allow the docstrings of built-in numpy objects to be set at run-time.
+
+.. branch: nditer-revisited
+
+Implement nditer 'buffered' flag and fix some edge cases

From noreply at buildbot.pypy.org  Tue Jul 28 18:45:40 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 28 Jul 2015 18:45:40 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: restricted the test environment,
 loop versioning only works resumedescr (not basicfaildescr)
Message-ID: <20150728164540.689D51C11FB@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78695:fce17b8ec26a
Date: 2015-07-28 17:02 +0200
http://bitbucket.org/pypy/pypy/changeset/fce17b8ec26a/

Log:	restricted the test environment, loop versioning only works
	resumedescr (not basicfaildescr) rawstart is now saved to the loop
	token to be more easily reconstructed when mc and datablockwrappers
	are removed from the assembler

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -258,6 +258,9 @@
         self.stats = stats or MiniStats()
         self.vinfo_for_tests = kwds.get('vinfo_for_tests', None)
 
+    def stitch_bridge(self, faildescr, jitcell_token):
+        pass
+
     def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
                      unique_id=0, log=True, name='', logger=None):
         clt = model.CompiledLoopToken(self, looptoken.number)
@@ -868,17 +871,23 @@
 
     def fail_guard(self, descr, saved_data=None):
         values = []
-        for box in self.current_op.getfailargs():
+        for i,box in enumerate(self.current_op.getfailargs()):
             if box is not None:
                 value = self.env[box]
             else:
                 value = None
-            if box and box.getaccum():
-                if box.getaccum().operator == '+':
+            accum = descr.rd_accum_list
+            while accum != None:
+                if accum.position != i:
+                    accum = accum.prev
+                    continue
+                if accum.operation == '+':
                     value = sum(value)
-                elif box.getaccum().operator == '*':
+                    break
+                elif accum.operation == '*':
                     def prod(acc, x): return acc * x
                     value = reduce(prod, value, 1)
+                    break
                 else:
                     raise NotImplementedError("accum operator in fail guard")
             values.append(value)
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -35,6 +35,7 @@
         self.exc = exc
         self.is_guard_not_invalidated = is_guard_not_invalidated
         self.is_guard_not_forced = is_guard_not_forced
+        self.rawstart = 0
 
     def compute_gcmap(self, gcmap, failargs, fail_locs, frame_depth):
         # note that regalloc has a very similar compute, but
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -503,6 +503,7 @@
         full_size = self.mc.get_relative_pos()
         #
         rawstart = self.materialize_loop(looptoken)
+        looptoken.rawstart = rawstart
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
         looptoken._ll_loop_code = looppos + rawstart
@@ -590,8 +591,8 @@
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
     def stitch_bridge(self, faildescr, token):
-        rawstart = self.materialize_loop(token)
-        self.patch_jump_for_descr(faildescr, rawstart)
+        assert token.rawstart != 0
+        self.patch_jump_for_descr(faildescr, token.rawstart)
 
     def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -1,4 +1,5 @@
 import py
+from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
     ConstFloat, BoxInt, BoxFloat, BoxVector, INT, REF,
     FLOAT, VECTOR, TargetToken)
@@ -65,6 +66,8 @@
     def _accum_update_at_exit(self, fail_locs, fail_args, faildescr, regalloc):
         """ If accumulation is done in this loop, at the guard exit
         some vector registers must be adjusted to yield the correct value"""
+        if not isinstance(faildescr, ResumeGuardDescr):
+            return
         assert regalloc is not None
         accum_info = faildescr.rd_accum_list
         while accum_info:

From noreply at buildbot.pypy.org  Tue Jul 28 18:45:41 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 28 Jul 2015 18:45:41 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: all but 2 vectoriztion tests passing
 again. the scheduling that prefers pure operations messes up these test
 cases
Message-ID: <20150728164541.916791C11FF@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78696:08d59f3ff88c
Date: 2015-07-28 18:45 +0200
http://bitbucket.org/pypy/pypy/changeset/08d59f3ff88c/

Log:	all but 2 vectoriztion tests passing again. the scheduling that
	prefers pure operations messes up these test cases

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -11,6 +11,7 @@
     xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
     X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, AddressLoc)
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
+from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem import lltype
@@ -170,19 +171,23 @@
                 self.mc.MOVUPD(dest_loc, value_loc)
 
     def genop_vec_int_is_true(self, op, arglocs, resloc):
-        loc, size = arglocs
+        loc, sizeloc = arglocs
         temp = X86_64_XMM_SCRATCH_REG
         self.mc.PXOR(temp, temp)
         # every entry that is non zero -> becomes zero
         # zero entries become ones
-        self.mc.PCMPEQ(loc, temp, size)
+        self.mc.PCMPEQ(loc, temp, sizeloc.value)
         # a second time -> every zero entry (corresponding to non zero
         # entries before) become ones
-        self.mc.PCMPEQ(loc, temp, size)
+        self.mc.PCMPEQ(loc, temp, sizeloc.value)
 
     def genop_guard_vec_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
         self._guard_vector_true(op, arglocs[0])
-        self.implement_guard(guard_token, 'NZ')
+        guard_opnum = guard_op.getopnum()
+        if guard_opnum == rop.GUARD_TRUE:
+            self.implement_guard(guard_token, 'NZ')
+        else:
+            self.implement_guard(guard_token, 'Z')
 
     def genop_vec_int_mul(self, op, arglocs, resloc):
         loc0, loc1, itemsize_loc = arglocs
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
 from rpython.jit.metainterp.optimizeopt.vectorize import (VecScheduleData,
         Pack, NotAProfitableLoop, VectorizingOptimizer)
-from rpython.jit.metainterp.optimizeopt.dependency import Node
+from rpython.jit.metainterp.optimizeopt.dependency import Node, DependencyGraph
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
 from rpython.jit.metainterp.optimizeopt.test.test_schedule import SchedulerBaseTest
 from rpython.jit.metainterp.optimizeopt.test.test_vectorize import (FakeMetaInterpStaticData,
@@ -35,8 +35,7 @@
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
         jitdriver_sd = FakeJitDriverStaticData()
         opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, [])
-        opt.build_dependency_graph()
-        graph = opt.dependency_graph
+        graph = opt.dependency_graph = DependencyGraph(loop)
         for k,m in graph.memory_refs.items():
             graph.memory_refs[k] = FakeMemoryRef(m.array, m.index_var)
         opt.find_adjacent_memory_refs()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -82,7 +82,7 @@
         opt.loop.operations = opt.get_newoperations()
         self.debug_print_operations(opt.loop)
         opt.clear_newoperations()
-        opt.build_dependency_graph()
+        opt.dependency_graph = DependencyGraph(loop)
         self.last_graph = opt.dependency_graph
         self.show_dot_graph(self.last_graph, self.test_name)
         return opt
@@ -278,20 +278,20 @@
         """
         opt_ops = """
         [p0,p1,p2,i0]
+        i4 = int_add(i0, 1)
+        i5 = int_le(i4, 10)
+        guard_true(i5) []
         i1 = raw_load(p1, i0, descr=floatarraydescr)
         i2 = raw_load(p2, i0, descr=floatarraydescr)
         i3 = int_add(i1,i2)
         raw_store(p0, i0, i3, descr=floatarraydescr)
-        i4 = int_add(i0, 1)
-        i5 = int_le(i4, 10)
-        guard_true(i5) []
+        i9 = int_add(i4, 1)
+        i10 = int_le(i9, 10)
+        guard_true(i10) []
         i6 = raw_load(p1, i4, descr=floatarraydescr)
         i7 = raw_load(p2, i4, descr=floatarraydescr)
         i8 = int_add(i6,i7)
         raw_store(p0, i4, i8, descr=floatarraydescr)
-        i9 = int_add(i4, 1)
-        i10 = int_le(i9, 10)
-        guard_true(i10) []
         jump(p0,p1,p2,i9)
         """
         self.assert_unroll_loop_equals(self.parse_loop(ops), self.parse_loop(opt_ops), 1)
@@ -334,8 +334,8 @@
         i4 = raw_load(p0,i1,descr=chararraydescr)
         jump(p0,i3,i4)
         """
-        vopt = self.vectoroptimizer_unrolled(self.parse_loop(ops),0)
-        vopt.build_dependency_graph()
+        loop = self.parse_loop(ops)
+        vopt = self.vectoroptimizer_unrolled(loop,0)
         assert len(vopt.dependency_graph.memory_refs) == 2
         self.assert_has_memory_ref_at(1)
         self.assert_has_memory_ref_at(2)
@@ -571,7 +571,7 @@
         """
         vopt = self.vectoroptimizer_unrolled(self.parse_loop(ops),0)
         vopt.find_adjacent_memory_refs()
-        mref = self.getmemref(3)
+        mref = self.getmemref(5)
         mref2 = self.getmemref(6)
 
         self.assert_memory_ref_not_adjacent(mref, mref2)
@@ -591,7 +591,7 @@
         """
         vopt = self.vectoroptimizer_unrolled(self.parse_loop(ops),0)
         vopt.find_adjacent_memory_refs()
-        mref = self.getmemref(3)
+        mref = self.getmemref(6)
         mref2 = self.getmemref(7)
 
         self.assert_memory_ref_not_adjacent(mref, mref2)
@@ -611,7 +611,7 @@
         """
         vopt = self.vectoroptimizer_unrolled(self.parse_loop(ops),0)
         vopt.find_adjacent_memory_refs()
-        mref = self.getmemref(3)
+        mref = self.getmemref(6)
         mref2 = self.getmemref(7)
 
         self.assert_memory_ref_not_adjacent(mref, mref2)
@@ -628,7 +628,7 @@
         """
         loop = self.parse_loop(ops)
         vopt = self.init_packset(loop,1)
-        self.assert_independent(1,5)
+        self.assert_independent(4,8)
         assert vopt.packset is not None
         assert len(vopt.dependency_graph.memory_refs) == 2
         assert len(vopt.packset.packs) == 1
@@ -748,18 +748,18 @@
         loop = self.parse_loop(ops)
         vopt = self.extend_packset(loop,1)
         assert len(vopt.dependency_graph.memory_refs) == 4
+        self.assert_independent(4,10)
         self.assert_independent(5,11)
         self.assert_independent(6,12)
-        self.assert_independent(7,13)
         assert len(vopt.packset.packs) == 3
         self.assert_packset_empty(vopt.packset, len(loop.operations),
-                                  [(6,12), (5,11), (7,13)])
+                                  [(6,12), (5,11), (4,10)])
 
     @pytest.mark.parametrize("descr,packs,packidx", 
-                             [('char',1,  [(0,(1,3,5,7))]),
-                              ('float',2, [(0,(1,3)),(1,(5,7))]),
-                              ('int',2,   [(0,(1,3)),(1,(5,7))]),
-                              ('singlefloat',1,[(0,(1,3,5,7))])])
+                             [('char',1,  [(0,(2,4,6,8))]),
+                              ('float',2, [(0,(2,4)),(1,(6,8))]),
+                              ('int',2,   [(0,(2,4)),(1,(6,8))]),
+                              ('singlefloat',1,[(0,(2,4,6,8))])])
     def test_packset_combine_simple(self,descr,packs,packidx):
         ops = """
         [p0,i0]
@@ -849,7 +849,7 @@
             assert len(vopt.packset.packs) == 4
 
         for opindices in [(5,12,19,26),(6,13,20,27),
-                          (7,14,21,28),(8,15,22,29)]:
+                          (7,14,21,28),(4,11,18,25)]:
             self.assert_has_pack_with(vopt.packset, opindices)
 
     @pytest.mark.parametrize('op,descr,stride',
@@ -874,7 +874,6 @@
         """.format(op=op,descr=descr,stride=1) # stride getarray is always 1
         vops = """
         [p0,p1,p2,i0]
-        guard_early_exit() []
         i10 = int_le(i0, 128)
         guard_true(i10) []
         i1 = int_add(i0, {stride})
@@ -907,7 +906,6 @@
         """
         opt="""
         [i0, i1, i2, i3, i4]
-        guard_early_exit() []
         i11 = int_add(i0, 1) 
         i6 = int_mul(i0, 8) 
         i12 = int_lt(i11, i1) 
@@ -941,7 +939,6 @@
           for i in range(0,14)])
         opt="""
         [p0,i0]
-        guard_early_exit() [p0,i0]
         i200 = int_add(i0, 1)
         i400 = int_lt(i200, 102)
         i2 = int_add(i0, 16)
@@ -989,7 +986,6 @@
         [p0,i0]
         v3 = vec_int_expand(42)
         label(p0,i0,v3)
-        guard_early_exit() [p0,i0]
         i20 = int_add(i0, 1)
         i30 = int_lt(i20, 10)
         i2 = int_add(i0, 2)
@@ -1019,7 +1015,6 @@
         [p0,i0,f3]
         v3 = vec_float_expand(f3)
         label(p0,i0,f3,v3)
-        guard_early_exit() [p0,i0]
         i20 = int_add(i0, 1)
         i30 = int_lt(i20, 10)
         i2 = int_add(i0, 2)
@@ -1047,7 +1042,6 @@
         """
         trace_opt = """
         [p0, i0, v2[f64|2]]
-        guard_early_exit() [p0, i0, v2[f64|2]]
         i1 = int_add(i0, 16)
         i2 = int_lt(i1, 100)
         guard_false(i2) [p0, i0, v[f64|2]]
@@ -1103,7 +1097,6 @@
         opt = """
         [p36, i28, p9, i37, p14, f34, p12, p38, f35, p39, i40, i41, p42, i43, i44, i21, i4, i0, i18]
         guard_not_invalidated() [p38, p12, p9, p14, p39, i37, i44, f35, i40, p42, i43, f34, i28, p36, i41]
-        guard_early_exit() [p38, p12, p9, p14, p39, i37, i44, f35, i40, p42, i43, f34, i28, p36, i41]
         i50 = int_add(i28, 1) 
         i46 = int_add(i44, 8) 
         i48 = int_add(i41, 8) 
@@ -1142,7 +1135,6 @@
         """
         opt = """
         [p0, p1, i1]
-        guard_early_exit() []
         i3 = int_add(i1, 1)
         i4 = int_ge(i3, 36)
         i50 = int_add(i1, 4)
@@ -1184,7 +1176,6 @@
         """
         opt = """
         [p0, p1, p2, i0, i4]
-        guard_early_exit() []
         i5 = int_add(i4, 4)
         i1 = int_add(i0, 4)
         i186 = int_lt(i5, 100)
@@ -1219,39 +1210,6 @@
         vopt = self.vectorize(self.parse_loop(ops))
         self.assert_equal(vopt.loop, self.parse_loop(opt))
 
-    def test_call_prohibits_vectorization(self):
-        # think about this
-        py.test.skip("")
-        ops = """
-        [p31, i32, p3, i33, f10, p24, p34, p35, i19, p5, i36, p37, i28, f13, i29, i15]
-        guard_early_exit() [p5,p37,p34,p3,p24,i32,p35,i36,i33,f10,p31,i19]
-        f38 = raw_load(i28, i33, descr=floatarraydescr)
-        guard_not_invalidated()[p5,p37,p34,p3,p24,f38,i32,p35,i36,i33,None,p31,i19]
-        i39 = int_add(i33, 8) 
-        f40 = float_mul(f38, 0.0)
-        i41 = float_eq(f40, f40)
-        guard_true(i41) [p5,p37,p34,p3,p24,f13,f38,i39,i32,p35,i36,None,None,p31,i19]
-        f42 = call(111, f38, f13, descr=writeadescr)
-        i43 = call(222, 333, descr=writeadescr)
-        f44 = float_mul(f42, 0.0)
-        i45 = float_eq(f44, f44)
-        guard_true(i45) [p5,p37,p34,p3,p24,f13,f38,i43,f42,i39,i32,p35,i36,None,None,p31,i19]
-        i46 = int_is_true(i43)
-        guard_false(i46) [p5,p37,p34,p3,p24,f13,f38,i43,f42,i39,i32,p35,i36,None,None,p31,i19]
-        raw_store(i29, i36, f42, descr=floatarraydescr)
-        i47 = int_add(i19, 1)
-        i48 = int_add(i36, 8)
-        i49 = int_ge(i47, i15)
-        guard_false(i49) [p5,p37,p34,p3,p24,i47,f38,i48,i39,i32,p35,None,None,None,p31,None]
-        jump(p31, i32, p3, i39, f38, p24, p34, p35, i47, p5, i48, p37, i28, f13, i29, i15)
-        """
-        try:
-            vopt = self.vectorize(self.parse_loop(ops))
-            self.debug_print_operations(vopt.loop)
-            py.test.fail("this loop should not be vectorized")
-        except NotAVectorizeableLoop:
-            pass
-
     def test_truediv_abs_neg_float(self):
         ops = """
         [f9,p10,i11,p4,i12,p2,p5,p13,i14,p7,i15,p8,i16,f17,i18,i19]
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -229,18 +229,17 @@
                 # that are needed to resume.
                 if copied_op.is_guard():
                     assert isinstance(copied_op, GuardResOp)
-                    target_guard = copied_op
                     descr = copied_op.getdescr()
-                    assert isinstance(descr, ResumeGuardDescr)
-                    copied_op.setdescr(descr.clone())
-                    descr = target_guard.getdescr()
-                    # copy failargs/snapshot
-                    copied_op.rd_snapshot = \
-                      renamer.rename_rd_snapshot(copied_op.rd_snapshot,
-                                                 clone=True)
-                    renamed_failargs = \
-                        renamer.rename_failargs(copied_op, clone=True)
-                    copied_op.setfailargs(renamed_failargs)
+                    if descr:
+                        assert isinstance(descr, ResumeGuardDescr)
+                        copied_op.setdescr(descr.clone())
+                        # copy failargs/snapshot
+                        copied_op.rd_snapshot = \
+                          renamer.rename_rd_snapshot(copied_op.rd_snapshot,
+                                                     clone=True)
+                        renamed_failargs = \
+                            renamer.rename_failargs(copied_op, clone=True)
+                        copied_op.setfailargs(renamed_failargs)
                 #
                 self.emit_unrolled_operation(copied_op)
 

From noreply at buildbot.pypy.org  Tue Jul 28 19:37:18 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 28 Jul 2015 19:37:18 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: Refactor do_axis_reduce() so that
 it takes the axis_flags list instead of a single axis
Message-ID: <20150728173718.B635E1C1247@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78697:77850d7d684c
Date: 2015-07-28 18:37 +0100
http://bitbucket.org/pypy/pypy/changeset/77850d7d684c/

Log:	Refactor do_axis_reduce() so that it takes the axis_flags list
	instead of a single axis

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -339,30 +339,60 @@
                                    greens=['shapelen', 'func', 'dtype'],
                                    reds='auto')
 
-def do_axis_reduce(space, func, arr, dtype, axis, out, identity):
-    out_iter = AxisIter(out.implementation, arr.get_shape(), axis)
-    out_state = out_iter.reset()
-    arr_iter, arr_state = arr.create_iter()
-    arr_iter.track_index = False
+def do_axis_reduce(space, func, arr, dtype, axis_flags, out, identity):
+    out_iter, out_state = out.create_iter()
+    out_iter.track_index = False
+    shape = arr.get_shape()
+    strides = arr.implementation.get_strides()
+    backstrides = arr.implementation.get_backstrides()
+    shapelen = len(shape)
+    inner_shape = [-1] * shapelen
+    inner_strides = [-1] * shapelen
+    inner_backstrides = [-1] * shapelen
+    outer_shape = [-1] * shapelen
+    outer_strides = [-1] * shapelen
+    outer_backstrides = [-1] * shapelen
+    for i in range(len(shape)):
+        if axis_flags[i]:
+            inner_shape[i] = shape[i]
+            inner_strides[i] = strides[i]
+            inner_backstrides[i] = backstrides[i]
+            outer_shape[i] = 1
+            outer_strides[i] = 0
+            outer_backstrides[i] = 0
+        else:
+            outer_shape[i] = shape[i]
+            outer_strides[i] = strides[i]
+            outer_backstrides[i] = backstrides[i]
+            inner_shape[i] = 1
+            inner_strides[i] = 0
+            inner_backstrides[i] = 0
+    inner_iter = ArrayIter(arr.implementation, support.product(inner_shape),
+                           inner_shape, inner_strides, inner_backstrides)
+    outer_iter = ArrayIter(arr.implementation, support.product(outer_shape),
+                           outer_shape, outer_strides, outer_backstrides)
+    assert outer_iter.size == out_iter.size
+
     if identity is not None:
         identity = identity.convert_to(space, dtype)
-    shapelen = len(out.get_shape())
-    while not out_iter.done(out_state):
-        axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
-                                           dtype=dtype)
-        w_val = arr_iter.getitem(arr_state).convert_to(space, dtype)
-        arr_state = arr_iter.next(arr_state)
-
-        out_indices = out_iter.indices(out_state)
-        if out_indices[axis] == 0:
-            if identity is not None:
-                w_val = func(dtype, identity, w_val)
+    outer_state = outer_iter.reset()
+    while not outer_iter.done(outer_state):
+        inner_state = inner_iter.reset()
+        inner_state.offset = outer_state.offset
+        if identity is not None:
+            w_val = identity
         else:
-            cur = out_iter.getitem(out_state)
-            w_val = func(dtype, cur, w_val)
-
+            w_val = inner_iter.getitem(inner_state).convert_to(space, dtype)
+            inner_state = inner_iter.next(inner_state)
+        while not inner_iter.done(inner_state):
+            axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
+                                            dtype=dtype)
+            w_item = inner_iter.getitem(inner_state).convert_to(space, dtype)
+            w_val = func(dtype, w_item, w_val)
+            inner_state = inner_iter.next(inner_state)
         out_iter.setitem(out_state, w_val)
         out_state = out_iter.next(out_state)
+        outer_state = outer_iter.next(outer_state)
     return out
 
 
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -410,7 +410,7 @@
                 if self.identity is not None:
                     out.fill(space, self.identity.convert_to(space, dtype))
                 return out
-            loop.do_axis_reduce(space, self.func, obj, dtype, axis,
+            loop.do_axis_reduce(space, self.func, obj, dtype, axis_flags,
                                 out, self.identity)
             if call__array_wrap__:
                 out = space.call_method(obj, '__array_wrap__', out)

From noreply at buildbot.pypy.org  Tue Jul 28 20:29:40 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 28 Jul 2015 20:29:40 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: stripping sse4 of the model returned
 from host platform in the test to pass it (c compiler will never append
 sse4)
Message-ID: <20150728182940.6E0FA1C11FF@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78698:152353977ab7
Date: 2015-07-28 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/152353977ab7/

Log:	stripping sse4 of the model returned from host platform in the test
	to pass it (c compiler will never append sse4) considering that
	descr can be a basicfaildescr in the tests

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -876,20 +876,21 @@
                 value = self.env[box]
             else:
                 value = None
-            accum = descr.rd_accum_list
-            while accum != None:
-                if accum.position != i:
-                    accum = accum.prev
-                    continue
-                if accum.operation == '+':
-                    value = sum(value)
-                    break
-                elif accum.operation == '*':
-                    def prod(acc, x): return acc * x
-                    value = reduce(prod, value, 1)
-                    break
-                else:
-                    raise NotImplementedError("accum operator in fail guard")
+            if hasattr(descr, 'rd_accum_list'):
+                accum = descr.rd_accum_list
+                while accum != None:
+                    if accum.position != i:
+                        accum = accum.prev
+                        continue
+                    if accum.operation == '+':
+                        value = sum(value)
+                        break
+                    elif accum.operation == '*':
+                        def prod(acc, x): return acc * x
+                        value = reduce(prod, value, 1)
+                        break
+                    else:
+                        raise NotImplementedError("accum operator in fail guard")
             values.append(value)
         if hasattr(descr, '_llgraph_bridge'):
             target = (descr._llgraph_bridge, -1)
diff --git a/rpython/jit/backend/test/test_detect_cpu.py b/rpython/jit/backend/test/test_detect_cpu.py
--- a/rpython/jit/backend/test/test_detect_cpu.py
+++ b/rpython/jit/backend/test/test_detect_cpu.py
@@ -31,6 +31,8 @@
 def test_detect_model_from_c_compiler():
     info1 = detect_model_from_host_platform()
     info2 = detect_model_from_c_compiler()
+    if info1.endswith("-sse4"):
+        info1 = info1[:-len("-sse4")]
     assert info1 == info2
 
 def test_getcpufeatures():
diff --git a/rpython/jit/metainterp/test/test_resoperation.py b/rpython/jit/metainterp/test/test_resoperation.py
--- a/rpython/jit/metainterp/test/test_resoperation.py
+++ b/rpython/jit/metainterp/test/test_resoperation.py
@@ -2,6 +2,7 @@
 import re
 from rpython.jit.metainterp import resoperation as rop
 from rpython.jit.metainterp.history import AbstractDescr, AbstractFailDescr
+from rpython.jit.metainterp.history import ConstInt
 
 def test_arity_mixins():
     cases = [
@@ -85,10 +86,10 @@
     py.test.raises(AssertionError, "newops[0].setdescr('foobar')")
 
 def test_cast_ops():
-    op = rop.ResOperation(rop.rop.INT_SIGNEXT, ['a', 1], 'c')
+    op = rop.ResOperation(rop.rop.INT_SIGNEXT, ['a', ConstInt(1)], 'c')
     assert op.casts_box()
     assert isinstance(op, rop.CastResOp)
-    assert op.cast_to == ('i',1)
+    assert op.cast_to() == ('i',1)
     op = rop.ResOperation(rop.rop.CAST_FLOAT_TO_INT, ['a'], 'c')
     assert op.casts_box()
     assert isinstance(op, rop.CastResOp)

From noreply at buildbot.pypy.org  Tue Jul 28 20:48:34 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 28 Jul 2015 20:48:34 +0200 (CEST)
Subject: [pypy-commit] pypy ufunc-reduce: Support multiple axes in
	ufunc.reduce()
Message-ID: <20150728184834.E97F01C1247@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: ufunc-reduce
Changeset: r78699:cfb5865ce1e9
Date: 2015-07-28 19:19 +0100
http://bitbucket.org/pypy/pypy/changeset/cfb5865ce1e9/

Log:	Support multiple axes in ufunc.reduce()

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1044,6 +1044,13 @@
             assert np.equal.reduce([1, 2], dtype=dtype) == True
             assert np.equal.reduce([1, 2, 0], dtype=dtype) == False
 
+    def test_reduce_axes(self):
+        import numpy as np
+        a = np.arange(24).reshape(2, 3, 4)
+        b = np.add.reduce(a, axis=(0, 1))
+        assert b.shape == (4,)
+        assert (b == [60, 66, 72, 78]).all()
+
     def test_reduce_fmax(self):
         import numpy as np
         assert np.fmax.reduce(np.arange(11).astype('b')) == 10
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -247,6 +247,18 @@
         if space.is_none(w_axis):
             axes = range(shapelen)
             axis = maxint
+        elif space.isinstance_w(w_axis, space.w_tuple):
+            axes_w = space.listview(w_axis)
+            axes = [0] * len(axes_w)
+            for i in range(len(axes_w)):
+                x = space.int_w(axes_w[i])
+                if x < 0:
+                    x += shapelen
+                if x < 0 or x >= shapelen:
+                    raise oefmt(space.w_ValueError, "'axis' entry is out of bounds")
+                axes[i] = x
+
+
         else:
             if space.isinstance_w(w_axis, space.w_tuple) and space.len_w(w_axis) == 1:
                 w_axis = space.getitem(w_axis, space.wrap(0))
@@ -256,7 +268,6 @@
             if axis < 0:
                 axis += shapelen
             axes = [axis]
-        assert axis >= 0
         dtype = decode_w_dtype(space, dtype)
 
         if dtype is None and out is not None:
@@ -277,12 +288,11 @@
             dtype = num2dtype(space, num)
 
         if self.identity is None:
-            for i in range(shapelen):
-                if space.is_none(w_axis) or i == axis:
-                    if obj_shape[i] == 0:
-                        raise oefmt(space.w_ValueError,
-                            "zero-size array to reduction operation %s "
-                            "which has no identity", self.name)
+            for i in axes:
+                if obj_shape[i] == 0:
+                    raise oefmt(space.w_ValueError,
+                        "zero-size array to reduction operation %s "
+                        "which has no identity", self.name)
 
         if variant == ACCUMULATE:
             if len(axes) != 1:
@@ -379,9 +389,16 @@
         else:
             temp = None
             if keepdims:
-                shape = obj_shape[:axis] + [1] + obj_shape[axis + 1:]
+                shape = obj_shape[:]
+                for axis in axes:
+                    shape[axis] = 1
             else:
-                shape = obj_shape[:axis] + obj_shape[axis + 1:]
+                shape = [0] * (shapelen - len(axes))
+                j = 0
+                for i in range(shapelen):
+                    if not axis_flags[i]:
+                        shape[j] = obj_shape[i]
+                        j += 1
             if out:
                 # Test for shape agreement
                 # XXX maybe we need to do broadcasting here, although I must

From noreply at buildbot.pypy.org  Wed Jul 29 09:16:33 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 29 Jul 2015 09:16:33 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: use ensure_gs_register in more places
Message-ID: <20150729071633.7176B1C056E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: use-gcc
Changeset: r1898:983a542003e9
Date: 2015-07-29 08:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/983a542003e9/

Log:	use ensure_gs_register in more places

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1345,6 +1345,9 @@
         write_fence();
         assert(_stm_detached_inevitable_from_thread == -1);
         _stm_detached_inevitable_from_thread = 0;
+    } else {
+        /* we certainly are not detached right now */
+        assert(_stm_detached_inevitable_from_thread != (intptr_t)STM_SEGMENT->running_thread);
     }
 
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -122,6 +122,8 @@
         dprintf(("reattach_transaction: commit detached from seg %d\n",
                  remote_seg_num));
 
+        assert(tl != old_tl);
+
         tl->last_associated_segment_num = remote_seg_num;
         ensure_gs_register(remote_seg_num);
         commit_external_inevitable_transaction();
@@ -135,6 +137,7 @@
 {
     dprintf(("> stm_force_transaction_break()\n"));
     assert(STM_SEGMENT->running_thread == tl);
+    assert(!stm_is_atomic(tl));
     _stm_commit_transaction();
     _stm_start_transaction(tl);
 }
@@ -180,14 +183,9 @@
     dprintf(("commit_fetched_detached_transaction from seg %d\n", segnum));
     assert(segnum > 0);
 
-    if (segnum != mysegnum) {
-        set_gs_register(get_segment_base(segnum));
-    }
+    ensure_gs_register(segnum);
     commit_external_inevitable_transaction();
-
-    if (segnum != mysegnum) {
-        set_gs_register(get_segment_base(mysegnum));
-    }
+    ensure_gs_register(mysegnum);
 }
 
 static void commit_detached_transaction_if_from(stm_thread_local_t *tl)
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -87,7 +87,7 @@
     assert(tl->last_associated_segment_num == i);
     assert(in_transaction(tl));
     assert(pr->transaction_state != TS_INEVITABLE);
-    set_gs_register(get_segment_base(i));
+    ensure_gs_register(i);
     assert(STM_SEGMENT->segment_num == i);
 
     s_mutex_lock();
@@ -155,7 +155,7 @@
     int segnum = fork_this_tl->last_associated_segment_num;
     assert(1 <= segnum && segnum < NB_SEGMENTS);
     *_get_cpth(fork_this_tl) = pthread_self();
-    set_gs_register(get_segment_base(segnum));
+    ensure_gs_register(segnum);
     assert(STM_SEGMENT->segment_num == segnum);
 
     if (!fork_was_in_transaction) {
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -723,7 +723,7 @@
 
     /* including the sharing seg0 */
     for (i = 0; i < NB_SEGMENTS; i++) {
-        set_gs_register(get_segment_base(i));
+        ensure_gs_register(i);
 
         bool ok = _stm_validate();
         assert(get_priv_segment(i)->last_commit_log_entry->next == NULL
@@ -769,7 +769,7 @@
         }
     }
 
-    set_gs_register(get_segment_base(original_num));
+    ensure_gs_register(original_num);
 }
 
 
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -66,7 +66,6 @@
 
 static void ensure_gs_register(long segnum)
 {
-    /* XXX use this instead of set_gs_register() in many places */
     if (STM_SEGMENT->segment_num != segnum) {
         set_gs_register(get_segment_base(segnum));
         assert(STM_SEGMENT->segment_num == segnum);
@@ -211,16 +210,12 @@
     assert(_has_mutex());
     assert(_is_tl_registered(tl));
 
-    int num = tl->last_associated_segment_num - 1; // 0..NB_SEG-1
+    int num = tl->last_associated_segment_num - 1; // 0..NB_SEG-2
     OPT_ASSERT(num >= 0);
     if (sync_ctl.in_use1[num+1] == 0) {
         /* fast-path: we can get the same segment number than the one
-           we had before.  The value stored in GS is still valid. */
-#ifdef STM_TESTS
-        /* that can be optimized away, except during tests, because
-           they use only one thread */
-        set_gs_register(get_segment_base(num+1));
-#endif
+           we had before.  The value stored in GS may still be valid. */
+        ensure_gs_register(num+1);
         dprintf(("acquired same segment: %d\n", num+1));
         goto got_num;
     }
@@ -234,7 +229,7 @@
             int old_num = tl->last_associated_segment_num;
             dprintf(("acquired different segment: %d->%d\n", old_num, num+1));
             tl->last_associated_segment_num = num+1;
-            set_gs_register(get_segment_base(num+1));
+            ensure_gs_register(num+1);
             dprintf(("                            %d->%d\n", old_num, num+1));
             (void)old_num;
             goto got_num;
@@ -313,14 +308,14 @@
 void _stm_test_switch(stm_thread_local_t *tl)
 {
     assert(_stm_in_transaction(tl));
-    set_gs_register(get_segment_base(tl->last_associated_segment_num));
+    ensure_gs_register(tl->last_associated_segment_num);
     assert(STM_SEGMENT->running_thread == tl);
     exec_local_finalizers();
 }
 
 void _stm_test_switch_segment(int segnum)
 {
-    set_gs_register(get_segment_base(segnum+1));
+    ensure_gs_register(segnum+1);
 }
 
 #if STM_TESTS

From noreply at buildbot.pypy.org  Wed Jul 29 09:16:34 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 29 Jul 2015 09:16:34 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: improve $psegment(None): the
 running_pthread info is not accurate enough
Message-ID: <20150729071634.767131C1158@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: use-gcc
Changeset: r1899:e2b46feac029
Date: 2015-07-29 09:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/e2b46feac029/

Log:	improve $psegment(None): the running_pthread info is not accurate
	enough

diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -77,34 +77,25 @@
         '((struct stm_priv_segment_info_s *)(stm_object_pages+%d))%s'
         % (get_segment_size() * segment_id + get_psegment_ofs(), field))
 
-def thread_to_segment_id(thread_id):
-    base = int_(gdb.parse_and_eval('stm_object_pages'))
+def current_segment():
+    mytl = int_(gdb.parse_and_eval('&stm_thread_local'))
     for j in range(1, get_nb_segments() + 1):
-        #ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
-        ti = get_psegment(j, '->running_pthread')
-        if int_(ti) == thread_id:
+        tl = get_psegment(j, '->pub.running_thread')
+        if int_(tl) == mytl:
             ts = get_psegment(j, '->transaction_state')
             if int_(ts) == 0:
                 print >> sys.stderr, "note: transaction_state == 0"
             return j
-    raise Exception("thread not found: %r" % (thread_id,))
+    raise Exception("no segment seems to be running this thread")
 
 def interactive_segment_base(thread=None):
     if thread is None:
-        s = gdb.execute('info threads', False, True)
-        i = s.find('\n* ')
-        assert i >= 0
-        fields = s[i+2:].split()
-        assert fields[1] == 'Thread'
-        assert fields[2].startswith('0x')
-        thread_id = int(fields[2], 16)
-        segment_id = thread_to_segment_id(thread_id)
+        segment_id = current_segment()
     elif thread.type.code == gdb.TYPE_CODE_INT:
         if 0 <= int_(thread) < 256:
             segment_id = int_(thread)
         else:
-            thread_id = int_(thread)
-            segment_id = thread_to_segment_id(thread_id)
+            raise TypeError("segment num not in range")
     else:
         raise TypeError("'thread' argument must be an int or not given")
     return get_segment_base(segment_id)

From noreply at buildbot.pypy.org  Wed Jul 29 09:31:10 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 29 Jul 2015 09:31:10 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: pblendw_xxi -> only tests byte for the i
	argument
Message-ID: <20150729073110.BE8D41C1158@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78700:1544d293e4b7
Date: 2015-07-29 08:56 +0200
http://bitbucket.org/pypy/pypy/changeset/1544d293e4b7/

Log:	pblendw_xxi -> only tests byte for the i argument last two tests in
	test_vectorize

diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -388,7 +388,8 @@
            instrname.find('INSERT') != -1 or \
            instrname.find('EXTRACT') != -1 or \
            instrname.find('SRLDQ') != -1 or \
-           instrname.find('SHUF') != -1:
+           instrname.find('SHUF') != -1 or \
+           instrname.find('PBLEND') != -1:
             realargmodes = []
             for mode in argmodes:
                 if mode == 'i':
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -734,13 +734,16 @@
     def repr_of_descr(self):
         return 'TargetToken(%d)' % compute_unique_id(self)
 
-def index_of_first(opnum, operations):
+def index_of_first(opnum, operations, pass_by=0):
     """ returns the position of the first operation matching the opnum.
     Or -1 if non is found
     """
     for i,op in enumerate(operations):
         if op.getopnum() == opnum:
-            return i
+            if pass_by == 0:
+                return i
+            else:
+                pass_by -= 1
     return -1
 
 
@@ -849,9 +852,9 @@
     def get_operations(self):
         return self.operations
 
-    def find_first_index(self, opnum):
+    def find_first_index(self, opnum, pass_by=0):
         """ return the first operation having the same opnum or -1 """
-        return index_of_first(opnum, self.operations)
+        return index_of_first(opnum, self.operations, pass_by)
 
     def snapshot(self):
         version = LoopVersion(self.copy_operations())
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -357,17 +357,13 @@
         i1 = int_add(i0,1)
         jump(p0,i1)
         """
-        vopt = self.vectoroptimizer_unrolled(self.parse_loop(ops),1)
-        self.assert_edges(vopt.dependency_graph,
-                [ [1,2,3,5], [5], [3,4], [5], [5], [] ], {})
-
+        loop = self.parse_loop(ops)
+        vopt = self.vectoroptimizer_unrolled(loop,1)
         vopt.find_adjacent_memory_refs()
-        self.assert_has_memory_ref_at(1)
-        self.assert_has_memory_ref_at(3)
         assert len(vopt.dependency_graph.memory_refs) == 2
 
-        mref1 = self.getmemref(1)
-        mref3 = self.getmemref(3)
+        mref1 = self.getmemref(loop.find_first_index(rop.RAW_LOAD))
+        mref3 = self.getmemref(loop.find_first_index(rop.RAW_LOAD,1))
         assert isinstance(mref1, MemoryRef)
         assert isinstance(mref3, MemoryRef)
 
@@ -493,37 +489,17 @@
         i6 = int_add(i4,1)
         jump(p0,i1,i6)
         """
-        ops2 = """
-        [p0,i0,i4]
-        i3 = raw_load(p0,i0,descr=chararraydescr)
-        i1 = int_add(i0,1)
-        i5 = raw_load(p0,i4,descr=chararraydescr)
-        i6 = int_add(i4,1)
-        i3 = raw_load(p0,i1,descr=chararraydescr)
-        i8 = int_add(i1,1)
-        i9 = raw_load(p0,i6,descr=chararraydescr)
-        i7 = int_add(i6,1)
-        jump(p0,i8,i7)
-        """
-
-        vopt = self.vectoroptimizer_unrolled(self.parse_loop(ops),1)
-        self.assert_edges(vopt.dependency_graph,
-                [ [1,2,3,4,5,7,9], 
-                    [9], [5,6], [9], [7,8],
-                    [9],  [9], [9], [9], 
-                  [],
-                ], {})
-
+        loop = self.parse_loop(ops)
+        vopt = self.vectoroptimizer_unrolled(loop,1)
         vopt.find_adjacent_memory_refs()
 
-        for i in [1,3,5,7]:
+        f = lambda x: loop.find_first_index(rop.RAW_LOAD, x)
+        indices = [f(0),f(1),f(2),f(3)]
+        for i in indices:
             self.assert_has_memory_ref_at(i)
         assert len(vopt.dependency_graph.memory_refs) == 4
 
-        mref1 = self.getmemref(1)
-        mref3 = self.getmemref(3)
-        mref5 = self.getmemref(5)
-        mref7 = self.getmemref(7)
+        mref1, mref3, mref5, mref7 = [self.getmemref(i) for i in indices]
         assert isinstance(mref1, MemoryRef)
         assert isinstance(mref3, MemoryRef)
         assert isinstance(mref5, MemoryRef)
@@ -1056,27 +1032,6 @@
         assert opt.loop.inputargs[2] in opt.packset.accum_vars
         self.debug_print_operations(opt.loop)
 
-    def test_accumulate_int16(self):
-        py.test.skip("only sum int64 on x64 is supported")
-        trace = """
-        [p3, i4, p1, i5, i6, i7, i8]
-        guard_early_exit() [p1, i4, i5, i6, p3]
-        i9 = raw_load(i7, i5, descr=int16arraydescr)
-        guard_not_invalidated() [p1, i9, i4, i5, i6, p3]
-        i10 = int_add(i6, i9)
-        i12 = int_add(i4, 1)
-        i14 = int_add(i5, 2)
-        i15 = int_ge(i12, i8)
-        guard_false(i15) [p1, i14, i10, i12, None, None, None, p3]
-        jump(p3, i12, p1, i14, i10, i7, i8)
-        """
-        opt = self.schedule(self.parse_loop(trace))
-        assert len(opt.packset.packs) == 2
-        assert len(opt.packset.accum_vars) == 1
-        assert opt.loop.inputargs[4] in opt.packset.accum_vars
-        self.debug_print_operations(opt.loop)
-
-
     def test_element_f45_in_guard_failargs(self):
         ops = """
         [p36, i28, p9, i37, p14, f34, p12, p38, f35, p39, i40, i41, p42, i43, i44, i21, i4, i0, i18]

From noreply at buildbot.pypy.org  Wed Jul 29 09:31:12 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 29 Jul 2015 09:31:12 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: moved vec_guard_false out of the
 assembler into the vector_ext file, 
Message-ID: <20150729073112.17E001C1158@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78701:9c7a794d8c78
Date: 2015-07-29 09:31 +0200
http://bitbucket.org/pypy/pypy/changeset/9c7a794d8c78/

Log:	moved vec_guard_false out of the assembler into the vector_ext file,
	adapted test_micronumpy to use int_is_ture instead of int_and(X,
	255) (which is wrong) refactored blend unused slots (out of
	_guard_true/false helper for vector arguments)

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -580,13 +580,13 @@
 
     def define_float_any():
         return """
-        a = [0,0,0,0,0,0,0,1,0,0,0]
+        a = [0,0,0,0,0,0,0,0.9,0,0,0]
         any(a)
         """
 
     def define_float32_any():
         return """
-        a = astype([0,0,0,0,0,0,0,1,0,0,0], float32)
+        a = astype([0,0,0,0,0,0,0,0.1,0,0,0], float32)
         any(a)
         """
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -113,7 +113,7 @@
             guard_true(i11, descr=...)
             guard_not_invalidated(descr=...)
             i12 = cast_float_to_int(f10)
-            i14 = int_and(i12, 255)
+            i14 = int_is_true(i12)
             guard_true(i14, descr=...)
             i15 = getfield_gc_pure(p1, descr=<FieldU pypy.module.micronumpy.boxes.W_BoolBox.inst_value \d+>)
             i16 = int_is_true(i15)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1741,24 +1741,6 @@
         self.mc.IMUL(arglocs[0], arglocs[1])
         return self._gen_guard_overflow(guard_op, guard_token)
 
-    def _guard_vector_false(self, guard_op, loc):
-        arg = guard_op.getarg(0)
-        assert isinstance(arg, BoxVector)
-        #
-        # if the vector is not fully packed blend 1s
-        if not arg.fully_packed(self.cpu.vector_register_size):
-            temp = X86_64_XMM_SCRATCH_REG
-            self.mc.PXOR(temp, temp)
-            select = 0
-            bits_used = (arg.item_count * arg.item_size * 8)
-            index = bits_used // 16
-            while index < 8:
-                select |= (1 << index)
-                index += 1
-            self.mc.PBLENDW_xxi(loc.value, temp.value, select)
-
-        self.mc.PTEST(loc, loc)
-
     def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
         if isinstance(loc, RegLoc):
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -34,6 +34,15 @@
 class VectorAssemblerMixin(object):
     _mixin_ = True
 
+    def _blend_unused_slots(self, loc, arg, temp):
+        select = 0
+        bits_used = (arg.item_count * arg.item_size * 8)
+        index = bits_used // 16
+        while index < 8:
+            select |= (1 << index)
+            index += 1
+        self.mc.PBLENDW_xxi(loc.value, temp.value, select)
+
     def _guard_vector_true(self, guard_op, loc, zero=False):
         arg = guard_op.getarg(0)
         assert isinstance(arg, BoxVector)
@@ -44,13 +53,7 @@
         # if the vector is not fully packed blend 1s
         if not arg.fully_packed(self.cpu.vector_register_size):
             self.mc.PCMPEQQ(temp, temp) # fill with ones
-            select = 0
-            bits_used = (arg.item_count * arg.item_size * 8)
-            index = bits_used // 16
-            while index < 8:
-                select |= (1 << index)
-                index += 1
-            self.mc.PBLENDW_xxi(loc.value, temp.value, select)
+            self._blend_unused_slots(loc, arg, temp)
             # reset to zeros
             self.mc.PXOR(temp, temp)
 
@@ -61,8 +64,17 @@
         # test if all slots are zero
         self.mc.PTEST(loc, temp)
 
-    # vector operations
-    # ________________________________________
+    def _guard_vector_false(self, guard_op, loc):
+        arg = guard_op.getarg(0)
+        assert isinstance(arg, BoxVector)
+        #
+        # if the vector is not fully packed blend 1s
+        if not arg.fully_packed(self.cpu.vector_register_size):
+            temp = X86_64_XMM_SCRATCH_REG
+            self.mc.PXOR(temp, temp)
+            self._blend_unused_slots(loc, arg, temp)
+        #
+        self.mc.PTEST(loc, loc)
 
     def _accum_update_at_exit(self, fail_locs, fail_args, faildescr, regalloc):
         """ If accumulation is done in this loop, at the guard exit
@@ -182,12 +194,12 @@
         self.mc.PCMPEQ(loc, temp, sizeloc.value)
 
     def genop_guard_vec_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
-        self._guard_vector_true(op, arglocs[0])
         guard_opnum = guard_op.getopnum()
         if guard_opnum == rop.GUARD_TRUE:
-            self.implement_guard(guard_token, 'NZ')
+            self._guard_vector_true(op, arglocs[0])
         else:
-            self.implement_guard(guard_token, 'Z')
+            self._guard_vector_false(op, arglocs[0])
+        self.implement_guard(guard_token, 'NZ')
 
     def genop_vec_int_mul(self, op, arglocs, resloc):
         loc0, loc1, itemsize_loc = arglocs

From noreply at buildbot.pypy.org  Wed Jul 29 11:38:11 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 29 Jul 2015 11:38:11 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: finally! make sure a thread has its
 previous transaction committed before starting another one
Message-ID: <20150729093811.576A21C1383@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: use-gcc
Changeset: r1900:36aff8ee0d87
Date: 2015-07-29 11:40 +0200
http://bitbucket.org/pypy/stmgc/changeset/36aff8ee0d87/

Log:	finally! make sure a thread has its previous transaction committed
	before starting another one

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -615,7 +615,8 @@
 
     new = _create_commit_log_entry();
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
-        assert(_stm_detached_inevitable_from_thread == 0);  /* running it */
+        assert(_stm_detached_inevitable_from_thread == 0  /* running it */
+               || _stm_detached_inevitable_from_thread == -1);  /* committing external */
 
         old = STM_PSEGMENT->last_commit_log_entry;
         new->rev_num = old->rev_num + 1;
@@ -1336,6 +1337,14 @@
     push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
 
+
+    /* before releasing _stm_detached_inevitable_from_thread, perform
+       the commit. Otherwise, the same thread whose (inev) transaction we try
+       to commit here may start a new one in another segment *but* w/o
+       the committed data from its previous inev transaction. */
+    bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
+    _validate_and_add_to_commit_log();
+
     if (external) {
         /* from this point on, unlink the original 'stm_thread_local_t *'
            from its segment.  Better do it as soon as possible, because
@@ -1345,13 +1354,8 @@
         write_fence();
         assert(_stm_detached_inevitable_from_thread == -1);
         _stm_detached_inevitable_from_thread = 0;
-    } else {
-        /* we certainly are not detached right now */
-        assert(_stm_detached_inevitable_from_thread != (intptr_t)STM_SEGMENT->running_thread);
     }
 
-    bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
-    _validate_and_add_to_commit_log();
 
     if (!was_inev) {
         assert(!external);
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -124,6 +124,7 @@
 
         assert(tl != old_tl);
 
+        // XXX: not sure if the next line is a good idea
         tl->last_associated_segment_num = remote_seg_num;
         ensure_gs_register(remote_seg_num);
         commit_external_inevitable_transaction();

From noreply at buildbot.pypy.org  Wed Jul 29 11:40:33 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 29 Jul 2015 11:40:33 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: import stmgc
Message-ID: <20150729094033.13AD21C056E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78702:5af34e2240be
Date: 2015-07-29 11:42 +0200
http://bitbucket.org/pypy/pypy/changeset/5af34e2240be/

Log:	import stmgc

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-088f807586c2
+36aff8ee0d87
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -615,7 +615,8 @@
 
     new = _create_commit_log_entry();
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
-        assert(_stm_detached_inevitable_from_thread == 0);  /* running it */
+        assert(_stm_detached_inevitable_from_thread == 0  /* running it */
+               || _stm_detached_inevitable_from_thread == -1);  /* committing external */
 
         old = STM_PSEGMENT->last_commit_log_entry;
         new->rev_num = old->rev_num + 1;
@@ -1336,6 +1337,14 @@
     push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
 
+
+    /* before releasing _stm_detached_inevitable_from_thread, perform
+       the commit. Otherwise, the same thread whose (inev) transaction we try
+       to commit here may start a new one in another segment *but* w/o
+       the committed data from its previous inev transaction. */
+    bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
+    _validate_and_add_to_commit_log();
+
     if (external) {
         /* from this point on, unlink the original 'stm_thread_local_t *'
            from its segment.  Better do it as soon as possible, because
@@ -1347,8 +1356,6 @@
         _stm_detached_inevitable_from_thread = 0;
     }
 
-    bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
-    _validate_and_add_to_commit_log();
 
     if (!was_inev) {
         assert(!external);
diff --git a/rpython/translator/stm/src_stm/stm/detach.c b/rpython/translator/stm/src_stm/stm/detach.c
--- a/rpython/translator/stm/src_stm/stm/detach.c
+++ b/rpython/translator/stm/src_stm/stm/detach.c
@@ -122,6 +122,9 @@
         dprintf(("reattach_transaction: commit detached from seg %d\n",
                  remote_seg_num));
 
+        assert(tl != old_tl);
+
+        // XXX: not sure if the next line is a good idea
         tl->last_associated_segment_num = remote_seg_num;
         ensure_gs_register(remote_seg_num);
         commit_external_inevitable_transaction();
@@ -135,6 +138,7 @@
 {
     dprintf(("> stm_force_transaction_break()\n"));
     assert(STM_SEGMENT->running_thread == tl);
+    assert(!stm_is_atomic(tl));
     _stm_commit_transaction();
     _stm_start_transaction(tl);
 }
@@ -180,14 +184,9 @@
     dprintf(("commit_fetched_detached_transaction from seg %d\n", segnum));
     assert(segnum > 0);
 
-    if (segnum != mysegnum) {
-        set_gs_register(get_segment_base(segnum));
-    }
+    ensure_gs_register(segnum);
     commit_external_inevitable_transaction();
-
-    if (segnum != mysegnum) {
-        set_gs_register(get_segment_base(mysegnum));
-    }
+    ensure_gs_register(mysegnum);
 }
 
 static void commit_detached_transaction_if_from(stm_thread_local_t *tl)
diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c
--- a/rpython/translator/stm/src_stm/stm/forksupport.c
+++ b/rpython/translator/stm/src_stm/stm/forksupport.c
@@ -87,7 +87,7 @@
     assert(tl->last_associated_segment_num == i);
     assert(in_transaction(tl));
     assert(pr->transaction_state != TS_INEVITABLE);
-    set_gs_register(get_segment_base(i));
+    ensure_gs_register(i);
     assert(STM_SEGMENT->segment_num == i);
 
     s_mutex_lock();
@@ -155,7 +155,7 @@
     int segnum = fork_this_tl->last_associated_segment_num;
     assert(1 <= segnum && segnum < NB_SEGMENTS);
     *_get_cpth(fork_this_tl) = pthread_self();
-    set_gs_register(get_segment_base(segnum));
+    ensure_gs_register(segnum);
     assert(STM_SEGMENT->segment_num == segnum);
 
     if (!fork_was_in_transaction) {
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -723,7 +723,7 @@
 
     /* including the sharing seg0 */
     for (i = 0; i < NB_SEGMENTS; i++) {
-        set_gs_register(get_segment_base(i));
+        ensure_gs_register(i);
 
         bool ok = _stm_validate();
         assert(get_priv_segment(i)->last_commit_log_entry->next == NULL
@@ -769,7 +769,7 @@
         }
     }
 
-    set_gs_register(get_segment_base(original_num));
+    ensure_gs_register(original_num);
 }
 
 
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -66,7 +66,6 @@
 
 static void ensure_gs_register(long segnum)
 {
-    /* XXX use this instead of set_gs_register() in many places */
     if (STM_SEGMENT->segment_num != segnum) {
         set_gs_register(get_segment_base(segnum));
         assert(STM_SEGMENT->segment_num == segnum);
@@ -211,16 +210,12 @@
     assert(_has_mutex());
     assert(_is_tl_registered(tl));
 
-    int num = tl->last_associated_segment_num - 1; // 0..NB_SEG-1
+    int num = tl->last_associated_segment_num - 1; // 0..NB_SEG-2
     OPT_ASSERT(num >= 0);
     if (sync_ctl.in_use1[num+1] == 0) {
         /* fast-path: we can get the same segment number than the one
-           we had before.  The value stored in GS is still valid. */
-#ifdef STM_TESTS
-        /* that can be optimized away, except during tests, because
-           they use only one thread */
-        set_gs_register(get_segment_base(num+1));
-#endif
+           we had before.  The value stored in GS may still be valid. */
+        ensure_gs_register(num+1);
         dprintf(("acquired same segment: %d\n", num+1));
         goto got_num;
     }
@@ -234,7 +229,7 @@
             int old_num = tl->last_associated_segment_num;
             dprintf(("acquired different segment: %d->%d\n", old_num, num+1));
             tl->last_associated_segment_num = num+1;
-            set_gs_register(get_segment_base(num+1));
+            ensure_gs_register(num+1);
             dprintf(("                            %d->%d\n", old_num, num+1));
             (void)old_num;
             goto got_num;
@@ -313,14 +308,14 @@
 void _stm_test_switch(stm_thread_local_t *tl)
 {
     assert(_stm_in_transaction(tl));
-    set_gs_register(get_segment_base(tl->last_associated_segment_num));
+    ensure_gs_register(tl->last_associated_segment_num);
     assert(STM_SEGMENT->running_thread == tl);
     exec_local_finalizers();
 }
 
 void _stm_test_switch_segment(int segnum)
 {
-    set_gs_register(get_segment_base(segnum+1));
+    ensure_gs_register(segnum+1);
 }
 
 #if STM_TESTS
diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -3,7 +3,7 @@
    of PyPy's #defines and #includes prepended. */
 
 __thread
-struct stm_thread_local_s stm_thread_local __attribute__((aligned(64)));
+struct stm_thread_local_s stm_thread_local __attribute__((aligned(64))) = {0};
 
 
 extern Signed pypy_stmcb_size_rounded_up(void*);

From noreply at buildbot.pypy.org  Wed Jul 29 11:54:58 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 29 Jul 2015 11:54:58 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: bool(0.1) return True,
 in the jit bool(0.1) transforms to cast_float_to_int which is wrong, 
Message-ID: <20150729095458.99CBC1C056E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78703:4fead7c7623f
Date: 2015-07-29 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/4fead7c7623f/

Log:	bool(0.1) return True, in the jit bool(0.1) transforms to
	cast_float_to_int which is wrong, updated the jtransform.py to emit
	float_is_true instead (similar to the issue with cast int -> bool
	added implementation for both float_ne, float_eq as vector
	operations

diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -272,6 +272,14 @@
         """)
         assert interp.results[0].value == 3
 
+    def test_any(self):
+        interp = self.run("""
+        a = [0,0,0,0,0.1,0,0,0,0]
+        b = any(a)
+        b -> 0
+        """)
+        assert interp.results[0].value == 1
+
     def test_where(self):
         interp = self.run('''
         a = [1, 0, 3, 0]
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -580,7 +580,7 @@
 
     def define_float_any():
         return """
-        a = [0,0,0,0,0,0,0,0.9,0,0,0]
+        a = [0,0,0,0,0,0,0,0.1,0,0,0]
         any(a)
         """
 
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -768,6 +768,8 @@
 
     PTEST_xx = xmminsn('\x66', rex_nw, '\x0F\x38\x17', register(1,8), register(2), '\xC0')
     PBLENDW_xxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x0E', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    CMPPD_xxi = xmminsn('\x66', rex_nw, '\x0F\xC2', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    CMPPS_xxi = xmminsn(        rex_nw, '\x0F\xC2', register(1,8), register(2), '\xC0', immediate(3, 'b'))
 
     # ------------------------------------------------------------
 
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -389,7 +389,8 @@
            instrname.find('EXTRACT') != -1 or \
            instrname.find('SRLDQ') != -1 or \
            instrname.find('SHUF') != -1 or \
-           instrname.find('PBLEND') != -1:
+           instrname.find('PBLEND') != -1 or \
+           instrname.find('CMPP') != -1:
             realargmodes = []
             for mode in argmodes:
                 if mode == 'i':
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -289,6 +289,43 @@
         elif size == 8:
             self.mc.XORPD(src, heap(self.float_const_neg_addr))
 
+    def genop_guard_vec_float_eq(self, op, guard_op, guard_token, arglocs, resloc):
+        lhsloc, rhsloc, sizeloc = arglocs
+        self.genop_vec_float_eq(op, arglocs, lhsloc) # yields one bits if they are equal
+        self.mc.PTEST(lhsloc, lhsloc)
+        guard_opnum = guard_op.getopnum()
+        if guard_opnum == rop.GUARD_TRUE:
+            self.implement_guard(guard_token, 'NZ')
+        else:
+            self.implement_guard(guard_token, 'Z')
+
+    def genop_vec_float_eq(self, op, arglocs, resloc):
+        _, rhsloc, sizeloc = arglocs
+        size = sizeloc.value
+        if size == 4:
+            self.mc.CMPPS_xxi(resloc.value, rhsloc.value, 0) # 0 means equal
+        else:
+            self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 0)
+
+    def genop_guard_vec_float_ne(self, op, guard_op, guard_token, arglocs, resloc):
+        lhsloc, rhsloc, sizeloc = arglocs
+        self.genop_vec_float_ne(op, arglocs, lhsloc) # yields one bits if they are equal
+        self.mc.PTEST(lhsloc, lhsloc)
+        guard_opnum = guard_op.getopnum()
+        if guard_opnum == rop.GUARD_TRUE:
+            self.implement_guard(guard_token, 'NZ')
+        else:
+            self.implement_guard(guard_token, 'Z')
+
+    def genop_vec_float_ne(self, op, arglocs, resloc):
+        _, rhsloc, sizeloc = arglocs
+        size = sizeloc.value
+        # b(100) == 1 << 2 means not equal
+        if size == 4:
+            self.mc.CMPPS_xxi(resloc.value, rhsloc.value, 1 << 2)
+        else:
+            self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 1 << 2)
+
     def genop_vec_int_signext(self, op, arglocs, resloc):
         srcloc, sizeloc, tosizeloc = arglocs
         size = sizeloc.value
@@ -556,7 +593,20 @@
         result = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
         self.perform(op, [source, imm(size)], result)
 
-    consider_vec_float_eq = consider_vec_logic
+    def consider_vec_float_eq(self, op, guard_op):
+        lhs = op.getarg(0)
+        assert isinstance(lhs, BoxVector)
+        size = lhs.item_size
+        args = op.getarglist()
+        lhsloc = self.xrm.force_result_in_reg(op.result, op.getarg(0), args)
+        rhsloc = self.make_sure_var_in_reg(op.getarg(1), args)
+        if guard_op:
+            self.perform_with_guard(op, guard_op, [lhsloc, rhsloc, imm(size)], None)
+        else:
+            self.perform(op, [lhsloc, rhsloc, imm(size)], lhsloc)
+
+    consider_vec_float_ne = consider_vec_float_eq
+
     consider_vec_int_and = consider_vec_logic
     consider_vec_int_or = consider_vec_logic
     consider_vec_int_xor = consider_vec_logic
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1178,7 +1178,12 @@
             else:
                 v1 = v_arg
             sizesign = rffi.size_and_sign(v_result.concretetype)
-            if sizesign <= rffi.size_and_sign(lltype.Signed):
+            if v_result.concretetype is lltype.Bool:
+                op = self.rewrite_operation(
+                        SpaceOperation('float_is_true', [v1], v_result)
+                )
+                ops.append(op)
+            elif sizesign <= rffi.size_and_sign(lltype.Signed):
                 # cast to a type that fits in an int: either the size is
                 # smaller, or it is equal and it is not unsigned
                 v2 = varoftype(lltype.Signed)
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -858,6 +858,12 @@
             cast_int_to_float %i0 -> %f0
             float_return %f0
         """, transform=True)
+        def f(n):
+            return rffi.cast(lltype.Bool, n)
+        self.encoding_test(f, [0.1], """
+            float_ne %f0, $0.0 -> %i0
+            int_return %i0
+        """, transform=True)
 
         # Casts to lltype.SingleFloat
         def g(n):
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -750,6 +750,7 @@
     rop.VEC_FLOAT_ABS:   FLOAT_SINGLE_ARG_OP_TO_VOP,
     rop.VEC_FLOAT_NEG:   FLOAT_SINGLE_ARG_OP_TO_VOP,
     rop.VEC_FLOAT_EQ:    OpToVectorOp((PT_FLOAT_GENERIC,PT_FLOAT_GENERIC), INT_RES),
+    rop.VEC_FLOAT_NE:    OpToVectorOp((PT_FLOAT_GENERIC,PT_FLOAT_GENERIC), INT_RES),
     rop.VEC_INT_IS_TRUE: OpToVectorOp((PT_INT_GENERIC,PT_INT_GENERIC), PT_INT_GENERIC),
 
     rop.VEC_RAW_LOAD:         LOAD_TRANS,
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -496,7 +496,8 @@
     'VEC_FLOAT_NEG/1',
     'VEC_FLOAT_ABS/1',
     '_VEC_ARITHMETIC_LAST',
-    'VEC_FLOAT_EQ/2',
+    'VEC_FLOAT_EQ/2b',
+    'VEC_FLOAT_NE/2b',
     'VEC_INT_IS_TRUE/1b',
 
     '_VEC_CAST_FIRST',
@@ -806,6 +807,7 @@
     rop.FLOAT_ABS: rop.VEC_FLOAT_ABS,
     rop.FLOAT_NEG: rop.VEC_FLOAT_NEG,
     rop.FLOAT_EQ:  rop.VEC_FLOAT_EQ,
+    rop.FLOAT_NE:  rop.VEC_FLOAT_NE,
     rop.INT_IS_TRUE: rop.VEC_INT_IS_TRUE,
 
     # casts

From noreply at buildbot.pypy.org  Wed Jul 29 12:11:59 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 29 Jul 2015 12:11:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: #2101 ported fix from vecopt to default
Message-ID: <20150729101159.063261C1158@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: 
Changeset: r78704:bdfcbfcfddb7
Date: 2015-07-29 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/bdfcbfcfddb7/

Log:	#2101 ported fix from vecopt to default

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1178,7 +1178,12 @@
             else:
                 v1 = v_arg
             sizesign = rffi.size_and_sign(v_result.concretetype)
-            if sizesign <= rffi.size_and_sign(lltype.Signed):
+            if v_result.concretetype is lltype.Bool:
+                op = self.rewrite_operation(
+                        SpaceOperation('float_is_true', [v1], v_result)
+                )
+                ops.append(op)
+            elif sizesign <= rffi.size_and_sign(lltype.Signed):
                 # cast to a type that fits in an int: either the size is
                 # smaller, or it is equal and it is not unsigned
                 v2 = varoftype(lltype.Signed)
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -868,6 +868,12 @@
             cast_int_to_float %i0 -> %f0
             float_return %f0
         """, transform=True)
+        def f(n):
+            return rffi.cast(lltype.Bool, n)
+        self.encoding_test(f, [0.1], """
+            float_ne %f0, $0.0 -> %i0
+            int_return %i0
+        """, transform=True)
 
         # Casts to lltype.SingleFloat
         def g(n):

From noreply at buildbot.pypy.org  Wed Jul 29 12:15:36 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 29 Jul 2015 12:15:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: added test to check if cast single
 float to bool is working properly aswell
Message-ID: <20150729101536.9AD231C1346@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: 
Changeset: r78705:d221a50d7898
Date: 2015-07-29 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/d221a50d7898/

Log:	added test to check if cast single float to bool is working properly
	aswell

diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -874,6 +874,11 @@
             float_ne %f0, $0.0 -> %i0
             int_return %i0
         """, transform=True)
+        self.encoding_test(f, [rffi.cast(lltype.SingleFloat, 0.5)], """
+            cast_singlefloat_to_float %i0 -> %f0
+            float_ne %f0, $0.0 -> %i1
+            int_return %i1
+        """, transform=True)
 
         # Casts to lltype.SingleFloat
         def g(n):

From noreply at buildbot.pypy.org  Wed Jul 29 13:00:55 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 29 Jul 2015 13:00:55 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: finished impl of float_ne with guards
Message-ID: <20150729110055.085E01C1382@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78706:924ff20716c0
Date: 2015-07-29 13:00 +0200
http://bitbucket.org/pypy/pypy/changeset/924ff20716c0/

Log:	finished impl of float_ne with guards

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -197,9 +197,10 @@
         guard_opnum = guard_op.getopnum()
         if guard_opnum == rop.GUARD_TRUE:
             self._guard_vector_true(op, arglocs[0])
+            self.implement_guard(guard_token, 'NZ')
         else:
             self._guard_vector_false(op, arglocs[0])
-        self.implement_guard(guard_token, 'NZ')
+            self.implement_guard(guard_token, 'NZ')
 
     def genop_vec_int_mul(self, op, arglocs, resloc):
         loc0, loc1, itemsize_loc = arglocs
@@ -289,16 +290,6 @@
         elif size == 8:
             self.mc.XORPD(src, heap(self.float_const_neg_addr))
 
-    def genop_guard_vec_float_eq(self, op, guard_op, guard_token, arglocs, resloc):
-        lhsloc, rhsloc, sizeloc = arglocs
-        self.genop_vec_float_eq(op, arglocs, lhsloc) # yields one bits if they are equal
-        self.mc.PTEST(lhsloc, lhsloc)
-        guard_opnum = guard_op.getopnum()
-        if guard_opnum == rop.GUARD_TRUE:
-            self.implement_guard(guard_token, 'NZ')
-        else:
-            self.implement_guard(guard_token, 'Z')
-
     def genop_vec_float_eq(self, op, arglocs, resloc):
         _, rhsloc, sizeloc = arglocs
         size = sizeloc.value
@@ -307,16 +298,6 @@
         else:
             self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 0)
 
-    def genop_guard_vec_float_ne(self, op, guard_op, guard_token, arglocs, resloc):
-        lhsloc, rhsloc, sizeloc = arglocs
-        self.genop_vec_float_ne(op, arglocs, lhsloc) # yields one bits if they are equal
-        self.mc.PTEST(lhsloc, lhsloc)
-        guard_opnum = guard_op.getopnum()
-        if guard_opnum == rop.GUARD_TRUE:
-            self.implement_guard(guard_token, 'NZ')
-        else:
-            self.implement_guard(guard_token, 'Z')
-
     def genop_vec_float_ne(self, op, arglocs, resloc):
         _, rhsloc, sizeloc = arglocs
         size = sizeloc.value
@@ -326,6 +307,28 @@
         else:
             self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 1 << 2)
 
+    def gen_float_cmp(func):
+        def genop_float_cmp(self, op, guard_op, guard_token, arglocs, resloc):
+            lhsloc, rhsloc, sizeloc = arglocs
+            size = sizeloc.value
+            func(self, op, arglocs, lhsloc) # yields one bits if they are equal
+            guard_opnum = guard_op.getopnum()
+            if guard_opnum == rop.GUARD_TRUE:
+                temp = X86_64_XMM_SCRATCH_REG
+                self.mc.PXOR(temp, temp) # set all to zero
+                self.mc.PCMPEQ(lhsloc, temp, size)
+                self.mc.PCMPEQQ(temp, temp) # set all bits to 1
+                self.mc.PTEST(lhsloc, temp)
+                self.implement_guard(guard_token, 'NZ')
+            else:
+                self.mc.PTEST(lhsloc, lhsloc)
+                self.implement_guard(guard_token, 'NZ')
+        return genop_float_cmp
+
+    genop_guard_vec_float_eq = gen_float_cmp(genop_vec_float_eq)
+    genop_guard_vec_float_ne = gen_float_cmp(genop_vec_float_ne)
+    del gen_float_cmp
+
     def genop_vec_int_signext(self, op, arglocs, resloc):
         srcloc, sizeloc, tosizeloc = arglocs
         size = sizeloc.value

From noreply at buildbot.pypy.org  Wed Jul 29 13:27:30 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 29 Jul 2015 13:27:30 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: another subtle race fixed,
 hopefully completing the previous commit
Message-ID: <20150729112730.EBAD41C0135@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: use-gcc
Changeset: r1901:cff6245b0bb4
Date: 2015-07-29 13:29 +0200
http://bitbucket.org/pypy/stmgc/changeset/cff6245b0bb4/

Log:	another subtle race fixed, hopefully completing the previous commit

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1341,7 +1341,9 @@
     /* before releasing _stm_detached_inevitable_from_thread, perform
        the commit. Otherwise, the same thread whose (inev) transaction we try
        to commit here may start a new one in another segment *but* w/o
-       the committed data from its previous inev transaction. */
+       the committed data from its previous inev transaction. (the
+       stm_validate() at the start of a new transaction is happy even
+       if there is an inevitable tx running) */
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
@@ -1636,6 +1638,16 @@
         if (!_validate_and_turn_inevitable())
             return;
     }
+
+    /* There may be a concurrent commit of a detached Tx going on.
+       Here, we may be right after the _validate_and_add_to_commit_log
+       and before resetting _stm_detached_inevitable_from_thread to
+       0. We have to wait for this to happen bc. otherwise, eg.
+       _stm_detach_inevitable_transaction is not safe to do yet */
+    while (_stm_detached_inevitable_from_thread == -1)
+        spin_loop();
+    assert(_stm_detached_inevitable_from_thread == 0);
+
     soon_finished_or_inevitable_thread_segment();
     STM_PSEGMENT->transaction_state = TS_INEVITABLE;
 

From noreply at buildbot.pypy.org  Wed Jul 29 13:28:30 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 29 Jul 2015 13:28:30 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: import stmgc
Message-ID: <20150729112830.C6AD91C0135@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78707:0e36d5505476
Date: 2015-07-29 13:30 +0200
http://bitbucket.org/pypy/pypy/changeset/0e36d5505476/

Log:	import stmgc

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-36aff8ee0d87
+cff6245b0bb4
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -1341,7 +1341,9 @@
     /* before releasing _stm_detached_inevitable_from_thread, perform
        the commit. Otherwise, the same thread whose (inev) transaction we try
        to commit here may start a new one in another segment *but* w/o
-       the committed data from its previous inev transaction. */
+       the committed data from its previous inev transaction. (the
+       stm_validate() at the start of a new transaction is happy even
+       if there is an inevitable tx running) */
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
@@ -1636,6 +1638,16 @@
         if (!_validate_and_turn_inevitable())
             return;
     }
+
+    /* There may be a concurrent commit of a detached Tx going on.
+       Here, we may be right after the _validate_and_add_to_commit_log
+       and before resetting _stm_detached_inevitable_from_thread to
+       0. We have to wait for this to happen bc. otherwise, eg.
+       _stm_detach_inevitable_transaction is not safe to do yet */
+    while (_stm_detached_inevitable_from_thread == -1)
+        spin_loop();
+    assert(_stm_detached_inevitable_from_thread == 0);
+
     soon_finished_or_inevitable_thread_segment();
     STM_PSEGMENT->transaction_state = TS_INEVITABLE;
 

From noreply at buildbot.pypy.org  Wed Jul 29 14:58:24 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 29 Jul 2015 14:58:24 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: add missing
	acquire_modification_lock_wr
Message-ID: <20150729125824.C68031C056E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: use-gcc
Changeset: r1902:6666c6fd1aad
Date: 2015-07-29 15:00 +0200
http://bitbucket.org/pypy/stmgc/changeset/6666c6fd1aad/

Log:	add missing acquire_modification_lock_wr

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -119,6 +119,7 @@
     dprintf(("copy_bk_objs_in_page_from(%d, %ld, %d)\n",
              from_segnum, (long)pagenum, only_if_not_modified));
 
+    assert(modification_lock_check_rdlock(from_segnum));
     struct list_s *list = get_priv_segment(from_segnum)->modified_old_objects;
     struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
     struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c
--- a/c8/stm/hashtable.c
+++ b/c8/stm/hashtable.c
@@ -379,11 +379,13 @@
                    will make the other transaction check that it didn't
                    do any stm_hashtable_list() on the complete hashtable.
             */
+            acquire_modification_lock_wr(STM_SEGMENT->segment_num);
             STM_PSEGMENT->modified_old_objects = list_append3(
                 STM_PSEGMENT->modified_old_objects,
                 TYPE_POSITION_MARKER,      /* type1 */
                 TYPE_MODIFIED_HASHTABLE,   /* type2 */
                 (uintptr_t)hobj);          /* modif_hashtable */
+            release_modification_lock_wr(STM_SEGMENT->segment_num);
         }
     }
     entry->object = nvalue;
diff --git a/c8/stm/marker.c b/c8/stm/marker.c
--- a/c8/stm/marker.c
+++ b/c8/stm/marker.c
@@ -73,12 +73,14 @@
     /* -2 is not odd */
     assert(marker.odd_number != (uintptr_t)TYPE_MODIFIED_HASHTABLE);
 
+    acquire_modification_lock_wr(STM_SEGMENT->segment_num);
     STM_PSEGMENT->position_markers_last = list_count(list);
     STM_PSEGMENT->modified_old_objects = list_append3(
         list,
         TYPE_POSITION_MARKER,         /* type */
         marker.odd_number,            /* marker_odd_number */
         (uintptr_t)marker.object);    /* marker_object */
+    release_modification_lock_wr(STM_SEGMENT->segment_num);
 }
 
 static void timing_write_read_contention(struct stm_undo_s *start,

From noreply at buildbot.pypy.org  Wed Jul 29 15:16:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 29 Jul 2015 15:16:15 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: If unicodedata_handler happens to be
 None here, don't interpret \N
Message-ID: <20150729131615.F27471C0135@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-strbuf
Changeset: r78708:522b1566ead2
Date: 2015-07-26 19:42 +0200
http://bitbucket.org/pypy/pypy/changeset/522b1566ead2/

Log:	If unicodedata_handler happens to be None here, don't interpret \N
	sequences at all

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1275,16 +1275,9 @@
                             "unicodeescape", errorhandler, message, errors)
 
         # \N{name}
-        elif ch == 'N':
+        elif ch == 'N' and unicodedata_handler is not None:
             message = "malformed \\N character escape"
             look = pos
-            if unicodedata_handler is None:
-                message = ("\\N escapes not supported "
-                           "(can't load unicodedata module)")
-                res, pos = errorhandler(errors, "unicodeescape",
-                                        message, s, pos-1, size)
-                builder.append(res)
-                continue
 
             if look < size and s[look] == '{':
                 # look for the closing brace

From noreply at buildbot.pypy.org  Wed Jul 29 15:16:17 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 29 Jul 2015 15:16:17 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: (fijal, arigo)
Message-ID: <20150729131617.36DE01C0135@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-strbuf
Changeset: r78709:735443d28cec
Date: 2015-07-26 19:47 +0200
http://bitbucket.org/pypy/pypy/changeset/735443d28cec/

Log:	(fijal, arigo)

	Adding the W_UnicodeBufferObject

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -47,6 +47,9 @@
             value = value[:]
         return W_BytearrayObject(value)
 
+    def _new_concat(self, space, value1, value2):
+        return self._new(value1 + value2)
+
     def _new_from_buffer(self, buffer):
         return W_BytearrayObject([buffer[i] for i in range(len(buffer))])
 
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -482,6 +482,13 @@
         """
         raise NotImplementedError
 
+    def buffer_w(self, space, flags):
+        space.check_buf_flags(flags, True)
+        return StringBuffer(self.str_w(space))
+
+    def readbuf_w(self, space):
+        return StringBuffer(self.str_w(space))
+
     def writebuf_w(self, space):
         raise OperationError(space.w_TypeError, space.wrap(
             "Cannot use string as modifiable buffer"))
@@ -499,12 +506,12 @@
 
     def descr_formatter_parser(self, space):
         from pypy.objspace.std.newformat import str_template_formatter
-        tformat = str_template_formatter(space, space.str_w(self))
+        tformat = str_template_formatter(space, self.str_w(space))
         return tformat.formatter_parser()
 
     def descr_formatter_field_name_split(self, space):
         from pypy.objspace.std.newformat import str_template_formatter
-        tformat = str_template_formatter(space, space.str_w(self))
+        tformat = str_template_formatter(space, self.str_w(space))
         return tformat.formatter_field_name_split()
 
 
@@ -526,19 +533,21 @@
     def str_w(self, space):
         return self._value
 
-    def buffer_w(self, space, flags):
-        space.check_buf_flags(flags, True)
-        return StringBuffer(self._value)
-
-    def readbuf_w(self, space):
-        return StringBuffer(self._value)
-
     def listview_bytes(self):
         return _create_list_from_bytes(self._value)
 
     def _new(self, value):
         return W_BytesObject(value)
 
+    def _new_concat(self, space, value1, value2):
+        if space.config.objspace.std.withstrbuf:
+            from pypy.objspace.std.strbufobject import W_StringBufferObject
+            builder = StringBuilder(len(value1) + len(value2))
+            builder.append(value1)
+            builder.append(value2)
+            return W_StringBufferObject(builder)
+        return self._new(value1 + value2)
+
     def _new_from_list(self, value):
         return W_BytesObject(''.join(value))
 
@@ -726,18 +735,6 @@
             from .bytearrayobject import W_BytearrayObject, _make_data
             self_as_bytearray = W_BytearrayObject(_make_data(self._value))
             return space.add(self_as_bytearray, w_other)
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            try:
-                other = self._op_val(space, w_other)
-            except OperationError as e:
-                if e.match(space, space.w_TypeError):
-                    return space.w_NotImplemented
-                raise
-            builder = StringBuilder()
-            builder.append(self._value)
-            builder.append(other)
-            return W_StringBufferObject(builder)
         return self._StringMethods_descr_add(space, w_other)
 
     _StringMethods__startswith = _startswith
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -31,7 +31,7 @@
 from pypy.objspace.std.sliceobject import W_SliceObject
 from pypy.objspace.std.tupleobject import W_AbstractTupleObject, W_TupleObject
 from pypy.objspace.std.typeobject import W_TypeObject, TypeCache
-from pypy.objspace.std.unicodeobject import W_UnicodeObject, wrapunicode
+from pypy.objspace.std.unicodeobject import W_AbstractUnicodeObject, W_UnicodeObject, wrapunicode
 
 
 class StdObjSpace(ObjSpace):
@@ -82,6 +82,8 @@
         }
         if self.config.objspace.std.withstrbuf:
             builtin_type_classes[W_BytesObject.typedef] = W_AbstractBytesObject
+            builtin_type_classes[W_UnicodeObject.typedef] = (
+                W_AbstractUnicodeObject)
 
         self.builtin_types = {}
         self._interplevel_classes = {}
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -1,10 +1,5 @@
-import inspect
-
-import py
-
-from pypy.objspace.std.bytesobject import (W_AbstractBytesObject,
-    W_BytesObject, StringBuffer)
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.objspace.std.bytesobject import W_AbstractBytesObject
+from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.interpreter.error import OperationError
 from rpython.rlib.rstring import StringBuilder
 
@@ -37,29 +32,22 @@
     def str_w(self, space):
         return self.force()
 
-    def buffer_w(self, space, flags):
-        return StringBuffer(self.force())
-
-    def readbuf_w(self, space):
-        return StringBuffer(self.force())
-
     def descr_len(self, space):
         return space.wrap(self.length)
 
     def descr_add(self, space, w_other):
-        try:
-            other = W_BytesObject._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        if self.builder.getlength() != self.length:
-            builder = StringBuilder()
-            builder.append(self.force())
+        if isinstance(w_other, W_AbstractBytesObject):
+            other = w_other.str_w(space)
+            if self.builder.getlength() != self.length:
+                builder = StringBuilder()
+                builder.append(self.force())
+            else:
+                builder = self.builder
+            builder.append(other)
+            return W_StringBufferObject(builder)
         else:
-            builder = self.builder
-        builder.append(other)
-        return W_StringBufferObject(builder)
+            self.force()
+            return self.w_str.descr_add(space, w_other)
 
     def descr_str(self, space):
         # you cannot get subclasses of W_StringBufferObject here
@@ -67,32 +55,42 @@
         return self
 
 
-for key, value in W_BytesObject.typedef.rawdict.iteritems():
-    if not isinstance(value, interp2app):
-        continue
-    if key in ('__len__', '__add__', '__str__'):
-        continue
+def copy_from_base_class(baseclass, bufclass, attr_name):
+    import inspect
+    import py
+    from pypy.interpreter.gateway import interp2app, unwrap_spec
 
-    func = value._code._bltin
-    args = inspect.getargs(func.func_code)
-    if args.varargs or args.keywords:
-        raise TypeError("Varargs and keywords not supported in unwrap_spec")
-    argspec = ', '.join([arg for arg in args.args[1:]])
-    func_code = py.code.Source("""
-    def f(self, %(args)s):
-        self.force()
-        return self.w_str.%(func_name)s(%(args)s)
-    """ % {'args': argspec, 'func_name': func.func_name})
-    d = {}
-    exec func_code.compile() in d
-    f = d['f']
-    f.func_defaults = func.func_defaults
-    f.__module__ = func.__module__
-    # necessary for unique identifiers for pickling
-    f.func_name = func.func_name
-    unwrap_spec_ = getattr(func, 'unwrap_spec', None)
-    if unwrap_spec_ is not None:
-        f = unwrap_spec(**unwrap_spec_)(f)
-    setattr(W_StringBufferObject, func.func_name, f)
+    for key, value in baseclass.typedef.rawdict.iteritems():
+        if not isinstance(value, interp2app):
+            continue
 
-W_StringBufferObject.typedef = W_BytesObject.typedef
+        func = value._code._bltin
+        if func.func_name in bufclass.__dict__:
+            assert key in ('__len__', '__add__', '__str__', '__unicode__')
+            continue
+
+        args = inspect.getargs(func.func_code)
+        if args.varargs or args.keywords:
+            raise TypeError("Varargs and keywords not supported in unwrap_spec")
+        argspec = ', '.join([arg for arg in args.args[1:]])
+        func_code = py.code.Source("""
+        def f(self, %(args)s):
+            self.force()
+            return self.%(attr_name)s.%(func_name)s(%(args)s)
+        """ % {'args': argspec, 'func_name': func.func_name,
+               'attr_name': attr_name})
+        d = {}
+        exec func_code.compile() in d
+        f = d['f']
+        f.func_defaults = func.func_defaults
+        f.__module__ = func.__module__
+        # necessary for unique identifiers for pickling
+        f.func_name = func.func_name
+        unwrap_spec_ = getattr(func, 'unwrap_spec', None)
+        if unwrap_spec_ is not None:
+            f = unwrap_spec(**unwrap_spec_)(f)
+        setattr(bufclass, func.func_name, f)
+
+    bufclass.typedef = baseclass.typedef
+
+copy_from_base_class(W_BytesObject, W_StringBufferObject, 'w_str')
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -62,7 +62,7 @@
                 if e.match(space, space.w_TypeError):
                     return space.w_NotImplemented
                 raise
-            return self._new(self._val(space) + other)
+            return self._new_concat(space, self._val(space), other)
 
         # Bytearray overrides this method, CPython doesn't support contacting
         # buffers and strs, and unicodes are always handled above
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ b/pypy/objspace/std/test/test_strbufobject.py
@@ -84,6 +84,13 @@
         a += 'b'
         raises(TypeError, "a += 5")
 
+    def test_add_unicode(self):
+        a = 'a'
+        a += 'b'
+        a += u'\u1234'
+        assert a == u'ab\u1234'
+        assert isinstance(a, unicode)
+
     def test_mix_strings_format(self):
         a = 'a'
         a += 'b'
@@ -99,3 +106,9 @@
         a = 'abc'
         a += 'bc'
         assert list(a._formatter_parser()) == [('abcbc', None, None, None)]
+
+    def test_startswith_u(self):
+        a = 'abc'
+        a += 'bc'
+        assert a.startswith(u'abcb')
+        assert not a.startswith(u'\u1234')
diff --git a/pypy/objspace/std/test/test_unibufobject.py b/pypy/objspace/std/test/test_unibufobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_unibufobject.py
@@ -0,0 +1,110 @@
+import py
+
+from pypy.objspace.std.test import test_unicodeobject
+
+class AppTestUnicodeObject(test_unicodeobject.AppTestUnicodeString):
+    spaceconfig = test_unicodeobject.AppTestUnicodeString.spaceconfig.copy()
+    spaceconfig.update({"objspace.std.withstrbuf": True})
+
+    def test_basic(self):
+        import __pypy__
+        # cannot do "Hello, " + "World!" because cpy2.5 optimises this
+        # away on AST level
+        s = u"Hello, ".__add__(u"World!")
+        assert type(s) is unicode
+        assert 'W_UnicodeBufferObject' in __pypy__.internal_repr(s)
+
+    def test_add_twice(self):
+        x = u"a".__add__(u"b")
+        y = x + u"c"
+        c = x + u"d"
+        assert y == u"abc"
+        assert c == u"abd"
+
+    def test_add(self):
+        import __pypy__
+        all = ""
+        for i in range(20):
+            all += unicode(i)
+        assert 'W_UnicodeBufferObject' in __pypy__.internal_repr(all)
+        assert all == u"012345678910111213141516171819"
+
+    def test_hash(self):
+        import __pypy__
+        def join(s): return s[:len(s) // 2] + s[len(s) // 2:]
+        t = u'a' * 101
+        s = join(t)
+        assert 'W_UnicodeBufferObject' in __pypy__.internal_repr(s)
+        assert hash(s) == hash(t)
+
+    def test_len(self):
+        s = u"a".__add__(u"b")
+        r = u"c".__add__(u"d")
+        t = s + r
+        assert len(s) == 2
+        assert len(r) == 2
+        assert len(t) == 4
+
+    def test_add_strbuf(self):
+        # make three strbuf objects
+        s = u'a'.__add__(u'b')
+        t = u'x'.__add__(u'c')
+        u = u'y'.__add__(u'd')
+
+        # add two different strbufs to the same string
+        v = s + t
+        w = s + u
+
+        # check that insanity hasn't resulted.
+        assert v == u"abxc"
+        assert w == u"abyd"
+
+    def test_more_adding_fun(self):
+        s = u'a'.__add__(u'b') # s is a strbuf now
+        t = s + u'c'
+        u = s + u'd'
+        v = s + u'e'
+        assert v == u'abe'
+        assert u == u'abd'
+        assert t == u'abc'
+
+    def test_buh_even_more(self):
+        a = u'a'.__add__(u'b')
+        b = a + u'c'
+        c = u'0'.__add__(u'1')
+        x = c + a
+        assert x == u'01ab'
+
+    def test_add_non_string(self):
+        a = u'a'
+        a += u'b'
+        raises(TypeError, "a += 5")
+
+    def test_add_plain_string(self):
+        a = u'a'
+        a += u'\u1234'
+        a += 'b'
+        assert a == u'a\u1234b'
+        assert isinstance(a, unicode)
+
+    def test_mix_strings_format(self):
+        a = u'a'
+        a += u'b'
+        assert u'foo%s' % a == u'fooab'
+        assert (a + u'%s') % (u'foo',) == u'abfoo'
+
+    def test_print(self):
+        a = u'abc'
+        a += u'bc'
+        print a
+
+    def test_formatter_parser(self):
+        a = u'abc'
+        a += u'bc'
+        assert list(a._formatter_parser()) == [(u'abcbc', None, None, None)]
+
+    def test_startswith_s(self):
+        a = u'abc'
+        a += u'bc'
+        assert a.startswith('abcb')
+        assert not a.startswith('1234')
diff --git a/pypy/objspace/std/unibufobject.py b/pypy/objspace/std/unibufobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/unibufobject.py
@@ -0,0 +1,67 @@
+from pypy.objspace.std.unicodeobject import W_AbstractUnicodeObject
+from pypy.objspace.std.unicodeobject import W_UnicodeObject, unicode_from_string
+from pypy.objspace.std.strbufobject import copy_from_base_class
+from pypy.interpreter.error import OperationError
+from rpython.rlib.rstring import UnicodeBuilder
+
+
+class W_UnicodeBufferObject(W_AbstractUnicodeObject):
+    w_unicode = None
+
+    def __init__(self, builder):
+        self.builder = builder             # UnicodeBuilder
+        self.length = builder.getlength()
+
+    def force(self):
+        if self.w_unicode is None:
+            s = self.builder.build()
+            if self.length < len(s):
+                s = s[:self.length]
+            self.w_unicode = W_UnicodeObject(s)
+            return s
+        else:
+            return self.w_unicode._value
+
+    def __repr__(w_self):
+        """ representation for debugging purposes """
+        return "%s(%r[:%d])" % (
+            w_self.__class__.__name__, w_self.builder, w_self.length)
+
+    def unwrap(self, space):
+        return self.force()
+
+    def unicode_w(self, space):
+        return self.force()
+
+    def descr_len(self, space):
+        return space.wrap(self.length)
+
+    def _new_concat_buffer(self, other):
+        if self.builder.getlength() != self.length:
+            builder = UnicodeBuilder()
+            builder.append(self.force())
+        else:
+            builder = self.builder
+        builder.append(other)
+        return W_UnicodeBufferObject(builder)
+
+    def descr_add(self, space, w_other):
+        from pypy.objspace.std.bytesobject import W_AbstractBytesObject
+
+        if isinstance(w_other, W_AbstractUnicodeObject):
+            other = w_other.unicode_w(space)
+            return self._new_concat_buffer(other)
+        elif isinstance(w_other, W_AbstractBytesObject):
+            other = unicode_from_string(space, w_other)._value
+            return self._new_concat_buffer(other)
+        else:
+            self.force()
+            return self.w_unicode.descr_add(space, w_other)
+
+    def descr_unicode(self, space):
+        # you cannot get subclasses of W_UnicodeBufferObject here
+        assert type(self) is W_UnicodeBufferObject
+        return self
+
+
+copy_from_base_class(W_UnicodeObject, W_UnicodeBufferObject, 'w_unicode')
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -495,14 +495,22 @@
         of the specified width. The string S is never truncated.
         """
 
+    def readbuf_w(self, space):
+        from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
+        value = self.unicode_w(space)
+        builder = StringBuilder(len(value) * UNICODE_SIZE)
+        for unich in value:
+            pack_unichar(unich, builder)
+        return StringBuffer(builder.build())
+
     def descr_formatter_parser(self, space):
         from pypy.objspace.std.newformat import unicode_template_formatter
-        tformat = unicode_template_formatter(space, space.unicode_w(self))
+        tformat = unicode_template_formatter(space, self.unicode_w(space))
         return tformat.formatter_parser()
 
     def descr_formatter_field_name_split(self, space):
         from pypy.objspace.std.newformat import unicode_template_formatter
-        tformat = unicode_template_formatter(space, space.unicode_w(self))
+        tformat = unicode_template_formatter(space, self.unicode_w(space))
         return tformat.formatter_field_name_split()
 
 
@@ -530,13 +538,6 @@
     def unicode_w(self, space):
         return self._value
 
-    def readbuf_w(self, space):
-        from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
-        builder = StringBuilder(len(self._value) * UNICODE_SIZE)
-        for unich in self._value:
-            pack_unichar(unich, builder)
-        return StringBuffer(builder.build())
-
     def writebuf_w(self, space):
         raise OperationError(space.w_TypeError, space.wrap(
             "cannot use unicode as modifiable buffer"))
@@ -554,6 +555,15 @@
     def _new(self, value):
         return W_UnicodeObject(value)
 
+    def _new_concat(self, space, value1, value2):
+        if space.config.objspace.std.withstrbuf:
+            from pypy.objspace.std.unibufobject import W_UnicodeBufferObject
+            builder = UnicodeBuilder(len(value1) + len(value2))
+            builder.append(value1)
+            builder.append(value2)
+            return W_UnicodeBufferObject(builder)
+        return self._new(value1 + value2)
+
     def _new_from_list(self, value):
         return W_UnicodeObject(u''.join(value))
 
@@ -573,9 +583,11 @@
 
     @staticmethod
     def _op_val(space, w_other):
-        if isinstance(w_other, W_UnicodeObject):
-            return w_other._value
-        if space.isinstance_w(w_other, space.w_str):
+        from pypy.objspace.std.bytesobject import W_AbstractBytesObject
+
+        if isinstance(w_other, W_AbstractUnicodeObject):
+            return w_other.unicode_w(space)
+        if isinstance(w_other, W_AbstractBytesObject):
             return unicode_from_string(space, w_other)._value
         return unicode_from_encoded_object(
             space, w_other, None, "strict")._value
@@ -664,9 +676,9 @@
             if space.is_w(w_unicodetype, space.w_unicode):
                 return w_value
 
-        assert isinstance(w_value, W_UnicodeObject)
+        value = w_value.unicode_w(space)
         w_newobj = space.allocate_instance(W_UnicodeObject, w_unicodetype)
-        W_UnicodeObject.__init__(w_newobj, w_value._value)
+        W_UnicodeObject.__init__(w_newobj, value)
         return w_newobj
 
     def descr_repr(self, space):
@@ -1035,7 +1047,7 @@
 
     __add__ = interpindirect2app(W_AbstractUnicodeObject.descr_add),
     __mul__ = interpindirect2app(W_AbstractUnicodeObject.descr_mul),
-    __rmul__ = interpindirect2app(W_AbstractUnicodeObject.descr_mul),
+    __rmul__ = interpindirect2app(W_AbstractUnicodeObject.descr_rmul),
 
     __getitem__ = interpindirect2app(W_AbstractUnicodeObject.descr_getitem),
     __getslice__ = interpindirect2app(W_AbstractUnicodeObject.descr_getslice),

From noreply at buildbot.pypy.org  Wed Jul 29 15:16:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 29 Jul 2015 15:16:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2097: corner case in _sqlite3
Message-ID: <20150729131618.678BF1C0135@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78710:3b425a82424c
Date: 2015-07-29 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/3b425a82424c/

Log:	Issue #2097: corner case in _sqlite3

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -345,7 +345,10 @@
 
     def _finalize_raw_statement(self, _statement):
         if self.__rawstatements is not None:
-            self.__rawstatements.remove(_statement)
+            try:
+                self.__rawstatements.remove(_statement)
+            except KeyError:
+                return    # rare case: already finalized, see issue #2097
             _lib.sqlite3_finalize(_statement)
 
     def __do_all_statements(self, action, reset_cursors):

From noreply at buildbot.pypy.org  Wed Jul 29 15:23:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 29 Jul 2015 15:23:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: If unicodedata_handler happens to be
 None here, don't interpret \N
Message-ID: <20150729132318.DA1561C056E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78711:81b300c1d878
Date: 2015-07-26 19:42 +0200
http://bitbucket.org/pypy/pypy/changeset/81b300c1d878/

Log:	If unicodedata_handler happens to be None here, don't interpret \N
	sequences at all (grafted from
	522b1566ead2b0fbdbe6952232268a07ad39eb3b)

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1275,16 +1275,9 @@
                             "unicodeescape", errorhandler, message, errors)
 
         # \N{name}
-        elif ch == 'N':
+        elif ch == 'N' and unicodedata_handler is not None:
             message = "malformed \\N character escape"
             look = pos
-            if unicodedata_handler is None:
-                message = ("\\N escapes not supported "
-                           "(can't load unicodedata module)")
-                res, pos = errorhandler(errors, "unicodeescape",
-                                        message, s, pos-1, size)
-                builder.append(res)
-                continue
 
             if look < size and s[look] == '{':
                 # look for the closing brace

From noreply at buildbot.pypy.org  Wed Jul 29 17:20:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 29 Jul 2015 17:20:56 +0200 (CEST)
Subject: [pypy-commit] pypy fix-strbuf: Fixes
Message-ID: <20150729152056.DEB241C1346@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-strbuf
Changeset: r78712:8736f18f1529
Date: 2015-07-29 17:21 +0200
http://bitbucket.org/pypy/pypy/changeset/8736f18f1529/

Log:	Fixes

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -196,7 +196,6 @@
 def PyUnicode_GET_SIZE(space, w_obj):
     """Return the size of the object.  o has to be a PyUnicodeObject (not
     checked)."""
-    assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
     return space.len_w(w_obj)
 
 @cpython_api([PyObject], rffi.CWCHARP, error=CANNOT_FAIL)
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -561,9 +561,10 @@
 
     @staticmethod
     def _use_rstr_ops(space, w_other):
-        from pypy.objspace.std.unicodeobject import W_UnicodeObject
+        from pypy.objspace.std.unicodeobject import W_AbstractUnicodeObject
+        W_AbstractUnicodeObject
         return (isinstance(w_other, W_AbstractBytesObject) or
-                isinstance(w_other, W_UnicodeObject))
+                isinstance(w_other, W_AbstractUnicodeObject))
 
     @staticmethod
     def _op_val(space, w_other):
@@ -760,12 +761,12 @@
     _StringMethods_descr_contains = descr_contains
     def descr_contains(self, space, w_sub):
         if space.isinstance_w(w_sub, space.w_unicode):
-            from pypy.objspace.std.unicodeobject import W_UnicodeObject
-            assert isinstance(w_sub, W_UnicodeObject)
+            from pypy.objspace.std.unicodeobject import W_AbstractUnicodeObject
+            assert isinstance(w_sub, W_AbstractUnicodeObject)
             self_as_unicode = unicode_from_encoded_object(space, self, None,
                                                           None)
             return space.newbool(
-                self_as_unicode._value.find(w_sub._value) >= 0)
+                self_as_unicode._value.find(w_sub.unicode_w(space)) >= 0)
         return self._StringMethods_descr_contains(space, w_sub)
 
     _StringMethods_descr_replace = descr_replace
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -8,7 +8,7 @@
 from pypy.interpreter.pycode import PyCode
 from pypy.interpreter import unicodehelper
 from pypy.objspace.std.boolobject import W_BoolObject
-from pypy.objspace.std.bytesobject import W_BytesObject
+from pypy.objspace.std.bytesobject import W_AbstractBytesObject
 from pypy.objspace.std.complexobject import W_ComplexObject
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 from pypy.objspace.std.intobject import W_IntObject
@@ -19,7 +19,7 @@
 from pypy.objspace.std.setobject import W_FrozensetObject, W_SetObject
 from pypy.objspace.std.tupleobject import W_AbstractTupleObject
 from pypy.objspace.std.typeobject import W_TypeObject
-from pypy.objspace.std.unicodeobject import W_UnicodeObject
+from pypy.objspace.std.unicodeobject import W_AbstractUnicodeObject
 
 
 TYPE_NULL      = '0'
@@ -244,7 +244,7 @@
     return space.newcomplex(real, imag)
 
 
- at marshaller(W_BytesObject)
+ at marshaller(W_AbstractBytesObject)
 def marshal_bytes(space, w_str, m):
     s = space.str_w(w_str)
     if m.version >= 1 and space.is_interned_str(s):
@@ -394,7 +394,7 @@
                   name, firstlineno, lnotab, freevars, cellvars)
 
 
- at marshaller(W_UnicodeObject)
+ at marshaller(W_AbstractUnicodeObject)
 def marshal_unicode(space, w_unicode, m):
     s = unicodehelper.encode_utf8(space, space.unicode_w(w_unicode))
     m.atom_str(TYPE_UNICODE, s)
diff --git a/pypy/objspace/std/test/test_unibufobject.py b/pypy/objspace/std/test/test_unibufobject.py
--- a/pypy/objspace/std/test/test_unibufobject.py
+++ b/pypy/objspace/std/test/test_unibufobject.py
@@ -108,3 +108,15 @@
         a += u'bc'
         assert a.startswith('abcb')
         assert not a.startswith('1234')
+
+    def test_str_contains_u(self):
+        a = u'abc'
+        a += u'bc'
+        assert a in '--abcbc--'
+        assert a not in '--abcBc--'
+
+    def test_int_unicode(self):
+        a = u'12'
+        a += u'3'
+        assert int(a) == 123
+        assert long(a) == 123
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -771,8 +771,7 @@
         spec = space.unicode_w(w_format_spec)
         formatter = newformat.unicode_formatter(space, spec)
         self2 = unicode_from_object(space, self)
-        assert isinstance(self2, W_UnicodeObject)
-        return formatter.format_string(self2._value)
+        return formatter.format_string(self2.unicode_w(space))
 
     def descr_mod(self, space, w_values):
         return mod_format(space, self, w_values, do_unicode=True)
@@ -977,11 +976,12 @@
         raise oefmt(space.w_TypeError, "decoding bytearray is not supported")
 
     w_retval = decode_object(space, w_obj, encoding, errors)
-    if not space.isinstance_w(w_retval, space.w_unicode):
-        raise oefmt(space.w_TypeError,
-                    "decoder did not return an unicode object (type '%T')",
-                    w_retval)
-    assert isinstance(w_retval, W_UnicodeObject)
+    if not isinstance(w_retval, W_UnicodeObject):
+        if not space.isinstance_w(w_retval, space.w_unicode):
+            raise oefmt(space.w_TypeError,
+                        "decoder did not return an unicode object (type '%T')",
+                        w_retval)
+        w_retval = W_UnicodeObject(w_retval.unicode_w(space))
     return w_retval
 
 
@@ -1115,9 +1115,9 @@
 
 # Helper for converting int/long
 def unicode_to_decimal_w(space, w_unistr):
-    if not isinstance(w_unistr, W_UnicodeObject):
+    if not space.isinstance_w(w_unistr, space.w_unicode):
         raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr)
-    unistr = w_unistr._value
+    unistr = w_unistr.unicode_w(space)
     result = ['\0'] * len(unistr)
     digits = ['0', '1', '2', '3', '4',
               '5', '6', '7', '8', '9']

From noreply at buildbot.pypy.org  Wed Jul 29 17:22:54 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 29 Jul 2015 17:22:54 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: import stmgc
Message-ID: <20150729152254.02ECA1C1346@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78713:3a55d30d7fe8
Date: 2015-07-29 17:23 +0200
http://bitbucket.org/pypy/pypy/changeset/3a55d30d7fe8/

Log:	import stmgc

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-cff6245b0bb4
+6666c6fd1aad
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -119,6 +119,7 @@
     dprintf(("copy_bk_objs_in_page_from(%d, %ld, %d)\n",
              from_segnum, (long)pagenum, only_if_not_modified));
 
+    assert(modification_lock_check_rdlock(from_segnum));
     struct list_s *list = get_priv_segment(from_segnum)->modified_old_objects;
     struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
     struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
--- a/rpython/translator/stm/src_stm/stm/hashtable.c
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -379,11 +379,13 @@
                    will make the other transaction check that it didn't
                    do any stm_hashtable_list() on the complete hashtable.
             */
+            acquire_modification_lock_wr(STM_SEGMENT->segment_num);
             STM_PSEGMENT->modified_old_objects = list_append3(
                 STM_PSEGMENT->modified_old_objects,
                 TYPE_POSITION_MARKER,      /* type1 */
                 TYPE_MODIFIED_HASHTABLE,   /* type2 */
                 (uintptr_t)hobj);          /* modif_hashtable */
+            release_modification_lock_wr(STM_SEGMENT->segment_num);
         }
     }
     entry->object = nvalue;
diff --git a/rpython/translator/stm/src_stm/stm/marker.c b/rpython/translator/stm/src_stm/stm/marker.c
--- a/rpython/translator/stm/src_stm/stm/marker.c
+++ b/rpython/translator/stm/src_stm/stm/marker.c
@@ -73,12 +73,14 @@
     /* -2 is not odd */
     assert(marker.odd_number != (uintptr_t)TYPE_MODIFIED_HASHTABLE);
 
+    acquire_modification_lock_wr(STM_SEGMENT->segment_num);
     STM_PSEGMENT->position_markers_last = list_count(list);
     STM_PSEGMENT->modified_old_objects = list_append3(
         list,
         TYPE_POSITION_MARKER,         /* type */
         marker.odd_number,            /* marker_odd_number */
         (uintptr_t)marker.object);    /* marker_object */
+    release_modification_lock_wr(STM_SEGMENT->segment_num);
 }
 
 static void timing_write_read_contention(struct stm_undo_s *start,

From noreply at buildbot.pypy.org  Wed Jul 29 17:22:55 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 29 Jul 2015 17:22:55 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: fix for interleaved jit-backend
	entries in pypylog
Message-ID: <20150729152255.223791C1346@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78714:3f9121c93d94
Date: 2015-07-29 17:24 +0200
http://bitbucket.org/pypy/pypy/changeset/3f9121c93d94/

Log:	fix for interleaved jit-backend entries in pypylog

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -348,6 +348,12 @@
         hooks = None
     operations = get_deep_immutable_oplist(loop.operations)
     metainterp_sd.profiler.start_backend()
+
+    if rgc.stm_is_enabled():
+        # become inevitable to avoid interleaving concurrent "{jit-backend"
+        # (fine bc. we become inevitable in assembler.setup() anyway)
+        rstm.become_inevitable()
+
     debug_start("jit-backend")
     try:
         loopname = jitdriver_sd.warmstate.get_location_str(greenkey)
@@ -395,6 +401,12 @@
         debug_info = None
     operations = get_deep_immutable_oplist(operations)
     metainterp_sd.profiler.start_backend()
+
+    if rgc.stm_is_enabled():
+        # become inevitable to avoid interleaving concurrent "{jit-backend"
+        # (fine bc. we become inevitable in assembler.setup() anyway)
+        rstm.become_inevitable()
+
     debug_start("jit-backend")
     try:
         asminfo = do_compile_bridge(metainterp_sd, faildescr, inputargs,
@@ -612,12 +624,12 @@
         increment = jitdriver_sd.warmstate.increment_trace_eagerness
         result = jitcounter.tick(hash, increment)
         if rgc.stm_is_enabled():
-            # The call to guard_already_patched is necessary because it is 
-            # possible that the current transaction didn't see the 
+            # The call to guard_already_patched is necessary because it is
+            # possible that the current transaction didn't see the
             # patched JMP yet, but already sees the ST_BUSY_FLAG as 0 (because
             # the patching is in raw-memory).
             # Thus it may try to compile a trace too and also patch the assembler.
-            # However, this would trigger the assertion in 
+            # However, this would trigger the assertion in
             #     x86.assembler.patch_jump_for_descr.
             result = result and not metainterp_sd.cpu.guard_already_patched(self)
         return result

From noreply at buildbot.pypy.org  Wed Jul 29 23:28:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 29 Jul 2015 23:28:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2100: massively improve the
 performance of map() with more than
Message-ID: <20150729212849.8FF7E1C0135@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78715:4012e89fb82d
Date: 2015-07-29 23:28 +0200
http://bitbucket.org/pypy/pypy/changeset/4012e89fb82d/

Log:	Issue #2100: massively improve the performance of map() with more
	than one sequence argument

diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -53,6 +53,33 @@
         last = last + x
     return last
 
+
+class _Cons(object):
+    def __init__(self, prev, iter):
+        self.prev = prev
+        self.iter = iter
+
+    def fetch(self):
+        # recursive, loop-less version of the algorithm: works best for a
+        # fixed number of "collections" in the call to map(func, *collections)
+        prev = self.prev
+        if prev is None:
+            args1 = ()
+            stop = True
+        else:
+            args1, stop = prev.fetch()
+        iter = self.iter
+        if iter is None:
+            val = None
+        else:
+            try:
+                val = next(iter)
+                stop = False
+            except StopIteration:
+                self.iter = None
+                val = None
+        return args1 + (val,), stop
+
 def map(func, *collections):
     """map(function, sequence[, sequence, ...]) -> list
 
@@ -69,45 +96,30 @@
     if num_collections == 1:
         if none_func:
             return list(collections[0])
-        # Special case for the really common case of a single collection,
-        # this can be eliminated if we could unroll that loop that creates
-        # `args` based on whether or not len(collections) was constant
+        # Special case for the really common case of a single collection
         seq = collections[0]
         with _ManagedNewlistHint(operator._length_hint(seq, 0)) as result:
             for item in seq:
                 result.append(func(item))
             return result
 
-    # Gather the iterators (pair of (iter, has_finished)) and guess the
+    # Gather the iterators into _Cons objects and guess the
     # result length (the max of the input lengths)
-    iterators = []
+    c = None
     max_hint = 0
     for seq in collections:
-        iterators.append((iter(seq), False))
+        c = _Cons(c, iter(seq))
         max_hint = max(max_hint, operator._length_hint(seq, 0))
 
     with _ManagedNewlistHint(max_hint) as result:
         while True:
-            cont = False
-            args = []
-            for idx, (iterator, has_finished) in enumerate(iterators):
-                val = None
-                if not has_finished:
-                    try:
-                        val = next(iterator)
-                    except StopIteration:
-                        iterators[idx] = (None, True)
-                    else:
-                        cont = True
-                args.append(val)
-            args = tuple(args)
-            if cont:
-                if none_func:
-                    result.append(args)
-                else:
-                    result.append(func(*args))
+            args, stop = c.fetch()
+            if stop:
+                return result
+            if none_func:
+                result.append(args)
             else:
-                return result
+                result.append(func(*args))
 
 class _ManagedNewlistHint(object):
     """ Context manager returning a newlist_hint upon entry.
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -57,6 +57,11 @@
         b = []
         assert map(lambda x, y: x, a, b) == a
 
+    def test_map_second_item(self):
+        a = []
+        b = [1, 2, 3, 4, 5]
+        assert map(lambda x, y: y, a, b) == b
+
     def test_map_iterables(self):
         class A(object):
             def __init__(self, n):

From noreply at buildbot.pypy.org  Thu Jul 30 10:33:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 10:33:26 +0200 (CEST)
Subject: [pypy-commit] cffi default: clean up
Message-ID: <20150730083326.D36361C136F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2227:7224ecb03870
Date: 2015-07-17 19:20 +0200
http://bitbucket.org/cffi/cffi/changeset/7224ecb03870/

Log:	clean up

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -4,11 +4,6 @@
 
 VERSION = "0x2601"
 
-try:
-    int_type = (int, long)
-except NameError:    # Python 3
-    int_type = int
-
 
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):

From noreply at buildbot.pypy.org  Thu Jul 30 10:33:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 10:33:28 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #214: parse function declarations
 like "int foo(void_t); " where
Message-ID: <20150730083328.039AE1C14CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2228:7dcd3c2ef47f
Date: 2015-07-30 10:33 +0200
http://bitbucket.org/cffi/cffi/changeset/7dcd3c2ef47f/

Log:	Issue #214: parse function declarations like "int foo(void_t);"
	where void_t is typedefed to void.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -423,13 +423,10 @@
                 raise api.CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
-        elif (len(params) == 1 and
-            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
-                and list(params[0].type.type.names) == ['void']):
-            del params[0]
         args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -237,6 +237,13 @@
     ffi = FFI()
     ffi.cdef("typedef _Bool bool; void f(bool);")
 
+def test_void_renamed_as_only_arg():
+    ffi = FFI()
+    ffi.cdef("typedef void void_t1;"
+             "typedef void_t1 void_t;"
+             "typedef int (*func_t)(void_t);")
+    assert ffi.typeof("func_t").args == ()
+
 def test_win_common_types():
     from cffi.commontypes import COMMON_TYPES, _CACHE
     from cffi.commontypes import win_common_types, resolve_common_type

From noreply at buildbot.pypy.org  Thu Jul 30 10:40:16 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 10:40:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal, arigo)
Message-ID: <20150730084016.31C881C026B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78716:ea3543fd65b8
Date: 2015-07-30 10:40 +0200
http://bitbucket.org/pypy/pypy/changeset/ea3543fd65b8/

Log:	(fijal, arigo)

	Don't use TypeError in RPython please (grafted from
	b2013723e0ecaefaf2f35f063c74c873bfd57667)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -200,7 +200,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
@@ -208,7 +208,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
@@ -216,7 +216,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
@@ -224,7 +224,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def str_w(self, space):
         self._typed_unwrap_error(space, "string")
@@ -354,6 +354,9 @@
 class DescrMismatch(Exception):
     pass
 
+class BufferInterfaceNotFound(Exception):
+    pass
+
 def wrappable_class_name(Class):
     try:
         return Class.typedef.name
@@ -1403,7 +1406,7 @@
         # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
         try:
             return w_obj.buffer_w(self, flags)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "'%T' does not have the buffer interface", w_obj)
 
@@ -1411,7 +1414,7 @@
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
             return w_obj.readbuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "expected a readable buffer object")
 
@@ -1419,7 +1422,7 @@
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
             return w_obj.writebuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "expected a writeable buffer object")
 
@@ -1427,7 +1430,7 @@
         # Old buffer interface, returns a character buffer (PyObject_AsCharBuffer)
         try:
             return w_obj.charbuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "expected a character buffer object")
 
@@ -1451,11 +1454,11 @@
                 return self.str(w_obj).readbuf_w(self)
             try:
                 return w_obj.buffer_w(self, 0)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 pass
             try:
                 return w_obj.readbuf_w(self)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("string or buffer", w_obj)
         elif code == 's#':
             if self.isinstance_w(w_obj, self.w_str):
@@ -1464,24 +1467,23 @@
                 return self.str(w_obj).str_w(self)
             try:
                 return w_obj.readbuf_w(self).as_str()
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only buffer", w_obj)
         elif code == 'w*':
             try:
-                try:
-                    return w_obj.buffer_w(self, self.BUF_WRITABLE)
-                except OperationError:
-                    self._getarg_error("read-write buffer", w_obj)
-            except TypeError:
+                return w_obj.buffer_w(self, self.BUF_WRITABLE)
+            except OperationError:
+                self._getarg_error("read-write buffer", w_obj)
+            except BufferInterfaceNotFound:
                 pass
             try:
                 return w_obj.writebuf_w(self)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("read-write buffer", w_obj)
         elif code == 't#':
             try:
                 return w_obj.charbuf_w(self)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only character buffer", w_obj)
         else:
             assert False
@@ -1503,13 +1505,13 @@
                 raise
         try:
             buf = w_obj.buffer_w(self, 0)
-        except TypeError:
+        except BufferInterfaceNotFound:
             pass
         else:
             return buf.as_str()
         try:
             buf = w_obj.readbuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             self._getarg_error("string or buffer", w_obj)
         else:
             return buf.as_str()
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -12,6 +12,7 @@
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
     interp_attrproperty, make_weakref_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.interpreter.streamutil import wrap_streamerror, wrap_oserror_as_ioerror
 
 
@@ -499,7 +500,7 @@
                         line = w_line.readbuf_w(space).as_str()
                     else:
                         line = w_line.charbuf_w(space)
-                except TypeError:
+                except BufferInterfaceNotFound:
                     raise OperationError(space.w_TypeError, space.wrap(
                         "writelines() argument must be a sequence of strings"))
                 else:
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.baseobjspace import W_Root, BufferInterfaceNotFound
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.error import OperationError, wrap_windowserror, oefmt
@@ -335,7 +335,7 @@
         else:
             try:
                 value = w_value.readbuf_w(space)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 raise oefmt(space.w_TypeError,
                             "Objects of type '%T' can not be used as binary "
                             "registry values", w_value)
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -9,6 +9,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
@@ -60,7 +61,7 @@
     try:
         a_buf = w_a.buffer_w(space, space.BUF_SIMPLE)
         b_buf = w_b.buffer_w(space, space.BUF_SIMPLE)
-    except TypeError:
+    except BufferInterfaceNotFound:
         raise oefmt(space.w_TypeError,
                     "unsupported operand types(s) or combination of types: "
                     "'%T' and '%T'", w_a, w_b)

From noreply at buildbot.pypy.org  Thu Jul 30 11:57:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 11:57:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add another test,
	which I'm unable to fix either
Message-ID: <20150730095714.8AC481C131A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78717:dfe31a786086
Date: 2015-07-30 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/dfe31a786086/

Log:	Add another test, which I'm unable to fix either

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4433,8 +4433,8 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f2, [int])
 
-    @py.test.mark.xfail("'__pypy__' in sys.modules")
     def test_property_union_2(self):
+        py.test.xfail("FIX ME")
         class Base(object):
             pass
 
@@ -4458,8 +4458,28 @@
             return obj.x
         a = self.RPythonAnnotator()
         # Ideally, this should translate to something sensible,
-        # but for now, UnionError is better than silently mistranslating.
-        with py.test.raises(annmodel.UnionError):
+        # but for now, AnnotatorError is better than silently mistranslating.
+        with py.test.raises(annmodel.AnnotatorError):
+            a.build_types(f, [int])
+
+    def test_property_union_3(self):
+        py.test.xfail("FIX ME")
+        class Base(object):
+            pass
+        class A(Base):
+            @property
+            def x(self):
+                return 42
+        class B(Base):
+            x = 43
+        def f(n):
+            if n < 0:
+                obj = A()
+            else:
+                obj = B()
+            return obj.x
+        a = self.RPythonAnnotator()
+        with py.test.raises(annmodel.AnnotatorError):
             a.build_types(f, [int])
 
 

From noreply at buildbot.pypy.org  Thu Jul 30 12:10:52 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 30 Jul 2015 12:10:52 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: enable 'perf' to display loop
 names for jit-trace profiling
Message-ID: <20150730101052.792CD1C06B9@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78718:df6612cd98c5
Date: 2015-07-30 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/df6612cd98c5/

Log:	enable 'perf' to display loop names for jit-trace profiling

	Add a script that extracts a perf-PID.map file from a pypylog. This
	allows 'perf report' to show the name of traces instead of code
	addresses. Still needs some work, but basic functionality is there.

diff --git a/pypy/stm/make_perf_map_from_pypylog.py b/pypy/stm/make_perf_map_from_pypylog.py
new file mode 100755
--- /dev/null
+++ b/pypy/stm/make_perf_map_from_pypylog.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env pypy
+from rpython.tool.logparser import extract_category
+from rpython.tool.jitlogparser.storage import LoopStorage
+from rpython.tool.jitlogparser.parser import adjust_bridges, import_log,\
+    parse_log_counts, SimpleParser
+
+
+import argparse
+import os
+
+
+loop_to_asm = {}
+
+class SymbolMapParser(SimpleParser):
+    def postprocess(self, loop, backend_dump=None, backend_tp=None,
+                    dump_start=0, symbols=None):
+        if backend_dump is not None:
+            loop_to_asm[loop] = (dump_start+loop.operations[0].offset,
+                                 loop.last_offset)#len(backend_dump.decode('hex')))
+
+            #import pdb;pdb.set_trace()
+            # print loop.comment
+            # print hex(dump_start), loop.last_offset
+            # print len(backend_dump), len(backend_dump.decode('hex'))
+
+            # raw_asm = self._asm_disassemble(backend_dump.decode('hex'),
+            #                                 backend_tp, dump_start)
+            # start = 0
+            # for elem in raw_asm:
+            #     if len(elem.split("\t")) < 3:
+            #         continue
+            #     e = elem.split("\t")
+            #     adr = e[0]
+            #     if not start:
+            #         start = int(adr.strip(":"), 16)
+            #         break
+            # print "real start", hex(start)
+        #return SimpleParser.postprocess(self, loop, backend_dump, backend_tp, dump_start, symbols)
+        return loop
+
+
+def mangle_descr(descr):
+    if descr.startswith('TargetToken('):
+        return descr[len('TargetToken('):-1]
+    if descr.startswith('<Guard'):
+        return 'bridge-' + str(int(descr[len('<Guard0x'):-1], 16))
+    if descr.startswith('<Loop'):
+        return 'entry-' + descr[len('<Loop'):-1]
+    return descr.replace(" ", '-')
+
+
+def create_loop_dict(loops):
+    d = {}
+    for loop in loops:
+        d[mangle_descr(loop.descr)] = loop
+    return d
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description = "creates a /tmp/perf-<pid>.map file from a pypylog",
+        epilog = "./make_perf_map_from_pypylog.py -l log.pypylog -p 1234",
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    parser.add_argument("-l", "--log", help="specify existing logfile")
+    parser.add_argument("-p", "--pid", help="specify original PID (process id)")
+    args = parser.parse_args()
+    if args.log is None:
+        print("--log is required")
+        return
+    if args.pid is None:
+        print("--pid is required")
+        return
+
+    filename = args.log
+    extra_path = os.path.dirname(filename)
+
+    storage = LoopStorage(extra_path)
+    log, loops = import_log(filename, SymbolMapParser)
+    parse_log_counts(extract_category(log, 'jit-backend-count'), loops)
+    storage.loops = loops
+    storage.loop_dict = create_loop_dict(loops)
+
+    for loop in storage.loops:
+        if hasattr(loop, 'force_asm'):
+            loop.force_asm(loop=loop)
+
+        comment = loop.comment
+        start, stop = comment.find('('), comment.rfind(')')
+        loop.name = comment[start+1:stop]
+
+    with open('/tmp/perf-%s.map' % args.pid, 'w') as f:
+        for loop, (start, size) in loop_to_asm.items():
+            line = "%x %x %s\n" % (start, size, loop.name)
+            os.write(1, line)
+            f.write(line)
+
+
+
+
+if __name__ == '__main__':
+    main()

From noreply at buildbot.pypy.org  Thu Jul 30 13:58:53 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 30 Jul 2015 13:58:53 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: improve information supplied to
	'perf'
Message-ID: <20150730115853.D67891C1347@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78719:6820a6ef6d83
Date: 2015-07-30 14:00 +0200
http://bitbucket.org/pypy/pypy/changeset/6820a6ef6d83/

Log:	improve information supplied to 'perf'

diff --git a/pypy/stm/make_perf_map_from_pypylog.py b/pypy/stm/make_perf_map_from_pypylog.py
--- a/pypy/stm/make_perf_map_from_pypylog.py
+++ b/pypy/stm/make_perf_map_from_pypylog.py
@@ -1,59 +1,29 @@
 #!/usr/bin/env pypy
 from rpython.tool.logparser import extract_category
 from rpython.tool.jitlogparser.storage import LoopStorage
-from rpython.tool.jitlogparser.parser import adjust_bridges, import_log,\
+from rpython.tool.jitlogparser.parser import import_log,\
     parse_log_counts, SimpleParser
 
-
+from collections import OrderedDict
 import argparse
 import os
 
 
-loop_to_asm = {}
+loop_to_asm = OrderedDict()
+global_dumps = OrderedDict()
 
 class SymbolMapParser(SimpleParser):
     def postprocess(self, loop, backend_dump=None, backend_tp=None,
                     dump_start=0, symbols=None):
         if backend_dump is not None:
-            loop_to_asm[loop] = (dump_start+loop.operations[0].offset,
-                                 loop.last_offset)#len(backend_dump.decode('hex')))
-
-            #import pdb;pdb.set_trace()
-            # print loop.comment
-            # print hex(dump_start), loop.last_offset
-            # print len(backend_dump), len(backend_dump.decode('hex'))
-
-            # raw_asm = self._asm_disassemble(backend_dump.decode('hex'),
-            #                                 backend_tp, dump_start)
-            # start = 0
-            # for elem in raw_asm:
-            #     if len(elem.split("\t")) < 3:
-            #         continue
-            #     e = elem.split("\t")
-            #     adr = e[0]
-            #     if not start:
-            #         start = int(adr.strip(":"), 16)
-            #         break
-            # print "real start", hex(start)
-        #return SimpleParser.postprocess(self, loop, backend_dump, backend_tp, dump_start, symbols)
+            if dump_start not in global_dumps:
+                global_dumps[dump_start] = (loop, backend_dump)
+            start_offset = loop.operations[0].offset
+            loop_to_asm[loop] = (dump_start + start_offset,
+                                 loop.last_offset - start_offset)
         return loop
 
 
-def mangle_descr(descr):
-    if descr.startswith('TargetToken('):
-        return descr[len('TargetToken('):-1]
-    if descr.startswith('<Guard'):
-        return 'bridge-' + str(int(descr[len('<Guard0x'):-1], 16))
-    if descr.startswith('<Loop'):
-        return 'entry-' + descr[len('<Loop'):-1]
-    return descr.replace(" ", '-')
-
-
-def create_loop_dict(loops):
-    d = {}
-    for loop in loops:
-        d[mangle_descr(loop.descr)] = loop
-    return d
 
 
 def main():
@@ -80,7 +50,6 @@
     log, loops = import_log(filename, SymbolMapParser)
     parse_log_counts(extract_category(log, 'jit-backend-count'), loops)
     storage.loops = loops
-    storage.loop_dict = create_loop_dict(loops)
 
     for loop in storage.loops:
         if hasattr(loop, 'force_asm'):
@@ -88,16 +57,32 @@
 
         comment = loop.comment
         start, stop = comment.find('('), comment.rfind(')')
-        loop.name = comment[start+1:stop]
+        count = loop.count if hasattr(loop, 'count') else '?'
+        loop.name = comment[start+1:stop] + " (ran %sx)" % count
 
     with open('/tmp/perf-%s.map' % args.pid, 'w') as f:
-        for loop, (start, size) in loop_to_asm.items():
-            line = "%x %x %s\n" % (start, size, loop.name)
+        lines = []
+        fmt = "%x %x %s\n"
+        # fine-grained first seems to work:
+        # output last entries first
+        for loop, (start, size) in reversed(loop_to_asm.items()):
+            lines.append(fmt % (start, size,
+                                "JIT: " + loop.name))
+
+        # coarse loop-pieces: they include e.g. frame-reallocation
+        # in compiled bridge (whatever jitviewer also doesn't show
+        # but is still part of a loop)
+        for start, (loop, dump) in reversed(global_dumps.items()):
+            lines.append(fmt % (start, len(dump.decode('hex')),
+                                "JIT-ext: " + loop.name))
+
+        for line in lines:
             os.write(1, line)
             f.write(line)
 
 
 
 
+
 if __name__ == '__main__':
     main()

From noreply at buildbot.pypy.org  Thu Jul 30 14:46:41 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 30 Jul 2015 14:46:41 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: fix
Message-ID: <20150730124641.6A9D51C026B@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78720:f43e39e2ebad
Date: 2015-07-30 14:48 +0200
http://bitbucket.org/pypy/pypy/changeset/f43e39e2ebad/

Log:	fix

diff --git a/pypy/stm/make_perf_map_from_pypylog.py b/pypy/stm/make_perf_map_from_pypylog.py
--- a/pypy/stm/make_perf_map_from_pypylog.py
+++ b/pypy/stm/make_perf_map_from_pypylog.py
@@ -19,8 +19,9 @@
             if dump_start not in global_dumps:
                 global_dumps[dump_start] = (loop, backend_dump)
             start_offset = loop.operations[0].offset
+            last_offset = loop.operations[-1].offset
             loop_to_asm[loop] = (dump_start + start_offset,
-                                 loop.last_offset - start_offset)
+                                 last_offset - start_offset)
         return loop
 
 

From noreply at buildbot.pypy.org  Thu Jul 30 15:47:26 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 30 Jul 2015 15:47:26 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: hopefully fix code dump assignment
	for good
Message-ID: <20150730134726.5ED5F1C026B@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78721:fe70fc5ea16b
Date: 2015-07-30 15:48 +0200
http://bitbucket.org/pypy/pypy/changeset/fe70fc5ea16b/

Log:	hopefully fix code dump assignment for good

diff --git a/pypy/stm/make_perf_map_from_pypylog.py b/pypy/stm/make_perf_map_from_pypylog.py
--- a/pypy/stm/make_perf_map_from_pypylog.py
+++ b/pypy/stm/make_perf_map_from_pypylog.py
@@ -8,20 +8,22 @@
 import argparse
 import os
 
-
-loop_to_asm = OrderedDict()
-global_dumps = OrderedDict()
+extended_ranges = {}
 
 class SymbolMapParser(SimpleParser):
     def postprocess(self, loop, backend_dump=None, backend_tp=None,
                     dump_start=0, symbols=None):
         if backend_dump is not None:
-            if dump_start not in global_dumps:
-                global_dumps[dump_start] = (loop, backend_dump)
+            loop.backend_dump = backend_dump
+            loop.dump_start = dump_start
+
             start_offset = loop.operations[0].offset
             last_offset = loop.operations[-1].offset
-            loop_to_asm[loop] = (dump_start + start_offset,
-                                 last_offset - start_offset)
+            loop.range = (start_offset, last_offset)
+
+            prev = extended_ranges.get(backend_dump, loop.range)
+            extended_ranges[backend_dump] = (min(start_offset, prev[0]),
+                                             max(last_offset, prev[1]))
         return loop
 
 
@@ -62,24 +64,24 @@
         loop.name = comment[start+1:stop] + " (ran %sx)" % count
 
     with open('/tmp/perf-%s.map' % args.pid, 'w') as f:
-        lines = []
         fmt = "%x %x %s\n"
-        # fine-grained first seems to work:
-        # output last entries first
-        for loop, (start, size) in reversed(loop_to_asm.items()):
-            lines.append(fmt % (start, size,
-                                "JIT: " + loop.name))
 
-        # coarse loop-pieces: they include e.g. frame-reallocation
-        # in compiled bridge (whatever jitviewer also doesn't show
-        # but is still part of a loop)
-        for start, (loop, dump) in reversed(global_dumps.items()):
-            lines.append(fmt % (start, len(dump.decode('hex')),
-                                "JIT-ext: " + loop.name))
+        for loop in storage.loops:
+            if hasattr(loop, 'backend_dump'):
+                lower, upper = loop.range
+                min_offset, max_offset = extended_ranges[loop.backend_dump]
+                if lower == min_offset:
+                    # include loop-setup
+                    lower = 0
+                if upper == max_offset:
+                    # include loop-teardown
+                    upper = loop.last_offset
 
-        for line in lines:
-            os.write(1, line)
-            f.write(line)
+                line = fmt % (lower + loop.dump_start,
+                              upper - lower,
+                              "JIT: " + loop.name)
+                f.write(line)
+                os.write(1, line)
 
 
 

From noreply at buildbot.pypy.org  Thu Jul 30 16:48:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 16:48:36 +0200 (CEST)
Subject: [pypy-commit] cffi default: Handle line continuations in the "//"
	comments
Message-ID: <20150730144836.6D37D1C026B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2229:024d67a5aa74
Date: 2015-07-30 16:49 +0200
http://bitbucket.org/cffi/cffi/changeset/024d67a5aa74/

Log:	Handle line continuations in the "//" comments

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -15,7 +15,8 @@
 except ImportError:
     lock = None
 
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
                         re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -159,6 +159,23 @@
     assert func.name == 'sin'
     assert func.BType == '<func (<double>, <double>), <double>, False>'
 
+def test_remove_line_continuation_comments():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        double // blah \\
+                  more comments
+        x(void);
+        double // blah\\\\
+        y(void);
+        double // blah\\ \
+                  etc
+        z(void);
+    """)
+    m = ffi.dlopen(lib_m)
+    m.x
+    m.y
+    m.z
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')

From noreply at buildbot.pypy.org  Thu Jul 30 17:04:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 17:04:25 +0200 (CEST)
Subject: [pypy-commit] cffi default: Support "\" escapes in a #define
Message-ID: <20150730150425.756751C06B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2230:d5d809e7081b
Date: 2015-07-30 17:05 +0200
http://bitbucket.org/cffi/cffi/changeset/d5d809e7081b/

Log:	Support "\" escapes in a #define

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -17,8 +17,9 @@
 
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
-                        re.MULTILINE)
+_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -40,6 +41,7 @@
     macros = {}
     for match in _r_define.finditer(csource):
         macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
         macros[macroname] = macrovalue
     csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -176,6 +176,18 @@
     m.y
     m.z
 
+def test_line_continuation_in_defines():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        #define ABC\\
+            42
+        #define BCD   \\
+            43
+    """)
+    m = ffi.dlopen(lib_m)
+    assert m.ABC == 42
+    assert m.BCD == 43
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:22 2015
From: noreply at buildbot.pypy.org (Alex Stapleton)
Date: Thu, 30 Jul 2015 22:41:22 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: Add Parser._extract_macros and
	_clean_macros
Message-ID: <20150730204122.3F4241C1347@cobra.cs.uni-duesseldorf.de>

Author: Alex Stapleton <alex at lyst.com>
Branch: cmacros
Changeset: r2231:254fab3123e3
Date: 2015-07-25 12:04 +0100
http://bitbucket.org/cffi/cffi/changeset/254fab3123e3/

Log:	Add Parser._extract_macros and _clean_macros

	Will be used for adding #if support

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -24,6 +24,7 @@
 _r_words = re.compile(r"\w+|\S")
 _parser_cache = None
 _r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE)
+_r_c_macro = re.compile(r"^\s*#")
 
 def _get_parser():
     global _parser_cache
@@ -104,6 +105,38 @@
         self._recomplete = []
         self._uses_new_feature = None
 
+    def _extract_macros(self, csource):
+        """
+        Extract macros from csource.
+
+        :returns: [(ln, macro), ...]
+        """
+
+        macros = []
+
+        for num, line in enumerate(csource.splitlines()):
+            if _r_c_macro.match(line):
+                macros.append((num, line))
+
+        return macros
+
+    def _clean_macros(self, csource):
+        """
+        Remove macros from the csource
+
+        :returns: csource minus any C macros
+        """
+
+        cleaned = []
+
+        for line in csource.splitlines():
+            if _r_c_macro.match(line):
+                cleaned.append("")
+            else:
+                cleaned.append(line)
+
+        return '\n'.join(cleaned)
+
     def _parse(self, csource):
         csource, macros = _preprocess(csource)
         # XXX: for more efficiency we would need to poke into the
@@ -122,7 +155,9 @@
         csourcelines = ['typedef int %s;' % typename for typename in typenames]
         csourcelines.append('typedef int __dotdotdot__;')
         csourcelines.append(csource)
+
         csource = '\n'.join(csourcelines)
+
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -1,5 +1,6 @@
-import py, sys, re
+import py, sys, re, textwrap
 from cffi import FFI, FFIError, CDefError, VerificationError
+from cffi.cparser import Parser
 
 class FakeBackend(object):
 
@@ -142,6 +143,39 @@
         BType = ffi._get_cached_btype(type)
     assert str(BType) == '<func (<pointer to <int>>), <int>, False>'
 
+def test_extract_macros():
+    parser = Parser()
+    macros = parser._extract_macros("""
+    int x;
+    #if defined(FOO)
+    int y;
+    #endif
+    int z;
+    """)
+
+    assert macros == [
+        (2, "    #if defined(FOO)"),
+        (4, "    #endif")
+    ]
+
+def test_clean_macros():
+    parser = Parser()
+    clean = parser._clean_macros("""
+    int x;
+    #if defined(FOO)
+    int y;
+    #endif
+    int z;
+    """)
+
+    assert clean == """
+    int x;
+
+    int y;
+
+    int z;
+    """
+
 def test_remove_comments():
     ffi = FFI(backend=FakeBackend())
     ffi.cdef("""

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:23 2015
From: noreply at buildbot.pypy.org (Alex Stapleton)
Date: Thu, 30 Jul 2015 22:41:23 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: #if defined() now might work for
 functions if you dont look too hard
Message-ID: <20150730204123.632471C1347@cobra.cs.uni-duesseldorf.de>

Author: Alex Stapleton <alex at lyst.com>
Branch: cmacros
Changeset: r2232:fb67276a8bd2
Date: 2015-07-25 14:48 +0100
http://bitbucket.org/cffi/cffi/changeset/fb67276a8bd2/

Log:	#if defined() now might work for functions if you dont look too hard

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -24,7 +24,8 @@
 _r_words = re.compile(r"\w+|\S")
 _parser_cache = None
 _r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE)
-_r_c_macro = re.compile(r"^\s*#")
+_r_enter_c_macro = re.compile(r"^\s*#\s*(if|elif|ifdef|ifndef|else)\s*(.*)")
+_r_exit_c_macro = re.compile(r"^\s*#\s*(endif)")
 
 def _get_parser():
     global _parser_cache
@@ -36,12 +37,6 @@
     # Remove comments.  NOTE: this only work because the cdef() section
     # should not contain any string literal!
     csource = _r_comment.sub(' ', csource)
-    # Remove the "#define FOO x" lines
-    macros = {}
-    for match in _r_define.finditer(csource):
-        macroname, macrovalue = match.groups()
-        macros[macroname] = macrovalue
-    csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
     csource = _r_partial_array.sub('[__dotdotdotarray__]', csource)
     # Replace "...}" with "__dotdotdotNUM__}".  This construction should
@@ -63,7 +58,7 @@
                                                  csource[p+3:])
     # Replace all remaining "..." with the same name, "__dotdotdot__",
     # which is declared with a typedef for the purpose of C parsing.
-    return csource.replace('...', ' __dotdotdot__ '), macros
+    return csource.replace('...', ' __dotdotdot__ ')
 
 def _common_type_names(csource):
     # Look in the source for what looks like usages of types from the
@@ -105,22 +100,51 @@
         self._recomplete = []
         self._uses_new_feature = None
 
-    def _extract_macros(self, csource):
+    def _extract_ifdefs(self, csource):
         """
-        Extract macros from csource.
-
-        :returns: [(ln, macro), ...]
+        Extract macros from csource. (Also defines.)
         """
 
-        macros = []
+        current = []
+        continuing = False
+
+        stack = []
+        ifdefs = []
+        defines = []
 
         for num, line in enumerate(csource.splitlines()):
-            if _r_c_macro.match(line):
-                macros.append((num, line))
+            if continuing or _r_enter_c_macro.match(line):
+                line_condition = ""
+                current.append(line)
+                continuing = line.endswith("\\")
 
-        return macros
+                if not continuing:
+                    macro = "".join(current)
+                    match = _r_enter_c_macro.match(macro)
 
-    def _clean_macros(self, csource):
+                    if match.group(1) == "else":
+                        stack.append("!({0})".format(stack.pop()))
+                    else:
+                        stack.append(match.group(2))
+
+                    current = []
+
+            elif _r_exit_c_macro.match(line) and stack:
+                line_condition = ""
+                stack.pop()
+
+            else:
+                line_condition = " && ".join(stack)
+
+                if _r_define.match(line):
+                    match = _r_define.match(line)
+                    defines.append((num, match.group(1), match.group(2)))
+
+            ifdefs.append(line_condition)
+
+        return ifdefs, defines
+
+    def _clean_ifdefs(self, csource):
         """
         Remove macros from the csource
 
@@ -130,7 +154,7 @@
         cleaned = []
 
         for line in csource.splitlines():
-            if _r_c_macro.match(line):
+            if _r_enter_c_macro.match(line) or _r_exit_c_macro.match(line) or _r_define.match(line):
                 cleaned.append("")
             else:
                 cleaned.append(line)
@@ -138,7 +162,8 @@
         return '\n'.join(cleaned)
 
     def _parse(self, csource):
-        csource, macros = _preprocess(csource)
+        csource = _preprocess(csource)
+
         # XXX: for more efficiency we would need to poke into the
         # internals of CParser...  the following registers the
         # typedefs, because their presence or absence influences the
@@ -151,12 +176,14 @@
                 typenames.append(name)
                 ctn.discard(name)
         typenames += sorted(ctn)
-        #
+
         csourcelines = ['typedef int %s;' % typename for typename in typenames]
         csourcelines.append('typedef int __dotdotdot__;')
         csourcelines.append(csource)
+        csource = '\n'.join(csourcelines)
 
-        csource = '\n'.join(csourcelines)
+        ifdefs, defines = self._extract_ifdefs(csource)
+        csource = self._clean_ifdefs(csource)
 
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
@@ -168,7 +195,7 @@
             if lock is not None:
                 lock.release()
         # csource will be used to find buggy source text
-        return ast, macros, csource
+        return ast, defines, csource, ifdefs
 
     def _convert_pycparser_error(self, e, csource):
         # xxx look for ":NUM:" at the start of str(e) and try to interpret
@@ -206,9 +233,13 @@
             self._packed = prev_packed
 
     def _internal_parse(self, csource):
-        ast, macros, csource = self._parse(csource)
-        # add the macros
-        self._process_macros(macros)
+        ast, defines, csource, ifdefs = self._parse(csource)
+
+        self._ifdefs = ifdefs
+
+        # add the defines
+        self._process_defines(defines)
+
         # find the first "__dotdotdot__" and use that as a separator
         # between the repeated typedefs and the real csource
         iterator = iter(ast.ext)
@@ -235,7 +266,9 @@
                         realtype = model.unknown_ptr_type(decl.name)
                     else:
                         realtype = self._get_type(decl.type, name=decl.name)
-                    self._declare('typedef ' + decl.name, realtype)
+
+                    ifdef = self._ifdefs[decl.coord.line - 1]
+                    self._declare(('typedef', decl.name, ifdef), realtype)
                 else:
                     raise api.CDefError("unrecognized construct", decl)
         except api.FFIError as e:
@@ -267,13 +300,13 @@
         self._add_constants(name, pyvalue)
         self._declare('macro ' + name, pyvalue)
 
-    def _process_macros(self, macros):
-        for key, value in macros.items():
+    def _process_defines(self, defines):
+        for ln, key, value in defines:
             value = value.strip()
             if _r_int_literal.match(value):
                 self._add_integer_constant(key, value)
-            elif value == '...':
-                self._declare('macro ' + key, value)
+            elif value == '__dotdotdot__':
+                self._declare(('macro', key), value)
             else:
                 raise api.CDefError(
                     'only supports one of the following syntax:\n'
@@ -285,12 +318,14 @@
                     % (key, key, key, value))
 
     def _parse_decl(self, decl):
+        ifdef = self._ifdefs[decl.coord.line - 1]
         node = decl.type
+
         if isinstance(node, pycparser.c_ast.FuncDecl):
             tp = self._get_type(node, name=decl.name)
             assert isinstance(tp, model.RawFunctionType)
             tp = self._get_type_pointer(tp)
-            self._declare('function ' + decl.name, tp)
+            self._declare(('function', decl.name, ifdef), tp)
         else:
             if isinstance(node, pycparser.c_ast.Struct):
                 self._get_struct_union_enum_type('struct', node)
@@ -306,7 +341,7 @@
                 tp = self._get_type(node, partial_length_ok=True)
                 if tp.is_raw_function:
                     tp = self._get_type_pointer(tp)
-                    self._declare('function ' + decl.name, tp)
+                    self._declare(('function', decl.name, ifdef), tp)
                 elif (tp.is_integer_type() and
                         hasattr(decl, 'init') and
                         hasattr(decl.init, 'value') and
@@ -320,9 +355,9 @@
                     self._add_integer_constant(decl.name,
                                                '-' + decl.init.expr.value)
                 elif self._is_constant_globalvar(node):
-                    self._declare('constant ' + decl.name, tp)
+                    self._declare(('constant', decl.name, ifdef), tp)
                 else:
-                    self._declare('variable ' + decl.name, tp)
+                    self._declare(('variable', decl.name, ifdef), tp)
 
     def parse_type(self, cdecl):
         ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2]
@@ -332,16 +367,19 @@
             raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
         return self._get_type(exprnode.type)
 
-    def _declare(self, name, obj, included=False):
-        if name in self._declarations:
-            if self._declarations[name] is obj:
+    def _declare(self, key, obj, included=False):
+        assert isinstance(key, tuple)
+
+        if key in self._declarations:
+            if self._declarations[key] is obj:
                 return
+
             if not self._override:
                 raise api.FFIError(
-                    "multiple declarations of %s (for interactive usage, "
-                    "try cdef(xx, override=True))" % (name,))
-        assert '__dotdotdot__' not in name.split()
-        self._declarations[name] = obj
+                    "multiple declarations of %s %s (for interactive usage, "
+                    "try cdef(xx, override=True))" % (key[0], key[1]))
+        assert '__dotdotdot__' != key[1]
+        self._declarations[key] = obj
         if included:
             self._included_declarations.add(obj)
 
@@ -520,7 +558,7 @@
             tp = None
         else:
             explicit_name = name
-            key = '%s %s' % (kind, name)
+            key = (kind, name)
             tp = self._declarations.get(key, None)
         #
         if tp is None:
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -10,6 +10,19 @@
     int_type = int
 
 
+class Ifdef:
+    def __init__(self, ifdef, child):
+        self.name = child.name
+        self._ifdef = ifdef
+        self._child = child
+
+    def as_c_expr(self):
+        return """\
+        #if {0}
+        {1}
+        #endif
+        """.format(self._ifdef, self._child.as_c_expr())
+
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):
         self.name = name
@@ -210,18 +223,20 @@
         return sorted(self.ffi._parser._declarations.items())
 
     def _generate(self, step_name):
-        for name, tp in self._get_declarations():
-            kind, realname = name.split(' ', 1)
+        for key, tp in self._get_declarations():
+            kind, realname, ifdef = key
+
             try:
                 method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                 step_name))
             except AttributeError:
                 raise ffiplatform.VerificationError(
-                    "not implemented in recompile(): %r" % name)
+                    "not implemented in recompile(): %r %r" % key[:2])
+
             try:
-                method(tp, realname)
+                method(tp, realname, ifdef)
             except Exception as e:
-                model.attach_exception_info(e, name)
+                model.attach_exception_info(e, key)
                 raise
 
     # ----------
@@ -356,7 +371,10 @@
                 prnt('  NULL,  /* no %ss */' % step_name)
         for step_name in self.ALL_STEPS:
             if step_name != "field":
-                prnt('  %d,  /* num_%ss */' % (nums[step_name], step_name))
+                if nums[step_name]:
+                    prnt('  sizeof(_cffi_{0}s)/sizeof(_cffi_{0}s[0]), /* num_{0}s */'.format(step_name))
+                else:
+                    prnt('  0, /* num_{0}s */'.format(step_name))
         if self.ffi._included_ffis:
             prnt('  _cffi_includes,')
         else:
@@ -574,12 +592,12 @@
     # ----------
     # function declarations
 
-    def _generate_cpy_function_collecttype(self, tp, name):
+    def _generate_cpy_function_collecttype(self, tp, name, ifdef):
         self._do_collect_type(tp.as_raw_function())
         if tp.ellipsis and not self.target_is_python:
             self._do_collect_type(tp)
 
-    def _generate_cpy_function_decl(self, tp, name):
+    def _generate_cpy_function_decl(self, tp, name, ifdef):
         assert not self.target_is_python
         assert isinstance(tp, model.FunctionPtrType)
         if tp.ellipsis:
@@ -596,6 +614,9 @@
             argname = 'arg0'
         else:
             argname = 'args'
+
+        prnt('#if {0}'.format(ifdef))
+
         #
         # ------------------------------
         # the 'd' version of the function, only for addressof(lib, 'func')
@@ -724,9 +745,10 @@
             prnt('#  define _cffi_f_%s _cffi_d_%s' % (name, name))
         #
         prnt('#endif')        # ------------------------------
+        prnt('#endif')
         prnt()
 
-    def _generate_cpy_function_ctx(self, tp, name):
+    def _generate_cpy_function_ctx(self, tp, name, ifdef):
         if tp.ellipsis and not self.target_is_python:
             self._generate_cpy_constant_ctx(tp, name)
             return
@@ -741,9 +763,12 @@
         else:
             meth_kind = OP_CPYTHON_BLTN_V   # 'METH_VARARGS'
         self._lsts["global"].append(
-            GlobalExpr(name, '_cffi_f_%s' % name,
-                       CffiOp(meth_kind, type_index),
-                       size='_cffi_d_%s' % name))
+            Ifdef(ifdef,
+                GlobalExpr(name, '_cffi_f_%s' % name,
+                    CffiOp(meth_kind, type_index),
+                    size='_cffi_d_%s' % name)
+            )
+        )
 
     # ----------
     # named structs or unions
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -143,36 +143,59 @@
         BType = ffi._get_cached_btype(type)
     assert str(BType) == '<func (<pointer to <int>>), <int>, False>'
 
-def test_extract_macros():
+def test_extract_ifdefs():
     parser = Parser()
-    macros = parser._extract_macros("""
+
+    macros = parser._extract_ifdefs("""
+    #if FOO
+    int q;
+    #else
     int x;
-    #if defined(FOO)
+    #if BAR
     int y;
     #endif
+    #endif
     int z;
     """)
 
     assert macros == [
-        (2, "    #if defined(FOO)"),
-        (4, "    #endif")
+        '',
+        '',
+        'FOO',
+        '',
+        '!(FOO)',
+        '',
+        '!(FOO) && BAR',
+        '',
+        '',
+        '',
+        ''
     ]
 
-def test_clean_macros():
+
+def test_clean_ifdefs():
     parser = Parser()
-    clean = parser._clean_macros("""
+    clean = parser._clean_ifdefs("""
+    #if FOO
+    int q;
+    #else
     int x;
-    #if defined(FOO)
+    #if BAR
     int y;
     #endif
+    #endif
     int z;
     """)
 
     assert clean == """
+
+    int q;
+
     int x;
 
     int y;
 
+
     int z;
     """
 
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -116,6 +116,17 @@
                      "(FUNCTION 1)(POINTER 3)(FUNCTION_END 0)(STRUCT_UNION 0)",
                      included="struct foo_s { int x, y; };")
 
+def test_math_sin_ifdef():
+    import math
+    ffi = FFI()
+    ffi.cdef("""
+    #if defined(MATH_PLEASE)
+    float sinxxx(double);
+    double cos(double);
+    #endif
+    """)
+    lib = verify(ffi, 'test_math_sin_ifdef', '#include <math.h>')
+    assert not hasattr(lib, "cos")
 
 def test_math_sin():
     import math

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:24 2015
From: noreply at buildbot.pypy.org (Alex Stapleton)
Date: Thu, 30 Jul 2015 22:41:24 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: #if defined() now works for unprotected
	functions too
Message-ID: <20150730204124.67CB21C1347@cobra.cs.uni-duesseldorf.de>

Author: Alex Stapleton <alex at lyst.com>
Branch: cmacros
Changeset: r2233:a9930603a424
Date: 2015-07-25 14:50 +0100
http://bitbucket.org/cffi/cffi/changeset/a9930603a424/

Log:	#if defined() now works for unprotected functions too

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -17,11 +17,14 @@
         self._child = child
 
     def as_c_expr(self):
-        return """\
-        #if {0}
-        {1}
-        #endif
-        """.format(self._ifdef, self._child.as_c_expr())
+        if self._ifdef:
+            return """\
+            #if {0}
+            {1}
+            #endif
+            """.format(self._ifdef, self._child.as_c_expr())
+        else:
+            return self._child.as_c_expr()
 
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):
@@ -615,7 +618,8 @@
         else:
             argname = 'args'
 
-        prnt('#if {0}'.format(ifdef))
+        if ifdef:
+            prnt('#if {0}'.format(ifdef))
 
         #
         # ------------------------------
@@ -745,7 +749,10 @@
             prnt('#  define _cffi_f_%s _cffi_d_%s' % (name, name))
         #
         prnt('#endif')        # ------------------------------
-        prnt('#endif')
+
+        if ifdef:
+            prnt('#endif')
+
         prnt()
 
     def _generate_cpy_function_ctx(self, tp, name, ifdef):

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 22:41:25 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: hg merge default
Message-ID: <20150730204125.62B6E1C1347@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cmacros
Changeset: r2234:55774b4c3293
Date: 2015-07-30 16:50 +0200
http://bitbucket.org/cffi/cffi/changeset/55774b4c3293/

Log:	hg merge default

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -15,7 +15,8 @@
 except ImportError:
     lock = None
 
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
                         re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
@@ -496,13 +497,10 @@
                 raise api.CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
-        elif (len(params) == 1 and
-            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
-                and list(params[0].type.type.names) == ['void']):
-            del params[0]
         args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -4,11 +4,6 @@
 
 VERSION = "0x2601"
 
-try:
-    int_type = (int, long)
-except NameError:    # Python 3
-    int_type = int
-
 
 class Ifdef:
     def __init__(self, ifdef, child):
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -216,6 +216,23 @@
     assert func.name == 'sin'
     assert func.BType == '<func (<double>, <double>), <double>, False>'
 
+def test_remove_line_continuation_comments():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        double // blah \\
+                  more comments
+        x(void);
+        double // blah\\\\
+        y(void);
+        double // blah\\ \
+                  etc
+        z(void);
+    """)
+    m = ffi.dlopen(lib_m)
+    m.x
+    m.y
+    m.z
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')
@@ -294,6 +311,13 @@
     ffi = FFI()
     ffi.cdef("typedef _Bool bool; void f(bool);")
 
+def test_void_renamed_as_only_arg():
+    ffi = FFI()
+    ffi.cdef("typedef void void_t1;"
+             "typedef void_t1 void_t;"
+             "typedef int (*func_t)(void_t);")
+    assert ffi.typeof("func_t").args == ()
+
 def test_win_common_types():
     from cffi.commontypes import COMMON_TYPES, _CACHE
     from cffi.commontypes import win_common_types, resolve_common_type

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 22:41:26 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: hg merge default
Message-ID: <20150730204126.5FB941C1347@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cmacros
Changeset: r2235:e40456e91447
Date: 2015-07-30 17:06 +0200
http://bitbucket.org/cffi/cffi/changeset/e40456e91447/

Log:	hg merge default

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -17,8 +17,9 @@
 
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
-                        re.MULTILINE)
+_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -38,6 +39,13 @@
     # Remove comments.  NOTE: this only work because the cdef() section
     # should not contain any string literal!
     csource = _r_comment.sub(' ', csource)
+    # Remove the "#define FOO x" lines
+    macros = {}
+    for match in _r_define.finditer(csource):
+        macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
+        macros[macroname] = macrovalue
+    csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
     csource = _r_partial_array.sub('[__dotdotdotarray__]', csource)
     # Replace "...}" with "__dotdotdotNUM__}".  This construction should
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -233,6 +233,18 @@
     m.y
     m.z
 
+def test_line_continuation_in_defines():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        #define ABC\\
+            42
+        #define BCD   \\
+            43
+    """)
+    m = ffi.dlopen(lib_m)
+    assert m.ABC == 42
+    assert m.BCD == 43
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 22:41:27 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: Partial rewrite of the core logic for
 extracting and interpreting #if
Message-ID: <20150730204127.567C81C1347@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cmacros
Changeset: r2236:5126820f564f
Date: 2015-07-30 18:25 +0200
http://bitbucket.org/cffi/cffi/changeset/5126820f564f/

Log:	Partial rewrite of the core logic for extracting and interpreting
	#if macros, including #else, #elif, and so on (incl. continuation
	lines)

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -17,7 +17,7 @@
 
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+_r_define  = re.compile(r"^[ \t\r\f\v]*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
                         r"\b((?:[^\n\\]|\\.)*?)$",
                         re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
@@ -26,8 +26,9 @@
 _r_words = re.compile(r"\w+|\S")
 _parser_cache = None
 _r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE)
-_r_enter_c_macro = re.compile(r"^\s*#\s*(if|elif|ifdef|ifndef|else)\s*(.*)")
-_r_exit_c_macro = re.compile(r"^\s*#\s*(endif)")
+_r_ifdef = re.compile(r"^[ \t\r\f\v]*#\s*(if|elif|ifdef|ifndef|else|endif)"
+                      r"\b((?:[^\n\\]|\\.)*?)$",
+                      re.DOTALL | re.MULTILINE)
 
 def _get_parser():
     global _parser_cache
@@ -67,7 +68,7 @@
                                                  csource[p+3:])
     # Replace all remaining "..." with the same name, "__dotdotdot__",
     # which is declared with a typedef for the purpose of C parsing.
-    return csource.replace('...', ' __dotdotdot__ ')
+    return csource.replace('...', ' __dotdotdot__ '), macros
 
 def _common_type_names(csource):
     # Look in the source for what looks like usages of types from the
@@ -111,67 +112,75 @@
 
     def _extract_ifdefs(self, csource):
         """
-        Extract macros from csource. (Also defines.)
+        Extract the "#if" conditions from csource.
+        Returns a list with one item per source line, which is an
+        empty string if that line appears outside "#if" blocks, or
+        otherwise a string giving the condition for that line.
         """
-
-        current = []
-        continuing = False
-
+        ifdefs = []
         stack = []
-        ifdefs = []
         defines = []
 
-        for num, line in enumerate(csource.splitlines()):
-            if continuing or _r_enter_c_macro.match(line):
-                line_condition = ""
-                current.append(line)
-                continuing = line.endswith("\\")
+        def flush():
+            n = len(ifdefs)
+            assert n <= linenum1
+            if len(stack) == 0:
+                result = ''
+            elif len(stack) == 1:
+                result = stack[0]
+            else:
+                result = ' && '.join(['(%s)' % s for s in stack])
+            result = result.replace('\x00', ' && ')
+            ifdefs.extend([result] * (linenum1 - n))
 
-                if not continuing:
-                    macro = "".join(current)
-                    match = _r_enter_c_macro.match(macro)
+        def pop():
+            if not stack:
+                raise api.CDefError("line %d: unexpected '#%s'" % (
+                    linenum1, keyword))
+            return stack.pop()
 
-                    if match.group(1) == "else":
-                        stack.append("!({0})".format(stack.pop()))
-                    else:
-                        stack.append(match.group(2))
+        def negate(cond):
+            limit = cond.rfind('\x00') + 1
+            old, recent = cond[:limit], cond[limit:]
+            return '%s!(%s)' % (old, recent)
 
-                    current = []
+        for match in _r_ifdef.finditer(csource):
+            linenum1 = csource[:match.start()].count('\n')
+            linenum2 = linenum1 + match.group().count('\n') + 1
+            flush()
+            keyword = match.group(1)
+            condition = match.group(2).replace('\\\n', '').strip()
+            if keyword == 'if':
+                stack.append(condition)
+            elif keyword == 'ifdef':
+                stack.append('defined(%s)' % condition)
+            elif keyword == 'ifndef':
+                stack.append('!defined(%s)' % condition)
+            elif keyword == 'elif':
+                condition1 = pop()
+                stack.append('%s\x00(%s)' % (negate(condition1), condition))
+            elif keyword == 'else':
+                condition1 = pop()
+                stack.append(negate(condition1))
+            elif keyword == 'endif':
+                pop()
+            else:
+                raise AssertionError(keyword)
+            assert len(ifdefs) == linenum1
+            ifdefs += [None] * (linenum2 - linenum1)
+        if stack:
+            raise api.CDefError("there are more '#ifXXX' than '#endif'")
+        linenum1 = csource.count('\n') + 1
+        flush()
 
-            elif _r_exit_c_macro.match(line) and stack:
-                line_condition = ""
-                stack.pop()
-
-            else:
-                line_condition = " && ".join(stack)
-
-                if _r_define.match(line):
-                    match = _r_define.match(line)
-                    defines.append((num, match.group(1), match.group(2)))
-
-            ifdefs.append(line_condition)
-
-        return ifdefs, defines
-
-    def _clean_ifdefs(self, csource):
-        """
-        Remove macros from the csource
-
-        :returns: csource minus any C macros
-        """
-
-        cleaned = []
-
-        for line in csource.splitlines():
-            if _r_enter_c_macro.match(line) or _r_exit_c_macro.match(line) or _r_define.match(line):
-                cleaned.append("")
-            else:
-                cleaned.append(line)
-
-        return '\n'.join(cleaned)
+        def replace_with_eol(m):
+            num_eol = m.group().count('\n')
+            return num_eol * '\n'
+        csource = _r_ifdef.sub(replace_with_eol, csource)
+        return csource, ifdefs
 
     def _parse(self, csource):
-        csource = _preprocess(csource)
+        csource, defines = _preprocess(csource)
 
         # XXX: for more efficiency we would need to poke into the
         # internals of CParser...  the following registers the
@@ -185,14 +194,13 @@
                 typenames.append(name)
                 ctn.discard(name)
         typenames += sorted(ctn)
-
+        #
         csourcelines = ['typedef int %s;' % typename for typename in typenames]
         csourcelines.append('typedef int __dotdotdot__;')
         csourcelines.append(csource)
         csource = '\n'.join(csourcelines)
 
-        ifdefs, defines = self._extract_ifdefs(csource)
-        csource = self._clean_ifdefs(csource)
+        csource, ifdefs = self._extract_ifdefs(csource)
 
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -143,10 +143,39 @@
         BType = ffi._get_cached_btype(type)
     assert str(BType) == '<func (<pointer to <int>>), <int>, False>'
 
-def test_extract_ifdefs():
+def test_extract_ifdefs_err():
+    parser = Parser()
+    py.test.raises(CDefError, parser._extract_ifdefs, "#if ABC")  # unbalanced
+    py.test.raises(CDefError, parser._extract_ifdefs, "#else")    # unexpected
+    py.test.raises(CDefError, parser._extract_ifdefs, "#endif")   # unexpected
+
+def test_extract_ifdefs_1():
     parser = Parser()
 
-    macros = parser._extract_ifdefs("""
+    _, macros = parser._extract_ifdefs("""
+    #ifdef FOO
+    int q;
+    #endif
+    #ifndef BAR
+    int b;
+    #endif
+    """)
+
+    assert macros == [
+        '',
+        None,
+        'defined(FOO)',
+        None,
+        None,
+        '!defined(BAR)',
+        None,
+        ''
+    ]
+
+def test_extract_ifdefs_2():
+    parser = Parser()
+
+    _, macros = parser._extract_ifdefs("""
     #if FOO
     int q;
     #else
@@ -160,22 +189,84 @@
 
     assert macros == [
         '',
-        '',
+        None,
         'FOO',
-        '',
+        None,
         '!(FOO)',
-        '',
-        '!(FOO) && BAR',
-        '',
-        '',
+        None,
+        '(!(FOO)) && (BAR)',
+        None,
+        None,
         '',
         ''
     ]
 
+def test_extract_ifdefs_3():
+    parser = Parser()
+
+    _, macros = parser._extract_ifdefs("""
+    #if FOO
+    int q;
+    #elif BAR
+    int x;
+    #elif BAZ
+    int y;
+    #else
+    int z;
+    #endif
+    """)
+
+    assert macros == [
+        '',
+        None,
+        'FOO',
+        None,
+        '!(FOO) && (BAR)',
+        None,
+        '!(FOO) && !((BAR)) && (BAZ)',
+        None,
+        '!(FOO) && !((BAR)) && !((BAZ))',
+        None,
+        ''
+    ]
+
+def test_extract_ifdefs_continuation():
+    parser = Parser()
+
+    clean, macros = parser._extract_ifdefs(r"""   // <= note the 'r' here
+    #if FOO \
+ FO\\O2
+    int q;
+    #elif BAR\
+BAR2
+    int x;
+    #endif
+    """)
+
+    assert macros == [
+        '',
+        None,
+        None,
+        r'FOO  FO\\O2',
+        None,
+        None,
+        r'!(FOO  FO\\O2) && (BARBAR2)',
+        None,
+        ''
+    ]
+    assert clean == r"""   // <= note the 'r' here
+
+
+    int q;
+
+
+    int x;
+
+    """
 
 def test_clean_ifdefs():
     parser = Parser()
-    clean = parser._clean_ifdefs("""
+    clean, _ = parser._extract_ifdefs("""
     #if FOO
     int q;
     #else

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 22:41:28 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: Replace long sequences of spaces with a
	single space
Message-ID: <20150730204128.4BA3D1C1347@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cmacros
Changeset: r2237:247fc2ae6f64
Date: 2015-07-30 18:28 +0200
http://bitbucket.org/cffi/cffi/changeset/247fc2ae6f64/

Log:	Replace long sequences of spaces with a single space

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -29,6 +29,7 @@
 _r_ifdef = re.compile(r"^[ \t\r\f\v]*#\s*(if|elif|ifdef|ifndef|else|endif)"
                       r"\b((?:[^\n\\]|\\.)*?)$",
                       re.DOTALL | re.MULTILINE)
+_r_multispaces = re.compile(r"\s+", re.MULTILINE)
 
 def _get_parser():
     global _parser_cache
@@ -150,6 +151,7 @@
             flush()
             keyword = match.group(1)
             condition = match.group(2).replace('\\\n', '').strip()
+            condition = _r_multispaces.sub(' ', condition)
             if keyword == 'if':
                 stack.append(condition)
             elif keyword == 'ifdef':
@@ -160,9 +162,11 @@
                 condition1 = pop()
                 stack.append('%s\x00(%s)' % (negate(condition1), condition))
             elif keyword == 'else':
+                # 'condition' ignored here
                 condition1 = pop()
                 stack.append(negate(condition1))
             elif keyword == 'endif':
+                # 'condition' ignored here
                 pop()
             else:
                 raise AssertionError(keyword)
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -247,10 +247,10 @@
         '',
         None,
         None,
-        r'FOO  FO\\O2',
+        r'FOO FO\\O2',
         None,
         None,
-        r'!(FOO  FO\\O2) && (BARBAR2)',
+        r'!(FOO FO\\O2) && (BARBAR2)',
         None,
         ''
     ]

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 22:41:29 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: One more passing test
Message-ID: <20150730204129.45DD71C1347@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cmacros
Changeset: r2238:b8ce44d5c791
Date: 2015-07-30 18:32 +0200
http://bitbucket.org/cffi/cffi/changeset/b8ce44d5c791/

Log:	One more passing test

diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -230,6 +230,37 @@
         ''
     ]
 
+def test_extract_ifdefs_4():
+    parser = Parser()
+
+    _, macros = parser._extract_ifdefs("""
+    #ifdef ABC
+    #ifdef BCD
+    int q;
+    #elif BAR
+    int x;
+    #else
+    int y;
+    #endif
+    int z;
+    #endif
+    """)
+
+    assert macros == [
+        '',
+        None,
+        None,
+        '(defined(ABC)) && (defined(BCD))',
+        None,
+        '(defined(ABC)) && (!(defined(BCD)) && (BAR))',
+        None,
+        '(defined(ABC)) && (!(defined(BCD)) && !((BAR)))',
+        None,
+        'defined(ABC)',
+        None,
+        ''
+    ]
+
 def test_extract_ifdefs_continuation():
     parser = Parser()
 

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 22:41:30 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: Reintroduce condition-controlled
	"#define"
Message-ID: <20150730204130.42E9A1C1347@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cmacros
Changeset: r2239:b9af889b244a
Date: 2015-07-30 22:13 +0200
http://bitbucket.org/cffi/cffi/changeset/b9af889b244a/

Log:	Reintroduce condition-controlled "#define"

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -17,19 +17,18 @@
 
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^[ \t\r\f\v]*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
-                        r"\b((?:[^\n\\]|\\.)*?)$",
-                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
 _r_words = re.compile(r"\w+|\S")
 _parser_cache = None
 _r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE)
-_r_ifdef = re.compile(r"^[ \t\r\f\v]*#\s*(if|elif|ifdef|ifndef|else|endif)"
-                      r"\b((?:[^\n\\]|\\.)*?)$",
-                      re.DOTALL | re.MULTILINE)
+_r_ifdef = re.compile(
+    r"^[ \t\r\f\v]*#\s*(if|elif|ifdef|ifndef|else|endif|define)"
+    r"\b((?:[^\n\\]|\\.)*?)$",
+    re.DOTALL | re.MULTILINE)
 _r_multispaces = re.compile(r"\s+", re.MULTILINE)
+_r_single_word = re.compile(r"[A-Za-z_][A-Za-z0-9_]+")
 
 def _get_parser():
     global _parser_cache
@@ -41,13 +40,6 @@
     # Remove comments.  NOTE: this only work because the cdef() section
     # should not contain any string literal!
     csource = _r_comment.sub(' ', csource)
-    # Remove the "#define FOO x" lines
-    macros = {}
-    for match in _r_define.finditer(csource):
-        macroname, macrovalue = match.groups()
-        macrovalue = macrovalue.replace('\\\n', '').strip()
-        macros[macroname] = macrovalue
-    csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
     csource = _r_partial_array.sub('[__dotdotdotarray__]', csource)
     # Replace "...}" with "__dotdotdotNUM__}".  This construction should
@@ -69,7 +61,7 @@
                                                  csource[p+3:])
     # Replace all remaining "..." with the same name, "__dotdotdot__",
     # which is declared with a typedef for the purpose of C parsing.
-    return csource.replace('...', ' __dotdotdot__ '), macros
+    return csource.replace('...', ' __dotdotdot__ ')
 
 def _common_type_names(csource):
     # Look in the source for what looks like usages of types from the
@@ -133,6 +125,7 @@
                 result = ' && '.join(['(%s)' % s for s in stack])
             result = result.replace('\x00', ' && ')
             ifdefs.extend([result] * (linenum1 - n))
+            return result
 
         def pop():
             if not stack:
@@ -148,11 +141,19 @@
         for match in _r_ifdef.finditer(csource):
             linenum1 = csource[:match.start()].count('\n')
             linenum2 = linenum1 + match.group().count('\n') + 1
-            flush()
+            current_outer_condition = flush()
             keyword = match.group(1)
             condition = match.group(2).replace('\\\n', '').strip()
             condition = _r_multispaces.sub(' ', condition)
-            if keyword == 'if':
+            if keyword == 'define':
+                match2 = _r_single_word.match(condition)
+                if not match2:
+                    raise api.CDefError("line %d: '#define' not followed by "
+                                        "a word" % linenum1)
+                singleword = match2.group()
+                defines.append((current_outer_condition, singleword,
+                                condition[len(singleword):].strip()))
+            elif keyword == 'if':
                 stack.append(condition)
             elif keyword == 'ifdef':
                 stack.append('defined(%s)' % condition)
@@ -181,10 +182,10 @@
             num_eol = m.group().count('\n')
             return num_eol * '\n'
         csource = _r_ifdef.sub(replace_with_eol, csource)
-        return csource, ifdefs
+        return csource, defines, ifdefs
 
     def _parse(self, csource):
-        csource, defines = _preprocess(csource)
+        csource = _preprocess(csource)
 
         # XXX: for more efficiency we would need to poke into the
         # internals of CParser...  the following registers the
@@ -204,7 +205,7 @@
         csourcelines.append(csource)
         csource = '\n'.join(csourcelines)
 
-        csource, ifdefs = self._extract_ifdefs(csource)
+        csource, defines, ifdefs = self._extract_ifdefs(csource)
 
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
@@ -381,8 +382,9 @@
                     self._declare(('variable', decl.name, ifdef), tp)
 
     def parse_type(self, cdecl):
-        ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2]
+        ast, macros, _, ifdefs = self._parse('void __dummy(\n%s\n);' % cdecl)
         assert not macros
+        assert not ifdefs
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
             raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -152,7 +152,7 @@
 def test_extract_ifdefs_1():
     parser = Parser()
 
-    _, macros = parser._extract_ifdefs("""
+    _, _, macros = parser._extract_ifdefs("""
     #ifdef FOO
     int q;
     #endif
@@ -175,7 +175,7 @@
 def test_extract_ifdefs_2():
     parser = Parser()
 
-    _, macros = parser._extract_ifdefs("""
+    _, _, macros = parser._extract_ifdefs("""
     #if FOO
     int q;
     #else
@@ -204,7 +204,7 @@
 def test_extract_ifdefs_3():
     parser = Parser()
 
-    _, macros = parser._extract_ifdefs("""
+    _, _, macros = parser._extract_ifdefs("""
     #if FOO
     int q;
     #elif BAR
@@ -233,7 +233,7 @@
 def test_extract_ifdefs_4():
     parser = Parser()
 
-    _, macros = parser._extract_ifdefs("""
+    _, _, macros = parser._extract_ifdefs("""
     #ifdef ABC
     #ifdef BCD
     int q;
@@ -264,7 +264,7 @@
 def test_extract_ifdefs_continuation():
     parser = Parser()
 
-    clean, macros = parser._extract_ifdefs(r"""   // <= note the 'r' here
+    clean, _, macros = parser._extract_ifdefs(r"""   // <= note the 'r' here
     #if FOO \
  FO\\O2
     int q;
@@ -297,7 +297,7 @@
 
 def test_clean_ifdefs():
     parser = Parser()
-    clean, _ = parser._extract_ifdefs("""
+    clean, _, _ = parser._extract_ifdefs("""
     #if FOO
     int q;
     #else
@@ -321,6 +321,28 @@
     int z;
     """
 
+def test_defines_with_ifdefs():
+    parser = Parser()
+    _, defines, macros = parser._extract_ifdefs("""
+    #if FOO
+    #  define ABC 42
+    #else
+    #  define BCD 4\\
+3
+    #endif
+    """)
+
+    assert macros == [
+        '',
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        '']
+    assert defines == [('FOO', 'ABC', '42'), ('!(FOO)', 'BCD', '43')]
+
 def test_remove_comments():
     ffi = FFI(backend=FakeBackend())
     ffi.cdef("""
@@ -362,10 +384,13 @@
             42
         #define BCD   \\
             43
+        #define CDE  3\\
+9
     """)
     m = ffi.dlopen(lib_m)
     assert m.ABC == 42
     assert m.BCD == 43
+    assert m.CDE == 39
 
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())

From noreply at buildbot.pypy.org  Thu Jul 30 22:41:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 30 Jul 2015 22:41:31 +0200 (CEST)
Subject: [pypy-commit] cffi cmacros: Fixes
Message-ID: <20150730204131.3B4881C1347@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: cmacros
Changeset: r2240:5cae421affd9
Date: 2015-07-30 22:17 +0200
http://bitbucket.org/cffi/cffi/changeset/5cae421affd9/

Log:	Fixes

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -607,7 +607,7 @@
     copied_enums = []
     #
     def make_accessor_locked(name):
-        key = 'function ' + name
+        key = ('function', name, '')
         if key in ffi._parser._declarations:
             tp = ffi._parser._declarations[key]
             BType = ffi._get_cached_btype(tp)
@@ -618,7 +618,7 @@
             library.__dict__[name] = value
             return
         #
-        key = 'variable ' + name
+        key = ('variable', name, '')
         if key in ffi._parser._declarations:
             tp = ffi._parser._declarations[key]
             BType = ffi._get_cached_btype(tp)
@@ -645,7 +645,7 @@
             if name in library.__dict__:
                 return
         #
-        key = 'constant ' + name
+        key = ('constant', name, '')
         if key in ffi._parser._declarations:
             raise NotImplementedError("fetching a non-integer constant "
                                       "after dlopen()")
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -392,6 +392,7 @@
 
     def _declare(self, key, obj, included=False):
         assert isinstance(key, tuple)
+        assert isinstance(key[2], str)
 
         if key in self._declarations:
             if self._declarations[key] is obj:

From noreply at buildbot.pypy.org  Fri Jul 31 10:56:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 31 Jul 2015 10:56:25 +0200 (CEST)
Subject: [pypy-commit] cffi default: Use section titles that start with a
 letter: docutils used to turn links
Message-ID: <20150731085625.EC5F01C0FE1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2241:f3fd064f445e
Date: 2015-07-31 10:57 +0200
http://bitbucket.org/cffi/cffi/changeset/f3fd064f445e/

Log:	Use section titles that start with a letter: docutils used to turn
	links to them to "id01", "id02", etc., and every new version, these
	would shift, giving the wrong "already-visited" colouring in web
	browsers

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,8 +3,8 @@
 ======================
 
 
-1.2.0
-=====
+v1.2.0
+======
 
 * Out-of-line mode: ``int a[][...];`` can be used to declare a structure
   field or global variable which is, simultaneously, of total length
@@ -56,15 +56,15 @@
 .. __: using.html#alternative-allocators
 
 
-1.1.2
-=====
+v1.1.2
+======
 
 * ``ffi.gc()``: fixed a race condition in multithreaded programs
   introduced in 1.1.1
 
 
-1.1.1
-=====
+v1.1.1
+======
 
 * Out-of-line mode: ``ffi.string()``, ``ffi.buffer()`` and
   ``ffi.getwinerror()`` didn't accept their arguments as keyword
@@ -81,8 +81,8 @@
 .. __: cdef.html#dlopen-note
 
 
-1.1.0
-=====
+v1.1.0
+======
 
 * Out-of-line API mode: we can now declare integer types with
   ``typedef int... foo_t;``.  The exact size and signedness of ``foo_t``
@@ -114,14 +114,14 @@
   in ``build/foo.c``, the .o file would be put in ``build/build/foo.o``.
 
 
-1.0.3
-=====
+v1.0.3
+======
 
 * Same as 1.0.2, apart from doc and test fixes on some platforms.
 
 
-1.0.2
-=====
+v1.0.2
+======
 
 * Variadic C functions (ending in a "..." argument) were not supported
   in the out-of-line ABI mode.  This was a bug---there was even a
@@ -130,8 +130,8 @@
 .. __: overview.html#out-of-line-abi-level
 
 
-1.0.1
-=====
+v1.0.1
+======
 
 * ``ffi.set_source()`` crashed if passed a ``sources=[..]`` argument.
   Fixed by chrippa on pull request #60.
@@ -143,8 +143,8 @@
 * Enums were buggy if you used too many "..." in their definition.
 
 
-1.0.0
-=====
+v1.0.0
+======
 
 * The main news item is out-of-line module generation:
 

From noreply at buildbot.pypy.org  Fri Jul 31 11:17:39 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 31 Jul 2015 11:17:39 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix locale_codec.c for MSVC
Message-ID: <20150731091739.190721C0EC1@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: py3k
Changeset: r78722:5ffd586cbbf4
Date: 2015-07-31 09:10 +0300
http://bitbucket.org/pypy/pypy/changeset/5ffd586cbbf4/

Log:	fix locale_codec.c for MSVC

diff --git a/pypy/module/_codecs/locale_codec.c b/pypy/module/_codecs/locale_codec.c
--- a/pypy/module/_codecs/locale_codec.c
+++ b/pypy/module/_codecs/locale_codec.c
@@ -5,7 +5,6 @@
 #include "Python.h"
 */
 #include <ctype.h>
-#include <locale.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
@@ -17,11 +16,12 @@
 #define PyMem_Free free
 /* C99 but recent Windows has it */
 #define HAVE_MBRTOWC 1
-/* Hopefully? */
-#define HAVE_LANGINFO_H
 
 #ifdef MS_WINDOWS
 #  include <windows.h>
+#else
+#include <locale.h>
+#define HAVE_LANGINFO_H
 #endif
 
 #ifdef HAVE_LANGINFO_H

From noreply at buildbot.pypy.org  Fri Jul 31 11:17:40 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 31 Jul 2015 11:17:40 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: make sure environ keys are upper case for
	tests
Message-ID: <20150731091740.5D02A1C0EC1@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: py3k
Changeset: r78723:c2e78e986378
Date: 2015-07-31 12:11 +0300
http://bitbucket.org/pypy/pypy/changeset/c2e78e986378/

Log:	make sure environ keys are upper case for tests

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -524,7 +524,7 @@
         # started through main() instead of wmain()
         rwin32._wgetenv(u"")
         for key, value in rwin32._wenviron_items():
-            space.setitem(w_env, space.wrap(key), space.wrap(value))
+            space.setitem(w_env, space.wrap(key.upper()), space.wrap(value))
 
     @unwrap_spec(name=unicode, value=unicode)
     def putenv(space, name, value):

From noreply at buildbot.pypy.org  Fri Jul 31 14:51:52 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 31 Jul 2015 14:51:52 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix complilation error
Message-ID: <20150731125152.F26941C065A@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: py3k
Changeset: r78724:4ece5080fa0f
Date: 2015-07-31 12:37 +0300
http://bitbucket.org/pypy/pypy/changeset/4ece5080fa0f/

Log:	fix complilation error

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -186,7 +186,7 @@
                              "RPY_EXTERN "
                              "char** pypy_get_tzname();\n"
                              "RPY_EXTERN "
-                             "void* pypy__tzset();"],
+                             "void pypy__tzset();"],
         separate_module_sources = ["""
         long pypy_get_timezone() { return timezone; }
         int pypy_get_daylight() { return daylight; }

From noreply at buildbot.pypy.org  Fri Jul 31 14:51:54 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 31 Jul 2015 14:51:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: update function name
Message-ID: <20150731125154.51C9E1C065A@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: py3k
Changeset: r78725:d9590a24d095
Date: 2015-07-31 13:05 +0300
http://bitbucket.org/pypy/pypy/changeset/d9590a24d095/

Log:	update function name

diff --git a/pypy/module/posix/interp_nt.py b/pypy/module/posix/interp_nt.py
--- a/pypy/module/posix/interp_nt.py
+++ b/pypy/module/posix/interp_nt.py
@@ -79,7 +79,7 @@
         with lltype.scoped_alloc(
             win32traits.BY_HANDLE_FILE_INFORMATION) as info:
             if win32traits.GetFileInformationByHandle(hFile, info) == 0:
-                raise rwin32.lastWindowsError("_getfileinformation")
+                raise rwin32.lastSavedWindowsError("_getfileinformation")
             return (rffi.cast(lltype.Signed, info.c_dwVolumeSerialNumber),
                     rffi.cast(lltype.Signed, info.c_nFileIndexHigh),
                     rffi.cast(lltype.Signed, info.c_nFileIndexLow))
@@ -101,7 +101,7 @@
                                        win32traits.FILE_FLAG_BACKUP_SEMANTICS,
                                        rwin32.NULL_HANDLE)
         if hFile == rwin32.INVALID_HANDLE_VALUE:
-            raise rwin32.lastWindowsError("CreateFile")
+            raise rwin32.lastSavedWindowsError("CreateFile")
 
         VOLUME_NAME_DOS = rffi.cast(rwin32.DWORD, win32traits.VOLUME_NAME_DOS)
         try:
@@ -111,7 +111,7 @@
                 rffi.cast(rwin32.DWORD, 0),
                 VOLUME_NAME_DOS)
             if usize == 0:
-                raise rwin32.lastWindowsError("GetFinalPathNameByHandle")
+                raise rwin32.lastSavedWindowsError("GetFinalPathNameByHandle")
 
             size = rffi.cast(lltype.Signed, usize)
             with rffi.scoped_alloc_unicodebuffer(size + 1) as buf:
@@ -121,7 +121,7 @@
                     usize,
                     VOLUME_NAME_DOS)
                 if result == 0:
-                    raise rwin32.lastWindowsError("GetFinalPathNameByHandle")
+                    raise rwin32.lastSavedWindowsError("GetFinalPathNameByHandle")
                 return buf.str(rffi.cast(lltype.Signed, result))
         finally:
             rwin32.CloseHandle(hFile)

From noreply at buildbot.pypy.org  Fri Jul 31 14:51:55 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 31 Jul 2015 14:51:55 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: translation fixes for win32
Message-ID: <20150731125155.834C71C065A@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: py3k
Changeset: r78726:7ed46662d44e
Date: 2015-07-31 15:51 +0300
http://bitbucket.org/pypy/pypy/changeset/7ed46662d44e/

Log:	translation fixes for win32

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -524,7 +524,9 @@
         # started through main() instead of wmain()
         rwin32._wgetenv(u"")
         for key, value in rwin32._wenviron_items():
-            space.setitem(w_env, space.wrap(key.upper()), space.wrap(value))
+            if isinstance(key, str):
+                key = key.upper()
+            space.setitem(w_env, space.wrap(key), space.wrap(value))
 
     @unwrap_spec(name=unicode, value=unicode)
     def putenv(space, name, value):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1,7 +1,7 @@
 import sys
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
-from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import jit
@@ -74,7 +74,7 @@
 else:
     def code_to_unichr(code):
         # generate surrogates for large codes
-        return unichr_returns_surrogate(code)
+        return unichr_returns_surrogate(widen(code))
 
 def _STORECHAR(result, CH, byteorder):
     hi = chr(((CH) >> 8) & 0xff)

From noreply at buildbot.pypy.org  Fri Jul 31 15:08:42 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 31 Jul 2015 15:08:42 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: make script for generating
	perf-maps "good-enough"
Message-ID: <20150731130842.834301C0FE1@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: stmgc-c8-gcc
Changeset: r78727:c63f782261ac
Date: 2015-07-31 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/c63f782261ac/

Log:	make script for generating perf-maps "good-enough"

	Remove some features to simplify the script a lot

diff --git a/pypy/stm/make_perf_map_from_pypylog.py b/pypy/stm/make_perf_map_from_pypylog.py
--- a/pypy/stm/make_perf_map_from_pypylog.py
+++ b/pypy/stm/make_perf_map_from_pypylog.py
@@ -1,33 +1,11 @@
 #!/usr/bin/env pypy
 from rpython.tool.logparser import extract_category
-from rpython.tool.jitlogparser.storage import LoopStorage
-from rpython.tool.jitlogparser.parser import import_log,\
-    parse_log_counts, SimpleParser
+from rpython.tool.jitlogparser.parser import (
+    parse_log_file, parse_addresses)
 
-from collections import OrderedDict
 import argparse
 import os
 
-extended_ranges = {}
-
-class SymbolMapParser(SimpleParser):
-    def postprocess(self, loop, backend_dump=None, backend_tp=None,
-                    dump_start=0, symbols=None):
-        if backend_dump is not None:
-            loop.backend_dump = backend_dump
-            loop.dump_start = dump_start
-
-            start_offset = loop.operations[0].offset
-            last_offset = loop.operations[-1].offset
-            loop.range = (start_offset, last_offset)
-
-            prev = extended_ranges.get(backend_dump, loop.range)
-            extended_ranges[backend_dump] = (min(start_offset, prev[0]),
-                                             max(last_offset, prev[1]))
-        return loop
-
-
-
 
 def main():
     parser = argparse.ArgumentParser(
@@ -47,45 +25,35 @@
         return
 
     filename = args.log
-    extra_path = os.path.dirname(filename)
+    log = parse_log_file(filename)
 
-    storage = LoopStorage(extra_path)
-    log, loops = import_log(filename, SymbolMapParser)
-    parse_log_counts(extract_category(log, 'jit-backend-count'), loops)
-    storage.loops = loops
+    addrs = []
+    full_dumps = {}
+    def address_cb(addr, stop_addr, bootstrap_addr, name, code_name):
+        addrs.append((addr, stop_addr, bootstrap_addr, name, code_name))
+        min_start_addr = full_dumps.setdefault(bootstrap_addr, addr)
+        full_dumps[bootstrap_addr] = min(min_start_addr, addr)
 
-    for loop in storage.loops:
-        if hasattr(loop, 'force_asm'):
-            loop.force_asm(loop=loop)
-
-        comment = loop.comment
-        start, stop = comment.find('('), comment.rfind(')')
-        count = loop.count if hasattr(loop, 'count') else '?'
-        loop.name = comment[start+1:stop] + " (ran %sx)" % count
+    parse_addresses(extract_category(log, 'jit-backend-addr'),
+                    callback=address_cb)
 
     with open('/tmp/perf-%s.map' % args.pid, 'w') as f:
-        fmt = "%x %x %s\n"
+        fmt = "%x %x JIT: %s - %s\n"
+        for addr, stop_addr, bootstrap_addr, name, code_name in addrs:
+            line = fmt % (addr, stop_addr - addr,
+                          name, code_name)
+            f.write(line)
+            os.write(1, line)
 
-        for loop in storage.loops:
-            if hasattr(loop, 'backend_dump'):
-                lower, upper = loop.range
-                min_offset, max_offset = extended_ranges[loop.backend_dump]
-                if lower == min_offset:
-                    # include loop-setup
-                    lower = 0
-                if upper == max_offset:
-                    # include loop-teardown
-                    upper = loop.last_offset
-
-                line = fmt % (lower + loop.dump_start,
-                              upper - lower,
-                              "JIT: " + loop.name)
+        fmt = "%x %x JIT: loop bootstrapping %x\n"
+        for bootstrap_addr, min_start_addr in full_dumps.items():
+            if bootstrap_addr != min_start_addr:
+                line = fmt % (bootstrap_addr,
+                              min_start_addr - bootstrap_addr,
+                              bootstrap_addr)
                 f.write(line)
                 os.write(1, line)
 
 
-
-
-
 if __name__ == '__main__':
     main()

From noreply at buildbot.pypy.org  Fri Jul 31 16:02:15 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 31 Jul 2015 16:02:15 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: that was a nasty problem. entering the
 vecopt trace through the preamble only worked for non accum/expanded traces,
 otherwise the arguments would not match
Message-ID: <20150731140215.44AD51C0FEC@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78728:948a227eab7f
Date: 2015-07-31 16:01 +0200
http://bitbucket.org/pypy/pypy/changeset/948a227eab7f/

Log:	that was a nasty problem. entering the vecopt trace through the
	preamble only worked for non accum/expanded traces, otherwise the
	arguments would not match the loop has now an original label, where
	invariant operations follow leading to an label that can carry
	expanded values

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -85,7 +85,7 @@
 call_many_to_one_driver = jit.JitDriver(
     name='numpy_call_many_to_one',
     greens=['shapelen', 'nin', 'func', 'res_dtype'],
-    reds='auto', vectorize=True)
+    reds='auto')
 
 def call_many_to_one(space, shape, func, res_dtype, in_args, out):
     # out must hav been built. func needs no calc_type, is usually an
@@ -119,7 +119,7 @@
 call_many_to_many_driver = jit.JitDriver(
     name='numpy_call_many_to_many',
     greens=['shapelen', 'nin', 'nout', 'func', 'res_dtype'],
-    reds='auto', vectorize=True)
+    reds='auto')
 
 def call_many_to_many(space, shape, func, res_dtype, in_args, out_args):
     # out must hav been built. func needs no calc_type, is usually an
@@ -228,7 +228,7 @@
 reduce_cum_driver = jit.JitDriver(
     name='numpy_reduce_cum_driver',
     greens=['shapelen', 'func', 'dtype', 'out_dtype'],
-    reds='auto', vectorize=True)
+    reds='auto')
 
 def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity):
     obj_iter, obj_state = obj.create_iter()
@@ -356,7 +356,7 @@
 def _new_argmin_argmax(op_name):
     arg_driver = jit.JitDriver(name='numpy_' + op_name,
                                greens = ['shapelen', 'dtype'],
-                               reds = 'auto', vectorize=True)
+                               reds = 'auto')
 
     def argmin_argmax(arr):
         result = 0
@@ -536,7 +536,7 @@
 
 flatiter_getitem_driver = jit.JitDriver(name = 'numpy_flatiter_getitem',
                                         greens = ['dtype'],
-                                        reds = 'auto')
+                                        reds = 'auto', vectorize=True)
 
 def flatiter_getitem(res, base_iter, base_state, step):
     ri, rs = res.create_iter()
@@ -570,7 +570,7 @@
 
 fromstring_driver = jit.JitDriver(name = 'numpy_fromstring',
                                   greens = ['itemsize', 'dtype'],
-                                  reds = 'auto', vectorize=True)
+                                  reds = 'auto')
 
 def fromstring_loop(space, a, dtype, itemsize, s):
     i = 0
@@ -604,7 +604,7 @@
 getitem_int_driver = jit.JitDriver(name = 'numpy_getitem_int',
                                    greens = ['shapelen', 'indexlen',
                                              'prefixlen', 'dtype'],
-                                   reds = 'auto', vectorize=True)
+                                   reds = 'auto')
 
 def getitem_array_int(space, arr, res, iter_shape, indexes_w, prefix_w):
     shapelen = len(iter_shape)
@@ -632,7 +632,7 @@
 setitem_int_driver = jit.JitDriver(name = 'numpy_setitem_int',
                                    greens = ['shapelen', 'indexlen',
                                              'prefixlen', 'dtype'],
-                                   reds = 'auto', vectorize=True)
+                                   reds = 'auto')
 
 def setitem_array_int(space, arr, iter_shape, indexes_w, val_arr,
                       prefix_w):
@@ -762,7 +762,7 @@
 
 diagonal_simple_driver = jit.JitDriver(name='numpy_diagonal_simple_driver',
                                        greens = ['axis1', 'axis2'],
-                                       reds = 'auto', vectorize=True)
+                                       reds = 'auto')
 
 def diagonal_simple(space, arr, out, offset, axis1, axis2, size):
     out_iter, out_state = out.create_iter()
@@ -806,7 +806,7 @@
 def _new_binsearch(side, op_name):
     binsearch_driver = jit.JitDriver(name='numpy_binsearch_' + side,
                                      greens=['dtype'],
-                                     reds='auto', vectorize=True)
+                                     reds='auto')
 
     def binsearch(space, arr, key, ret):
         assert len(arr.get_shape()) == 1
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -466,8 +466,6 @@
     def test_cumsum(self):
         result = self.run("cumsum")
         assert result == 15
-        # not vectorizable, has one back edge
-        self.check_vectorized(1, 0)
 
     def define_axissum():
         return """
@@ -803,7 +801,7 @@
     def test_flat_getitem(self):
         result = self.run("flat_getitem")
         assert result == 10.0
-        self.check_vectorized(0,0)
+        self.check_vectorized(1,1)
 
     def define_flat_setitem():
         return '''
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -149,23 +149,14 @@
                           [inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
                           [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
                                         None, descr=jitcell_token)]
-        target_token = part.operations[0].getdescr()
-        assert isinstance(target_token, TargetToken)
-        all_target_tokens.append(target_token)
-        inputargs = jumpargs
-        jumpargs = part.operations[-1].getarglist()
-
         try:
             optimize_trace(metainterp_sd, jitdriver_sd, part, warmstate,
                            start_state=start_state, export_state=False,
                            try_disabling_unroll=try_disabling_unroll)
         except InvalidLoop:
             return None
-
-        loop.operations = loop.operations[:-1] + part.operations
-        loop.versions = part.versions
-        if part.quasi_immutable_deps:
-            loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
+        #
+        loop.append_loop(part, all_target_tokens)
     assert part.operations[-1].getopnum() != rop.LABEL
 
     if loop.versions is not None:
@@ -197,7 +188,6 @@
     metainterp_sd = metainterp.staticdata
     cpu = metainterp_sd.cpu
     if loop.versions is not None:
-        token = jitcell_token
         for version in loop.versions:
             if len(version.faildescrs) == 0:
                 continue
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -761,7 +761,10 @@
 
     def register_all_guards(self, opt_ops, invariant_arg_count=0):
         from rpython.jit.metainterp.compile import CompileLoopVersionDescr
+        pass_by = 0
         idx = index_of_first(rop.LABEL, opt_ops)
+        if opt_ops[idx].getdescr() is not opt_ops[-1].getdescr():
+            idx = index_of_first(rop.LABEL, opt_ops, pass_by=1)
         assert idx >= 0
         version_failargs = opt_ops[idx].getarglist()
         if invariant_arg_count > 0:
@@ -799,6 +802,7 @@
         op.rd_snapshot = None
 
     def update_token(self, jitcell_token):
+        # this is only invoked for versioned loops!
         label = self.operations[self.label_pos]
         jump = self.operations[-1]
         #
@@ -849,6 +853,29 @@
             insns[opname] = insns.get(opname, 0) + 1
         return insns
 
+    def append_loop(self, loop, all_target_tokens):
+        # append e.g. the peeled loop to this loop!
+        label, jump = loop.operations[0], loop.operations[-1]
+        assert label.getopnum() == rop.LABEL
+        assert jump.getopnum() == rop.JUMP
+        target_token = None
+        i = 0
+        # adds all target token until the one is found that jumps from the 
+        # last instruction to the label
+        while target_token is not jump.getdescr():
+            # there is another label
+            op = loop.operations[i]
+            if op.getopnum() == rop.LABEL:
+                target_token = op.getdescr()
+                assert isinstance(target_token, TargetToken)
+                all_target_tokens.append(target_token)
+            i += 1
+        #
+        self.operations = self.operations[:-1] + loop.operations
+        self.versions = loop.versions
+        if loop.quasi_immutable_deps:
+            self.quasi_immutable_deps.update(loop.quasi_immutable_deps)
+
     def get_operations(self):
         return self.operations
 
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -119,8 +119,8 @@
         descr = myop.getdescr()
         descr.copy_all_attributes_from(other.op.getdescr())
         myop.rd_frame_info_list = otherop.rd_frame_info_list
+        myop.setfailargs(otherop.getfailargs())
         myop.rd_snapshot = otherop.rd_snapshot
-        myop.setfailargs(otherop.getfailargs())
 
     def emit_varops(self, opt, var, old_arg):
         assert isinstance(var, IndexVar)
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -1,6 +1,6 @@
 
 from rpython.jit.metainterp.history import (VECTOR,FLOAT,INT,ConstInt,BoxVector,
-        BoxFloat,BoxInt,ConstFloat)
+        BoxFloat,BoxInt,ConstFloat,TargetToken)
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph,
         MemoryRef, Node, IndexVar)
@@ -839,24 +839,32 @@
         assert off < vector.getcount()
         self.box_to_vbox[box] = (off, vector)
 
-    def prepend_invariant_operations(self, oplist):
+    def prepend_invariant_operations(self, oplist, orig_label_args):
         if len(self.invariant_oplist) > 0:
             label = oplist[0]
             assert label.getopnum() == rop.LABEL
+            #
             jump = oplist[-1]
             assert jump.getopnum() == rop.JUMP
-
-            label_args = label.getarglist()
+            #
+            label_args = label.getarglist()[:]
             jump_args = jump.getarglist()
             for var in self.invariant_vector_vars:
                 label_args.append(var)
                 jump_args.append(var)
-
-            oplist[0] = label.copy_and_change(label.getopnum(), label_args, None, label.getdescr())
-            oplist[-1] = jump.copy_and_change(jump.getopnum(), jump_args, None, jump.getdescr())
-
-            return self.invariant_oplist + oplist
-
+            #
+            # in case of any invariant_vector_vars, the label is restored
+            # and the invariant operations are added between the original label
+            # and the new label
+            descr = label.getdescr()
+            assert isinstance(descr, TargetToken)
+            token = TargetToken(descr.targeting_jitcell_token)
+            oplist[0] = label.copy_and_change(label.getopnum(), label_args, None, token)
+            oplist[-1] = jump.copy_and_change(jump.getopnum(), jump_args, None, token)
+            #
+            return [ResOperation(rop.LABEL, orig_label_args, None, descr)] + \
+                   self.invariant_oplist + oplist
+        #
         return oplist
 
 class Pack(object):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -107,10 +107,12 @@
         self.cpu = metainterp_sd.cpu
         self.costmodel = X86_CostModel(cost_threshold, self.cpu.vector_register_size)
         self.appended_arg_count = 0
+        self.orig_label_args = None
 
     def propagate_all_forward(self, clear=True):
         self.clear_newoperations()
         label = self.loop.operations[0]
+        self.orig_label_args = label.getarglist()[:]
         jump = self.loop.operations[-1]
         if jump.getopnum() not in (rop.LABEL, rop.JUMP) or \
            label.getopnum() != rop.LABEL:
@@ -463,7 +465,8 @@
                     if accum:
                         accum.save_to_descr(op.getdescr(),i)
         self.loop.operations = \
-            sched_data.prepend_invariant_operations(self._newoperations)
+            sched_data.prepend_invariant_operations(self._newoperations,
+                                                    self.orig_label_args)
         self.clear_newoperations()
 
     def unpack_from_vector(self, op, sched_data, renamer):
@@ -577,7 +580,7 @@
         #
         tgt_op.setdescr(descr)
         tgt_op.rd_snapshot = op.rd_snapshot
-        tgt_op.setfailargs(op.getfailargs())
+        tgt_op.setfailargs(op.getfailargs()[:])
 
 
 class CostModel(object):

From noreply at buildbot.pypy.org  Fri Jul 31 16:31:33 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 31 Jul 2015 16:31:33 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt: once again I did not copy the descr...
Message-ID: <20150731143133.6A5471C0FEC@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger <rich at pasra.at>
Branch: vecopt
Changeset: r78729:95b6c048c720
Date: 2015-07-31 16:31 +0200
http://bitbucket.org/pypy/pypy/changeset/95b6c048c720/

Log:	once again I did not copy the descr...

diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -10,6 +10,7 @@
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.jit.metainterp.history import (ConstInt, BoxVector, 
         BoxFloat, BoxInt, ConstFloat, Box, Const)
+from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.rlib.objectmodel import we_are_translated
 
 class Guard(object):
@@ -90,6 +91,9 @@
         opt.emit_operation(ResOperation(opnum, [box_rhs, other_rhs], box_result))
         # guard
         guard = self.op.clone()
+        descr = guard.getdescr()
+        assert isinstance(descr, ResumeGuardDescr)
+        guard.setdescr(descr.clone())
         guard.setarg(0, box_result)
         opt.emit_operation(guard)
 

From noreply at buildbot.pypy.org  Fri Jul 31 17:42:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 31 Jul 2015 17:42:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2095: test and fix
Message-ID: <20150731154258.CA7C01C0EC1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78730:f467823cb467
Date: 2015-07-31 17:43 +0200
http://bitbucket.org/pypy/pypy/changeset/f467823cb467/

Log:	Issue #2095: test and fix

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -711,11 +711,17 @@
         w_item = self.popvalue()
         if self.space.is_w(w_stream, self.space.w_None):
             w_stream = sys_stdout(self.space)   # grumble grumble special cases
-        print_item_to(self.space, w_item, w_stream)
+        print_item_to(self.space, self._printable_object(w_item), w_stream)
 
     def PRINT_ITEM(self, oparg, next_instr):
         w_item = self.popvalue()
-        print_item(self.space, w_item)
+        print_item(self.space, self._printable_object(w_item))
+
+    def _printable_object(self, w_obj):
+        space = self.space
+        if not space.isinstance_w(w_obj, space.w_unicode):
+            w_obj = space.str(w_obj)
+        return w_obj
 
     def PRINT_NEWLINE_TO(self, oparg, next_instr):
         w_stream = self.popvalue()
@@ -1535,9 +1541,9 @@
            stream.write(" ")
 
         # give to write() an argument which is either a string or a unicode
-        # (and let it deals itself with unicode handling)
-        if not isinstance(x, unicode):
-            x = str(x)
+        # (and let it deals itself with unicode handling).  The check "is
+        # unicode" should not use isinstance() at app-level, because that
+        # could be fooled by strange objects, so it is done at interp-level.
         stream.write(x)
 
         # add a softspace unless we just printed a string which ends in a '\t'
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -299,6 +299,30 @@
         finally:
             sys.stdout = save
 
+    def test_print_strange_object(self):
+        import sys
+
+        class A(object):
+            def __getattribute__(self, name):
+                print "seeing", name
+            def __str__(self):
+                return 'A!!'
+        save = sys.stdout
+        class Out(object):
+            def __init__(self):
+                self.data = []
+            def write(self, x):
+                self.data.append((type(x), x))
+        sys.stdout = out = Out()
+        try:
+            a = A()
+            assert out.data == []
+            print a
+            assert out.data == [(str, 'A!!'),
+                                (str, '\n')]
+        finally:
+            sys.stdout = save
+
     def test_identity(self):
         def f(x): return x
         assert f(666) == 666

From noreply at buildbot.pypy.org  Fri Jul 31 17:47:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 31 Jul 2015 17:47:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: This line looks suspiciously like a
 leftover from testing
Message-ID: <20150731154718.0D2A11C0EC1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78731:5cf79ac6a1b8
Date: 2015-07-31 17:47 +0200
http://bitbucket.org/pypy/pypy/changeset/5cf79ac6a1b8/

Log:	This line looks suspiciously like a leftover from testing

diff --git a/pypy/module/__builtin__/test/test_abstractinst.py b/pypy/module/__builtin__/test/test_abstractinst.py
--- a/pypy/module/__builtin__/test/test_abstractinst.py
+++ b/pypy/module/__builtin__/test/test_abstractinst.py
@@ -208,7 +208,6 @@
         
         class M(type):
             def __instancecheck__(self, obj):
-                saddsadsa
                 called.append("called")
 
         class C:

From noreply at buildbot.pypy.org  Fri Jul 31 19:59:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 31 Jul 2015 19:59:32 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20150731175932.3DAA41C0FF6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r624:3cd948df449b
Date: 2015-07-31 20:00 +0200
http://bitbucket.org/pypy/pypy.org/changeset/3cd948df449b/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 57.0
+      value: 57.1
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $59897 of $105000 (57.0%)
+   $59988 of $105000 (57.1%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5333 left
+    we have $5415 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 87.1
+      value: 87.4
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Numpy / 0.95 -->
-   $52285 of $60000 (87.1%)
+   $52438 of $60000 (87.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting NumPy in PyPy.
   Current status:
-    we have $11891 left
+    we have $12031 left
   in the account.</br> <a href="numpydonate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -9,7 +9,7 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 36.5
+      value: 36.6
    });
   });
 </script>
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $29225 of $80000 (36.5%)
+   $29248 of $80000 (36.6%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $22113 left
+    we have $22132 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From noreply at buildbot.pypy.org  Fri Jul 31 20:15:07 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 31 Jul 2015 20:15:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: a bit just because,
 use sets instead of dicts with None/True values
Message-ID: <20150731181507.B3B551C120F@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r78732:477e0e47a7cb
Date: 2015-07-31 20:11 +0200
http://bitbucket.org/pypy/pypy/changeset/477e0e47a7cb/

Log:	a bit just because, use sets instead of dicts with None/True values

diff --git a/rpython/translator/backendopt/malloc.py b/rpython/translator/backendopt/malloc.py
--- a/rpython/translator/backendopt/malloc.py
+++ b/rpython/translator/backendopt/malloc.py
@@ -10,9 +10,9 @@
 
     def __init__(self, (block, var)):
         assert isinstance(var, Variable)
-        self.variables = {(block, var) : True}
-        self.creationpoints = {}   # set of ("type of creation point", ...)
-        self.usepoints = {}        # set of ("type of use point",      ...)
+        self.variables = {(block, var)}
+        self.creationpoints = set()   # set of ("type of creation point", ...)
+        self.usepoints = set()        # set of ("type of use point",      ...)
 
     def absorb(self, other):
         self.variables.update(other.variables)
@@ -126,11 +126,11 @@
 
         def set_creation_point(block, var, *cp):
             _, _, info = lifetimes.find((block, var))
-            info.creationpoints[cp] = True
+            info.creationpoints.add(cp)
 
         def set_use_point(block, var, *up):
             _, _, info = lifetimes.find((block, var))
-            info.usepoints[up] = True
+            info.usepoints.add(up)
 
         def union(block1, var1, block2, var2):
             if isinstance(var1, Variable):
@@ -170,7 +170,7 @@
             if isinstance(node.last_exc_value, Variable):
                 set_creation_point(node.prevblock, node.last_exc_value,
                                    "last_exc_value")
-            d = {}
+            d = set()
             for i, arg in enumerate(node.args):
                 union(node.prevblock, arg,
                       node.target, node.target.inputargs[i])
@@ -181,7 +181,7 @@
                         # will disable malloc optimization (aliasing problems)
                         set_use_point(node.prevblock, arg, "dup", node, i)
                     else:
-                        d[arg] = True
+                        d.add(arg)
 
         return lifetimes.infos()
 
@@ -189,7 +189,7 @@
         """Try to inline the mallocs creation and manipulation of the Variables
         in the given LifeTime."""
         # the values must be only ever created by a "malloc"
-        lltypes = {}
+        lltypes = set()
         for cp in info.creationpoints:
             if cp[0] != "op":
                 return False
@@ -199,18 +199,19 @@
             if not self.inline_type(op.args[0].value):
                 return False
 
-            lltypes[op.result.concretetype] = True
+            lltypes.add(op.result.concretetype)
 
         # there must be a single largest malloced GcStruct;
         # all variables can point to it or to initial substructures
         if len(lltypes) != 1:
             return False
-        STRUCT = self.get_STRUCT(lltypes.keys()[0])
+        concretetype, = lltypes
+        STRUCT = self.get_STRUCT(concretetype)
 
         # must be only ever accessed via getfield/setfield/getsubstruct/
         # direct_fieldptr, or touched by ptr_iszero/ptr_nonzero.
         # Note that same_as and cast_pointer are not recorded in usepoints.
-        self.accessed_substructs = {}
+        self.accessed_substructs = set()
 
         for up in info.usepoints:
             if up[0] != "op":
@@ -259,8 +260,8 @@
 
         variables_by_block = {}
         for block, var in info.variables:
-            vars = variables_by_block.setdefault(block, {})
-            vars[var] = True
+            vars = variables_by_block.setdefault(block, set())
+            vars.add(var)
 
         count = [0]
 
@@ -269,10 +270,10 @@
             # look for variables arriving from outside the block
             newvarsmap = None
             newinputargs = []
-            inputvars = {}
+            inputvars = set()
             for var in block.inputargs:
                 if var in vars:
-                    inputvars[var] = None
+                    inputvars.add(var)
                     if newvarsmap is None:
                         newvarsmap = {}
                         for key in self.flatnames:
@@ -292,7 +293,7 @@
                 if self.check_malloc(op) and op.result in vars:
                     vars_created_here.append(op.result)
             for var in vars_created_here:
-                self.flowin(block, count, {var: True}, newvarsmap=None)
+                self.flowin(block, count, {var}, newvarsmap=None)
 
         return count[0]
 
@@ -370,7 +371,7 @@
         name = op.args[1].value
         if not isinstance(name, str):      # access by index
             name = 'item%d' % (name,)
-        self.accessed_substructs[S, name] = True
+        self.accessed_substructs.add((S, name))
 
     def inline_type(self, TYPE):
         return True
@@ -493,7 +494,7 @@
             else:
                 newvarsmap[key] = op.args[2]
         elif op.opname in ("same_as", "cast_pointer"):
-            vars[op.result] = True
+            vars.add(op.result)
             # Consider the two pointers (input and result) as
             # equivalent.  We can, and indeed must, use the same
             # flattened list of variables for both, as a "setfield"
@@ -508,7 +509,7 @@
             if equiv:
                 # exactly like a cast_pointer
                 assert op.result not in vars
-                vars[op.result] = True
+                vars.add(op.result)
             else:
                 # do it with a getsubstruct on the independently
                 # malloc'ed GcStruct